Commit 8b97a1aee2b161b9604d3b0c88166d0f0aef7e64 | zig

author	Andrew Kelley <andrew@ziglang.org> 2019-08-15 14:01:01-07:00
committer	GitHub <noreply@github.com> 2019-08-15 14:01:01-07:00
log	8b97a1aee2b161b9604d3b0c88166d0f0aef7e64
tree	7a8d65ad3ef59679cf4318a2fea88b6979207de4
parent	729807203a4ef162f39656be062dd11a428af8e3
parent	d3672493cc6ad5085f202df1859b13b4ae4dec96
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3033 from ziglang/rewrite-coroutines

rework async function semantics

54 files changed, 5742 insertions(+), 5189 deletions(-)

CMakeLists.txt+1-1

...	...	@@ -426,7 +426,6 @@ set(ZIG_MAIN_SRC "${CMAKE_SOURCE_DIR}/src/main.cpp")
426	426	set(ZIG0_SHIM_SRC "${CMAKE_SOURCE_DIR}/src/userland.cpp")
427	427
428	428	set(ZIG_SOURCES
429		"${CMAKE_SOURCE_DIR}/src/glibc.cpp"
430	429	"${CMAKE_SOURCE_DIR}/src/analyze.cpp"
431	430	"${CMAKE_SOURCE_DIR}/src/ast_render.cpp"
432	431	"${CMAKE_SOURCE_DIR}/src/bigfloat.cpp"
...	...	@@ -438,6 +437,7 @@ set(ZIG_SOURCES
438	437	"${CMAKE_SOURCE_DIR}/src/compiler.cpp"
439	438	"${CMAKE_SOURCE_DIR}/src/errmsg.cpp"
440	439	"${CMAKE_SOURCE_DIR}/src/error.cpp"
	440	"${CMAKE_SOURCE_DIR}/src/glibc.cpp"
441	441	"${CMAKE_SOURCE_DIR}/src/ir.cpp"
442	442	"${CMAKE_SOURCE_DIR}/src/ir_print.cpp"
443	443	"${CMAKE_SOURCE_DIR}/src/libc_installation.cpp"

build.zig+3-1

...	...	@@ -375,7 +375,9 @@ fn addLibUserlandStep(b: *Builder) void {
375	375	artifact.bundle_compiler_rt = true;
376	376	artifact.setTarget(builtin.arch, builtin.os, builtin.abi);
377	377	artifact.linkSystemLibrary("c");
378		artifact.linkSystemLibrary("ntdll");
	378	if (builtin.os == .windows) {
	379	artifact.linkSystemLibrary("ntdll");
	380	}
379	381	const libuserland_step = b.step("libuserland", "Build the userland compiler library for use in stage1");
380	382	libuserland_step.dependOn(&artifact.step);
381	383

doc/docgen.zig+1-2

...	...	@@ -750,7 +750,6 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok
750	750	.Keyword_async,
751	751	.Keyword_await,
752	752	.Keyword_break,
753		.Keyword_cancel,
754	753	.Keyword_catch,
755	754	.Keyword_comptime,
756	755	.Keyword_const,
...	...	@@ -770,7 +769,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok
770	769	.Keyword_or,
771	770	.Keyword_orelse,
772	771	.Keyword_packed,
773		.Keyword_promise,
	772	.Keyword_anyframe,
774	773	.Keyword_pub,
775	774	.Keyword_resume,
776	775	.Keyword_return,

doc/langref.html.in+332-170

...	...	@@ -5968,55 +5968,27 @@ test "global assembly" {
5968	5968	<p>TODO: @atomic rmw</p>
5969	5969	<p>TODO: builtin atomic memory ordering enum</p>
5970	5970	{#header_close#}
5971		{#header_open\|Coroutines#}
	5971	{#header_open\|Async Functions#}
5972	5972	<p>
5973		A coroutine is a generalization of a function.
	5973	When a function is called, a frame is pushed to the stack,
	5974	the function runs until it reaches a return statement, and then the frame is popped from the stack.
	5975	At the callsite, the following code does not run until the function returns.
5974	5976	</p>
5975	5977	<p>
5976		When you call a function, it creates a stack frame,
5977		and then the function runs until it reaches a return
5978		statement, and then the stack frame is destroyed.
5979		At the callsite, the next line of code does not run
5980		until the function returns.
	5978	An async function is a function whose callsite is split into an {#syntax#}async{#endsyntax#} initiation,
	5979	followed by an {#syntax#}await{#endsyntax#} completion. Its frame is
	5980	provided explicitly by the caller, and it can be suspended and resumed any number of times.
5981	5981	</p>
5982	5982	<p>
5983		A coroutine is like a function, but it can be suspended
5984		and resumed any number of times, and then it must be
5985		explicitly destroyed. When a coroutine suspends, it
5986		returns to the resumer.
5987		</p>
5988		{#header_open\|Minimal Coroutine Example#}
5989		<p>
5990		Declare a coroutine with the {#syntax#}async{#endsyntax#} keyword.
5991		The expression in angle brackets must evaluate to a struct
5992		which has these fields:
5993		</p>
5994		<ul>
5995		<li>{#syntax#}allocFn: fn (self: *Allocator, byte_count: usize, alignment: u29) Error![]u8{#endsyntax#} - where {#syntax#}Error{#endsyntax#} can be any error set.</li>
5996		<li>{#syntax#}freeFn: fn (self: *Allocator, old_mem: []u8) void{#endsyntax#}</li>
5997		</ul>
5998		<p>
5999		You may notice that this corresponds to the {#syntax#}std.mem.Allocator{#endsyntax#} interface.
6000		This makes it convenient to integrate with existing allocators. Note, however,
6001		that the language feature does not depend on the standard library, and any struct which
6002		has these fields is allowed.
6003		</p>
6004		<p>
6005		Omitting the angle bracket expression when defining an async function makes
6006		the function generic. Zig will infer the allocator type when the async function is called.
6007		</p>
6008		<p>
6009		Call a coroutine with the {#syntax#}async{#endsyntax#} keyword. Here, the expression in angle brackets
6010		is a pointer to the allocator struct that the coroutine expects.
6011		</p>
6012		<p>
6013		The result of an async function call is a {#syntax#}promise->T{#endsyntax#} type, where {#syntax#}T{#endsyntax#}
6014		is the return type of the async function. Once a promise has been created, it must be
6015		consumed, either with {#syntax#}cancel{#endsyntax#} or {#syntax#}await{#endsyntax#}:
	5983	Zig infers that a function is {#syntax#}async{#endsyntax#} when it observes that the function contains
	5984	a <strong>suspension point</strong>. Async functions can be called the same as normal functions. A
	5985	function call of an async function is a suspend point.
6016	5986	</p>
	5987	{#header_open\|Suspend and Resume#}
6017	5988	<p>
6018		Async functions start executing when created, so in the following example, the entire
6019		async function completes before it is canceled:
	5989	At any point, a function may suspend itself. This causes control flow to
	5990	return to the callsite (in the case of the first suspension),
	5991	or resumer (in the case of subsequent suspensions).
6020	5992	</p>
6021	5993	{#code_begin\|test#}
6022	5994	const std = @import("std");
...	...	@@ -6024,99 +5996,62 @@ const assert = std.debug.assert;
6024	5996
6025	5997	var x: i32 = 1;
6026	5998
6027		test "create a coroutine and cancel it" {
6028		const p = try async<std.debug.global_allocator> simpleAsyncFn();
6029		comptime assert(@typeOf(p) == promise->void);
6030		cancel p;
	5999	test "suspend with no resume" {
	6000	var frame = async func();
6031	6001	assert(x == 2);
6032	6002	}
6033		async<*std.mem.Allocator> fn simpleAsyncFn() void {
6034		x += 1;
6035		}
6036		{#code_end#}
6037		{#header_close#}
6038		{#header_open\|Suspend and Resume#}
6039		<p>
6040		At any point, an async function may suspend itself. This causes control flow to
6041		return to the caller or resumer. The following code demonstrates where control flow
6042		goes:
6043		</p>
6044		{#code_begin\|test#}
6045		const std = @import("std");
6046		const assert = std.debug.assert;
6047
6048		test "coroutine suspend, resume, cancel" {
6049		seq('a');
6050		const p = try async<std.debug.global_allocator> testAsyncSeq();
6051		seq('c');
6052		resume p;
6053		seq('f');
6054		cancel p;
6055		seq('g');
6056	6003
6057		assert(std.mem.eql(u8, points, "abcdefg"));
6058		}
6059		async fn testAsyncSeq() void {
6060		defer seq('e');
6061
6062		seq('b');
	6004	fn func() void {
	6005	x += 1;
6063	6006	suspend;
6064		seq('d');
6065		}
6066		var points = [_]u8{0} ** "abcdefg".len;
6067		var index: usize = 0;
6068
6069		fn seq(c: u8) void {
6070		points[index] = c;
6071		index += 1;
	6007	// This line is never reached because the suspend has no matching resume.
	6008	x += 1;
6072	6009	}
6073	6010	{#code_end#}
6074	6011	<p>
6075		When an async function suspends itself, it must be sure that it will be
6076		resumed or canceled somehow, for example by registering its promise handle
6077		in an event loop. Use a suspend capture block to gain access to the
6078		promise:
	6012	In the same way that each allocation should have a corresponding free,
	6013	Each {#syntax#}suspend{#endsyntax#} should have a corresponding {#syntax#}resume{#endsyntax#}.
	6014	A <strong>suspend block</strong> allows a function to put a pointer to its own
	6015	frame somewhere, for example into an event loop, even if that action will perform a
	6016	{#syntax#}resume{#endsyntax#} operation on a different thread.
	6017	{#link\|@frame#} provides access to the async function frame pointer.
6079	6018	</p>
6080	6019	{#code_begin\|test#}
6081	6020	const std = @import("std");
6082	6021	const assert = std.debug.assert;
6083	6022
6084		test "coroutine suspend with block" {
6085		const p = try async<std.debug.global_allocator> testSuspendBlock();
6086		std.debug.assert(!result);
6087		resume a_promise;
6088		std.debug.assert(result);
6089		cancel p;
	6023	var the_frame: anyframe = undefined;
	6024	var result = false;
	6025
	6026	test "async function suspend with block" {
	6027	_ = async testSuspendBlock();
	6028	assert(!result);
	6029	resume the_frame;
	6030	assert(result);
6090	6031	}
6091	6032
6092		var a_promise: promise = undefined;
6093		var result = false;
6094		async fn testSuspendBlock() void {
	6033	fn testSuspendBlock() void {
6095	6034	suspend {
6096		comptime assert(@typeOf(@handle()) == promise->void);
6097		a_promise = @handle();
	6035	comptime assert(@typeOf(@frame()) == *@Frame(testSuspendBlock));
	6036	the_frame = @frame();
6098	6037	}
6099	6038	result = true;
6100	6039	}
6101	6040	{#code_end#}
6102	6041	<p>
6103		Every suspend point in an async function represents a point at which the coroutine
6104		could be destroyed. If that happens, {#syntax#}defer{#endsyntax#} expressions that are in
6105		scope are run, as well as {#syntax#}errdefer{#endsyntax#} expressions.
6106		</p>
6107		<p>
6108		{#link\|Await#} counts as a suspend point.
	6042	{#syntax#}suspend{#endsyntax#} causes a function to be {#syntax#}async{#endsyntax#}.
6109	6043	</p>
	6044
6110	6045	{#header_open\|Resuming from Suspend Blocks#}
6111	6046	<p>
6112		Upon entering a {#syntax#}suspend{#endsyntax#} block, the coroutine is already considered
	6047	Upon entering a {#syntax#}suspend{#endsyntax#} block, the async function is already considered
6113	6048	suspended, and can be resumed. For example, if you started another kernel thread,
6114		and had that thread call {#syntax#}resume{#endsyntax#} on the promise handle provided by the
6115		{#syntax#}suspend{#endsyntax#} block, the new thread would begin executing after the suspend
	6049	and had that thread call {#syntax#}resume{#endsyntax#} on the frame pointer provided by the
	6050	{#link\|@frame#}, the new thread would begin executing after the suspend
6116	6051	block, while the old thread continued executing the suspend block.
6117	6052	</p>
6118	6053	<p>
6119		However, the coroutine can be directly resumed from the suspend block, in which case it
	6054	However, the async function can be directly resumed from the suspend block, in which case it
6120	6055	never returns to its resumer and continues executing.
6121	6056	</p>
6122	6057	{#code_begin\|test#}
...	...	@@ -6124,16 +6059,13 @@ const std = @import("std");
6124	6059	const assert = std.debug.assert;
6125	6060
6126	6061	test "resume from suspend" {
6127		var buf: [500]u8 = undefined;
6128		var a = &std.heap.FixedBufferAllocator.init(buf[0..]).allocator;
6129	6062	var my_result: i32 = 1;
6130		const p = try async<a> testResumeFromSuspend(&my_result);
6131		cancel p;
	6063	_ = async testResumeFromSuspend(&my_result);
6132	6064	std.debug.assert(my_result == 2);
6133	6065	}
6134		async fn testResumeFromSuspend(my_result: *i32) void {
	6066	fn testResumeFromSuspend(my_result: *i32) void {
6135	6067	suspend {
6136		resume @handle();
	6068	resume @frame();
6137	6069	}
6138	6070	my_result.* += 1;
6139	6071	suspend;
...	...	@@ -6141,61 +6073,88 @@ async fn testResumeFromSuspend(my_result: *i32) void {
6141	6073	}
6142	6074	{#code_end#}
6143	6075	<p>
6144		This is guaranteed to be a tail call, and therefore will not cause a new stack frame.
	6076	This is guaranteed to tail call, and therefore will not cause a new stack frame.
6145	6077	</p>
6146	6078	{#header_close#}
6147	6079	{#header_close#}
6148		{#header_open\|Await#}
	6080
	6081	{#header_open\|Async and Await#}
6149	6082	<p>
6150		The {#syntax#}await{#endsyntax#} keyword is used to coordinate with an async function's
6151		{#syntax#}return{#endsyntax#} statement.
	6083	In the same way that every {#syntax#}suspend{#endsyntax#} has a matching
	6084	{#syntax#}resume{#endsyntax#}, every {#syntax#}async{#endsyntax#} has a matching {#syntax#}await{#endsyntax#}.
6152	6085	</p>
	6086	{#code_begin\|test#}
	6087	const std = @import("std");
	6088	const assert = std.debug.assert;
	6089
	6090	test "async and await" {
	6091	// Here we have an exception where we do not match an async
	6092	// with an await. The test block is not async and so cannot
	6093	// have a suspend point in it.
	6094	// This is well-defined behavior, and everything is OK here.
	6095	// Note however that there would be no way to collect the
	6096	// return value of amain, if it were something other than void.
	6097	_ = async amain();
	6098	}
	6099
	6100	fn amain() void {
	6101	var frame = async func();
	6102	comptime assert(@typeOf(frame) == @Frame(func));
	6103
	6104	const ptr: anyframe->void = &frame;
	6105	const any_ptr: anyframe = ptr;
	6106
	6107	resume any_ptr;
	6108	await ptr;
	6109	}
	6110
	6111	fn func() void {
	6112	suspend;
	6113	}
	6114	{#code_end#}
6153	6115	<p>
6154		{#syntax#}await{#endsyntax#} is valid only in an {#syntax#}async{#endsyntax#} function, and it takes
6155		as an operand a promise handle.
6156		If the async function associated with the promise handle has already returned,
6157		then {#syntax#}await{#endsyntax#} destroys the target async function, and gives the return value.
6158		Otherwise, {#syntax#}await{#endsyntax#} suspends the current async function, registering its
6159		promise handle with the target coroutine. It becomes the target coroutine's responsibility
6160		to have ensured that it will be resumed or destroyed. When the target coroutine reaches
6161		its return statement, it gives the return value to the awaiter, destroys itself, and then
6162		resumes the awaiter.
	6116	The {#syntax#}await{#endsyntax#} keyword is used to coordinate with an async function's
	6117	{#syntax#}return{#endsyntax#} statement.
6163	6118	</p>
6164	6119	<p>
6165		A promise handle must be consumed exactly once after it is created, either by {#syntax#}cancel{#endsyntax#} or {#syntax#}await{#endsyntax#}.
	6120	{#syntax#}await{#endsyntax#} is a suspend point, and takes as an operand anything that
	6121	implicitly casts to {#syntax#}anyframe->T{#endsyntax#}.
6166	6122	</p>
6167	6123	<p>
6168		{#syntax#}await{#endsyntax#} counts as a suspend point, and therefore at every {#syntax#}await{#endsyntax#},
6169		a coroutine can be potentially destroyed, which would run {#syntax#}defer{#endsyntax#} and {#syntax#}errdefer{#endsyntax#} expressions.
	6124	There is a common misconception that {#syntax#}await{#endsyntax#} resumes the target function.
	6125	It is the other way around: it suspends until the target function completes.
	6126	In the event that the target function has already completed, {#syntax#}await{#endsyntax#}
	6127	does not suspend; instead it copies the
	6128	return value directly from the target function's frame.
6170	6129	</p>
6171	6130	{#code_begin\|test#}
6172	6131	const std = @import("std");
6173	6132	const assert = std.debug.assert;
6174	6133
6175		var a_promise: promise = undefined;
	6134	var the_frame: anyframe = undefined;
6176	6135	var final_result: i32 = 0;
6177	6136
6178		test "coroutine await" {
	6137	test "async function await" {
6179	6138	seq('a');
6180		const p = async<std.debug.global_allocator> amain() catch unreachable;
	6139	_ = async amain();
6181	6140	seq('f');
6182		resume a_promise;
	6141	resume the_frame;
6183	6142	seq('i');
6184	6143	assert(final_result == 1234);
6185	6144	assert(std.mem.eql(u8, seq_points, "abcdefghi"));
6186	6145	}
6187		async fn amain() void {
	6146	fn amain() void {
6188	6147	seq('b');
6189		const p = async another() catch unreachable;
	6148	var f = async another();
6190	6149	seq('e');
6191		final_result = await p;
	6150	final_result = await f;
6192	6151	seq('h');
6193	6152	}
6194		async fn another() i32 {
	6153	fn another() i32 {
6195	6154	seq('c');
6196	6155	suspend {
6197	6156	seq('d');
6198		a_promise = @handle();
	6157	the_frame = @frame();
6199	6158	}
6200	6159	seq('g');
6201	6160	return 1234;
...	...	@@ -6211,31 +6170,156 @@ fn seq(c: u8) void {
6211	6170	{#code_end#}
6212	6171	<p>
6213	6172	In general, {#syntax#}suspend{#endsyntax#} is lower level than {#syntax#}await{#endsyntax#}. Most application
6214		code will use only {#syntax#}async{#endsyntax#} and {#syntax#}await{#endsyntax#}, but event loop
6215		implementations will make use of {#syntax#}suspend{#endsyntax#} internally.
	6173	code will use only {#syntax#}async{#endsyntax#} and {#syntax#}await{#endsyntax#}, but event loop
	6174	implementations will make use of {#syntax#}suspend{#endsyntax#} internally.
6216	6175	</p>
6217	6176	{#header_close#}
6218		{#header_open\|Open Issues#}
	6177
	6178	{#header_open\|Async Function Example#}
6219	6179	<p>
6220		There are a few issues with coroutines that are considered unresolved. Best be aware of them,
6221		as the situation is likely to change before 1.0.0:
	6180	Putting all of this together, here is an example of typical
	6181	{#syntax#}async{#endsyntax#}/{#syntax#}await{#endsyntax#} usage:
	6182	</p>
	6183	{#code_begin\|exe\|async#}
	6184	const std = @import("std");
	6185	const Allocator = std.mem.Allocator;
	6186
	6187	pub fn main() void {
	6188	_ = async amainWrap();
	6189
	6190	// Typically we would use an event loop to manage resuming async functions,
	6191	// but in this example we hard code what the event loop would do,
	6192	// to make things deterministic.
	6193	resume global_file_frame;
	6194	resume global_download_frame;
	6195	}
	6196
	6197	fn amainWrap() void {
	6198	amain() catch \|e\| {
	6199	std.debug.warn("{}\n", e);
	6200	if (@errorReturnTrace()) \|trace\| {
	6201	std.debug.dumpStackTrace(trace.*);
	6202	}
	6203	std.process.exit(1);
	6204	};
	6205	}
	6206
	6207	fn amain() !void {
	6208	const allocator = std.heap.direct_allocator;
	6209	var download_frame = async fetchUrl(allocator, "https://example.com/");
	6210	var awaited_download_frame = false;
	6211	errdefer if (!awaited_download_frame) {
	6212	if (await download_frame) \|r\| allocator.free(r) else \|_\| {}
	6213	};
	6214
	6215	var file_frame = async readFile(allocator, "something.txt");
	6216	var awaited_file_frame = false;
	6217	errdefer if (!awaited_file_frame) {
	6218	if (await file_frame) \|r\| allocator.free(r) else \|_\| {}
	6219	};
	6220
	6221	awaited_file_frame = true;
	6222	const file_text = try await file_frame;
	6223	defer allocator.free(file_text);
	6224
	6225	awaited_download_frame = true;
	6226	const download_text = try await download_frame;
	6227	defer allocator.free(download_text);
	6228
	6229	std.debug.warn("download_text: {}\n", download_text);
	6230	std.debug.warn("file_text: {}\n", file_text);
	6231	}
	6232
	6233	var global_download_frame: anyframe = undefined;
	6234	fn fetchUrl(allocator: *Allocator, url: []const u8) ![]u8 {
	6235	const result = try std.mem.dupe(allocator, u8, "this is the downloaded url contents");
	6236	errdefer allocator.free(result);
	6237	suspend {
	6238	global_download_frame = @frame();
	6239	}
	6240	std.debug.warn("fetchUrl returning\n");
	6241	return result;
	6242	}
	6243
	6244	var global_file_frame: anyframe = undefined;
	6245	fn readFile(allocator: *Allocator, filename: []const u8) ![]u8 {
	6246	const result = try std.mem.dupe(allocator, u8, "this is the file contents");
	6247	errdefer allocator.free(result);
	6248	suspend {
	6249	global_file_frame = @frame();
	6250	}
	6251	std.debug.warn("readFile returning\n");
	6252	return result;
	6253	}
	6254	{#code_end#}
	6255	<p>
	6256	Now we remove the {#syntax#}suspend{#endsyntax#} and {#syntax#}resume{#endsyntax#} code, and
	6257	observe the same behavior, with one tiny difference:
	6258	</p>
	6259	{#code_begin\|exe\|blocking#}
	6260	const std = @import("std");
	6261	const Allocator = std.mem.Allocator;
	6262
	6263	pub fn main() void {
	6264	_ = async amainWrap();
	6265	}
	6266
	6267	fn amainWrap() void {
	6268	amain() catch \|e\| {
	6269	std.debug.warn("{}\n", e);
	6270	if (@errorReturnTrace()) \|trace\| {
	6271	std.debug.dumpStackTrace(trace.*);
	6272	}
	6273	std.process.exit(1);
	6274	};
	6275	}
	6276
	6277	fn amain() !void {
	6278	const allocator = std.heap.direct_allocator;
	6279	var download_frame = async fetchUrl(allocator, "https://example.com/");
	6280	var awaited_download_frame = false;
	6281	errdefer if (!awaited_download_frame) {
	6282	if (await download_frame) \|r\| allocator.free(r) else \|_\| {}
	6283	};
	6284
	6285	var file_frame = async readFile(allocator, "something.txt");
	6286	var awaited_file_frame = false;
	6287	errdefer if (!awaited_file_frame) {
	6288	if (await file_frame) \|r\| allocator.free(r) else \|_\| {}
	6289	};
	6290
	6291	awaited_file_frame = true;
	6292	const file_text = try await file_frame;
	6293	defer allocator.free(file_text);
	6294
	6295	awaited_download_frame = true;
	6296	const download_text = try await download_frame;
	6297	defer allocator.free(download_text);
	6298
	6299	std.debug.warn("download_text: {}\n", download_text);
	6300	std.debug.warn("file_text: {}\n", file_text);
	6301	}
	6302
	6303	fn fetchUrl(allocator: *Allocator, url: []const u8) ![]u8 {
	6304	const result = try std.mem.dupe(allocator, u8, "this is the downloaded url contents");
	6305	errdefer allocator.free(result);
	6306	std.debug.warn("fetchUrl returning\n");
	6307	return result;
	6308	}
	6309
	6310	fn readFile(allocator: *Allocator, filename: []const u8) ![]u8 {
	6311	const result = try std.mem.dupe(allocator, u8, "this is the file contents");
	6312	errdefer allocator.free(result);
	6313	std.debug.warn("readFile returning\n");
	6314	return result;
	6315	}
	6316	{#code_end#}
	6317	<p>
	6318	Previously, the {#syntax#}fetchUrl{#endsyntax#} and {#syntax#}readFile{#endsyntax#} functions suspended,
	6319	and were resumed in an order determined by the {#syntax#}main{#endsyntax#} function. Now,
	6320	since there are no suspend points, the order of the printed "... returning" messages
	6321	is determined by the order of {#syntax#}async{#endsyntax#} callsites.
6222	6322	</p>
6223		<ul>
6224		<li>Async functions have optimizations disabled - even in release modes - due to an
6225		<a href="https://github.com/ziglang/zig/issues/802">LLVM bug</a>.
6226		</li>
6227		<li>
6228		There are some situations where we can know statically that there will not be
6229		memory allocation failure, but Zig still forces us to handle it.
6230		TODO file an issue for this and link it here.
6231		</li>
6232		<li>
6233		Zig does not take advantage of LLVM's allocation elision optimization for
6234		coroutines. It crashed LLVM when I tried to do it the first time. This is
6235		related to the other 2 bullet points here. See
6236		<a href="https://github.com/ziglang/zig/issues/802">#802</a>.
6237		</li>
6238		</ul>
6239	6323	{#header_close#}
6240	6324
6241	6325	{#header_close#}
...	...	@@ -6293,6 +6377,49 @@ comptime {
6293	6377	Note: This function is deprecated. Use {#link\|@typeInfo#} instead.
6294	6378	</p>
6295	6379	{#header_close#}
	6380
	6381	{#header_open\|@asyncCall#}
	6382	<pre>{#syntax#}@asyncCall(frame_buffer: []u8, result_ptr, function_ptr, args: ...) anyframe->T{#endsyntax#}</pre>
	6383	<p>
	6384	{#syntax#}@asyncCall{#endsyntax#} performs an {#syntax#}async{#endsyntax#} call on a function pointer,
	6385	which may or may not be an {#link\|async function\|Async Functions#}.
	6386	</p>
	6387	<p>
	6388	The provided {#syntax#}frame_buffer{#endsyntax#} must be large enough to fit the entire function frame.
	6389	This size can be determined with {#link\|@frameSize#}. To provide a too-small buffer
	6390	invokes safety-checked {#link\|Undefined Behavior#}.
	6391	</p>
	6392	<p>
	6393	{#syntax#}result_ptr{#endsyntax#} is optional ({#link\|null#} may be provided). If provided,
	6394	the function call will write its result directly to the result pointer, which will be available to
	6395	read after {#link\|await\|Async and Await#} completes. Any result location provided to
	6396	{#syntax#}await{#endsyntax#} will copy the result from {#syntax#}result_ptr{#endsyntax#}.
	6397	</p>
	6398	{#code_begin\|test#}
	6399	const std = @import("std");
	6400	const assert = std.debug.assert;
	6401
	6402	test "async fn pointer in a struct field" {
	6403	var data: i32 = 1;
	6404	const Foo = struct {
	6405	bar: async fn (*i32) void,
	6406	};
	6407	var foo = Foo{ .bar = func };
	6408	var bytes: [64]u8 = undefined;
	6409	const f = @asyncCall(&bytes, {}, foo.bar, &data);
	6410	assert(data == 2);
	6411	resume f;
	6412	assert(data == 4);
	6413	}
	6414
	6415	async fn func(y: *i32) void {
	6416	defer y.* += 2;
	6417	y.* += 1;
	6418	suspend;
	6419	}
	6420	{#code_end#}
	6421	{#header_close#}
	6422
6296	6423	{#header_open\|@atomicLoad#}
6297	6424	<pre>{#syntax#}@atomicLoad(comptime T: type, ptr: *const T, comptime ordering: builtin.AtomicOrder) T{#endsyntax#}</pre>
6298	6425	<p>
...	...	@@ -6883,6 +7010,44 @@ export fn @"A function name that is a complete sentence."() void {}
6883	7010	{#see_also\|@intToFloat#}
6884	7011	{#header_close#}
6885	7012
	7013	{#header_open\|@frame#}
	7014	<pre>{#syntax#}@frame() *@Frame(func){#endsyntax#}</pre>
	7015	<p>
	7016	This function returns a pointer to the frame for a given function. This type
	7017	can be {#link\|implicitly cast\|Implicit Casts#} to {#syntax#}anyframe->T{#endsyntax#} and
	7018	to {#syntax#}anyframe{#endsyntax#}, where {#syntax#}T{#endsyntax#} is the return type
	7019	of the function in scope.
	7020	</p>
	7021	<p>
	7022	This function does not mark a suspension point, but it does cause the function in scope
	7023	to become an {#link\|async function\|Async Functions#}.
	7024	</p>
	7025	{#header_close#}
	7026
	7027	{#header_open\|@Frame#}
	7028	<pre>{#syntax#}@Frame(func: var) type{#endsyntax#}</pre>
	7029	<p>
	7030	This function returns the frame type of a function. This works for {#link\|Async Functions#}
	7031	as well as any function without a specific calling convention.
	7032	</p>
	7033	<p>
	7034	This type is suitable to be used as the return type of {#link\|async\|Async and Await#} which
	7035	allows one to, for example, heap-allocate an async function frame:
	7036	</p>
	7037	{#code_begin\|test#}
	7038	const std = @import("std");
	7039
	7040	test "heap allocated frame" {
	7041	const frame = try std.heap.direct_allocator.create(@Frame(func));
	7042	frame.* = async func();
	7043	}
	7044
	7045	fn func() void {
	7046	suspend;
	7047	}
	7048	{#code_end#}
	7049	{#header_close#}
	7050
6886	7051	{#header_open\|@frameAddress#}
6887	7052	<pre>{#syntax#}@frameAddress() usize{#endsyntax#}</pre>
6888	7053	<p>
...	...	@@ -6898,14 +7063,14 @@ export fn @"A function name that is a complete sentence."() void {}
6898	7063	</p>
6899	7064	{#header_close#}
6900	7065
6901		{#header_open\|@handle#}
6902		<pre>{#syntax#}@handle(){#endsyntax#}</pre>
	7066	{#header_open\|@frameSize#}
	7067	<pre>{#syntax#}@frameSize() usize{#endsyntax#}</pre>
6903	7068	<p>
6904		This function returns a {#syntax#}promise->T{#endsyntax#} type, where {#syntax#}T{#endsyntax#}
6905		is the return type of the async function in scope.
	7069	This is the same as {#syntax#}@sizeOf(@Frame(func)){#endsyntax#}, where {#syntax#}func{#endsyntax#}
	7070	may be runtime-known.
6906	7071	</p>
6907	7072	<p>
6908		This function is only valid within an async function scope.
	7073	This function is typically used in conjunction with {#link\|@asyncCall#}.
6909	7074	</p>
6910	7075	{#header_close#}
6911	7076
...	...	@@ -8045,8 +8210,7 @@ pub fn build(b: *Builder) void {
8045	8210	<p>Zig has a compile option <code>--single-threaded</code> which has the following effects:
8046	8211	<ul>
8047	8212	<li>All {#link\|Thread Local Variables#} are treated as {#link\|Global Variables#}.</li>
8048		<li>The overhead of {#link\|Coroutines#} becomes equivalent to function call overhead.
8049		TODO: please note this will not be implemented until the upcoming Coroutine Rewrite</li>
	8213	<li>The overhead of {#link\|Async Functions#} becomes equivalent to function call overhead.</li>
8050	8214	<li>The {#syntax#}@import("builtin").single_threaded{#endsyntax#} becomes {#syntax#}true{#endsyntax#}
8051	8215	and therefore various userland APIs which read this variable become more efficient.
8052	8216	For example {#syntax#}std.Mutex{#endsyntax#} becomes
...	...	@@ -9793,7 +9957,6 @@ PrimaryExpr
9793	9957	<- AsmExpr
9794	9958	/ IfExpr
9795	9959	/ KEYWORD_break BreakLabel? Expr?
9796		/ KEYWORD_cancel Expr
9797	9960	/ KEYWORD_comptime Expr
9798	9961	/ KEYWORD_continue BreakLabel?
9799	9962	/ KEYWORD_resume Expr
...	...	@@ -10149,7 +10312,6 @@ KEYWORD_asm <- 'asm' end_of_word
10149	10312	KEYWORD_async <- 'async' end_of_word
10150	10313	KEYWORD_await <- 'await' end_of_word
10151	10314	KEYWORD_break <- 'break' end_of_word
10152		KEYWORD_cancel <- 'cancel' end_of_word
10153	10315	KEYWORD_catch <- 'catch' end_of_word
10154	10316	KEYWORD_comptime <- 'comptime' end_of_word
10155	10317	KEYWORD_const <- 'const' end_of_word
...	...	@@ -10194,7 +10356,7 @@ KEYWORD_volatile <- 'volatile' end_of_word
10194	10356	KEYWORD_while <- 'while' end_of_word
10195	10357
10196	10358	keyword <- KEYWORD_align / KEYWORD_and / KEYWORD_allowzero / KEYWORD_asm
10197		/ KEYWORD_async / KEYWORD_await / KEYWORD_break / KEYWORD_cancel
	10359	/ KEYWORD_async / KEYWORD_await / KEYWORD_break
10198	10360	/ KEYWORD_catch / KEYWORD_comptime / KEYWORD_const / KEYWORD_continue
10199	10361	/ KEYWORD_defer / KEYWORD_else / KEYWORD_enum / KEYWORD_errdefer
10200	10362	/ KEYWORD_error / KEYWORD_export / KEYWORD_extern / KEYWORD_false

src-self-hosted/ir.zig-14

...	...	@@ -1904,20 +1904,6 @@ pub const Builder = struct {
1904	1904	}
1905	1905	return error.Unimplemented;
1906	1906
1907		//ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_field_ptr, return_value);
1908		//IrInstruction *promise_type_val = ir_build_const_type(irb, scope, node,
1909		// get_optional_type(irb->codegen, irb->codegen->builtin_types.entry_promise));
1910		//// TODO replace replacement_value with @intToPtr(?promise, 0x1) when it doesn't crash zig
1911		//IrInstruction *replacement_value = irb->exec->coro_handle;
1912		//IrInstruction *maybe_await_handle = ir_build_atomic_rmw(irb, scope, node,
1913		// promise_type_val, irb->exec->coro_awaiter_field_ptr, nullptr, replacement_value, nullptr,
1914		// AtomicRmwOp_xchg, AtomicOrderSeqCst);
1915		//ir_build_store_ptr(irb, scope, node, irb->exec->await_handle_var_ptr, maybe_await_handle);
1916		//IrInstruction *is_non_null = ir_build_test_nonnull(irb, scope, node, maybe_await_handle);
1917		//IrInstruction *is_comptime = ir_build_const_bool(irb, scope, node, false);
1918		//return ir_build_cond_br(irb, scope, node, is_non_null, irb->exec->coro_normal_final, irb->exec->coro_early_final,
1919		// is_comptime);
1920		//// the above blocks are rendered by ir_gen after the rest of codegen
1921	1907	}
1922	1908
1923	1909	const Ident = union(enum) {

src-self-hosted/link.zig+1-1

...	...	@@ -627,7 +627,7 @@ fn constructLinkerArgsWasm(ctx: *Context) void {
627	627
628	628	fn addFnObjects(ctx: *Context) !void {
629	629	// at this point it's guaranteed nobody else has this lock, so we circumvent it
630		// and avoid having to be a coroutine
	630	// and avoid having to be an async function
631	631	const fn_link_set = &ctx.comp.fn_link_set.private_data;
632	632
633	633	var it = fn_link_set.first;

src-self-hosted/main.zig+9-12

...	...	@@ -52,7 +52,7 @@ const Command = struct {
52	52
53	53	pub fn main() !void {
54	54	// This allocator needs to be thread-safe because we use it for the event.Loop
55		// which multiplexes coroutines onto kernel threads.
	55	// which multiplexes async functions onto kernel threads.
56	56	// libc allocator is guaranteed to have this property.
57	57	const allocator = std.heap.c_allocator;
58	58
...	...	@@ -466,8 +466,7 @@ fn buildOutputType(allocator: *Allocator, args: []const []const u8, out_type: Co
466	466	comp.link_objects = link_objects;
467	467
468	468	comp.start();
469		const process_build_events_handle = try async<loop.allocator> processBuildEvents(comp, color);
470		defer cancel process_build_events_handle;
	469	// TODO const process_build_events_handle = try async<loop.allocator> processBuildEvents(comp, color);
471	470	loop.run();
472	471	}
473	472
...	...	@@ -578,8 +577,7 @@ fn cmdLibC(allocator: *Allocator, args: []const []const u8) !void {
578	577	var zig_compiler = try ZigCompiler.init(&loop);
579	578	defer zig_compiler.deinit();
580	579
581		const handle = try async<loop.allocator> findLibCAsync(&zig_compiler);
582		defer cancel handle;
	580	// TODO const handle = try async<loop.allocator> findLibCAsync(&zig_compiler);
583	581
584	582	loop.run();
585	583	}
...	...	@@ -663,13 +661,12 @@ fn cmdFmt(allocator: *Allocator, args: []const []const u8) !void {
663	661	defer loop.deinit();
664	662
665	663	var result: FmtError!void = undefined;
666		const main_handle = try async<allocator> asyncFmtMainChecked(
667		&result,
668		&loop,
669		&flags,
670		color,
671		);
672		defer cancel main_handle;
	664	// TODO const main_handle = try async<allocator> asyncFmtMainChecked(
	665	// TODO &result,
	666	// TODO &loop,
	667	// TODO &flags,
	668	// TODO color,
	669	// TODO );
673	670	loop.run();
674	671	return result;
675	672	}

src-self-hosted/stage1.zig+2-1

...	...	@@ -142,7 +142,8 @@ export fn stage2_render_ast(tree: ast.Tree, output_file: FILE) Error {
142	142	return Error.None;
143	143	}
144	144
145		// TODO: just use the actual self-hosted zig fmt. Until the coroutine rewrite, we use a blocking implementation.
	145	// TODO: just use the actual self-hosted zig fmt. Until https://github.com/ziglang/zig/issues/2377,
	146	// we use a blocking implementation.
146	147	export fn stage2_fmt(argc: c_int, argv: []const []const u8) c_int {
147	148	if (std.debug.runtime_safety) {
148	149	fmtMain(argc, argv) catch unreachable;

src/all_types.hpp+161-216

...	...	@@ -35,6 +35,7 @@ struct ConstExprValue;
35	35	struct IrInstruction;
36	36	struct IrInstructionCast;
37	37	struct IrInstructionAllocaGen;
	38	struct IrInstructionCallGen;
38	39	struct IrBasicBlock;
39	40	struct ScopeDecls;
40	41	struct ZigWindowsSDK;
...	...	@@ -70,20 +71,10 @@ struct IrExecutable {
70	71	Scope *begin_scope;
71	72	ZigList<Tld *> tld_list;
72	73
73		IrInstruction *coro_handle;
74		IrInstruction *atomic_state_field_ptr; // this one is shared and in the promise
75		IrInstruction *coro_result_ptr_field_ptr;
76		IrInstruction *coro_result_field_ptr;
77		IrInstruction *await_handle_var_ptr; // this one is where we put the one we extracted from the promise
78		IrBasicBlock *coro_early_final;
79		IrBasicBlock *coro_normal_final;
80		IrBasicBlock *coro_suspend_block;
81		IrBasicBlock *coro_final_cleanup_block;
82		ZigVar *coro_allocator_var;
83
84	74	bool invalid;
85	75	bool is_inline;
86	76	bool is_generic_instantiation;
	77	bool need_err_code_spill;
87	78	};
88	79
89	80	enum OutType {
...	...	@@ -485,11 +476,10 @@ enum NodeType {
485	476	NodeTypeIfErrorExpr,
486	477	NodeTypeIfOptional,
487	478	NodeTypeErrorSetDecl,
488		NodeTypeCancel,
489	479	NodeTypeResume,
490	480	NodeTypeAwaitExpr,
491	481	NodeTypeSuspend,
492		NodeTypePromiseType,
	482	NodeTypeAnyFrameType,
493	483	NodeTypeEnumLiteral,
494	484	};
495	485
...	...	@@ -522,7 +512,6 @@ struct AstNodeFnProto {
522	512	AstNode *section_expr;
523	513
524	514	bool auto_err_set;
525		AstNode *async_allocator_type;
526	515	};
527	516
528	517	struct AstNodeFnDef {
...	...	@@ -657,7 +646,6 @@ struct AstNodeFnCallExpr {
657	646	bool is_builtin;
658	647	bool is_async;
659	648	bool seen; // used by @compileLog
660		AstNode *async_allocator;
661	649	};
662	650
663	651	struct AstNodeArrayAccessExpr {
...	...	@@ -922,10 +910,6 @@ struct AstNodeBreakExpr {
922	910	AstNode *expr; // may be null
923	911	};
924	912
925		struct AstNodeCancelExpr {
926		AstNode *expr;
927		};
928
929	913	struct AstNodeResumeExpr {
930	914	AstNode *expr;
931	915	};
...	...	@@ -949,7 +933,7 @@ struct AstNodeSuspend {
949	933	AstNode *block;
950	934	};
951	935
952		struct AstNodePromiseType {
	936	struct AstNodeAnyFrameType {
953	937	AstNode *payload_type; // can be NULL
954	938	};
955	939
...	...	@@ -1014,13 +998,16 @@ struct AstNode {
1014	998	AstNodeInferredArrayType inferred_array_type;
1015	999	AstNodeErrorType error_type;
1016	1000	AstNodeErrorSetDecl err_set_decl;
1017		AstNodeCancelExpr cancel_expr;
1018	1001	AstNodeResumeExpr resume_expr;
1019	1002	AstNodeAwaitExpr await_expr;
1020	1003	AstNodeSuspend suspend;
1021		AstNodePromiseType promise_type;
	1004	AstNodeAnyFrameType anyframe_type;
1022	1005	AstNodeEnumLiteral enum_literal;
1023	1006	} data;
	1007
	1008	// This is a function for use in the debugger to print
	1009	// the source location.
	1010	void src();
1024	1011	};
1025	1012
1026	1013	// this struct is allocated with allocate_nonzero
...	...	@@ -1047,7 +1034,6 @@ struct FnTypeId {
1047	1034	bool is_var_args;
1048	1035	CallingConvention cc;
1049	1036	uint32_t alignment;
1050		ZigType *async_allocator_type;
1051	1037	};
1052	1038
1053	1039	uint32_t fn_type_id_hash(FnTypeId*);
...	...	@@ -1095,6 +1081,7 @@ struct TypeStructField {
1095	1081	ConstExprValue *init_val; // null and then memoized
1096	1082	uint32_t bit_offset_in_host; // offset from the memory at gen_index
1097	1083	uint32_t host_int_bytes; // size of host integer
	1084	uint32_t align;
1098	1085	};
1099	1086
1100	1087	enum ResolveStatus {
...	...	@@ -1156,6 +1143,8 @@ struct ZigTypeOptional {
1156	1143	struct ZigTypeErrorUnion {
1157	1144	ZigType *err_set_type;
1158	1145	ZigType *payload_type;
	1146	size_t pad_bytes;
	1147	LLVMTypeRef pad_llvm_type;
1159	1148	};
1160	1149
1161	1150	struct ZigTypeErrorSet {
...	...	@@ -1241,11 +1230,6 @@ struct ZigTypeBoundFn {
1241	1230	ZigType *fn_type;
1242	1231	};
1243	1232
1244		struct ZigTypePromise {
1245		// null if `promise` instead of `promise->T`
1246		ZigType *result_type;
1247		};
1248
1249	1233	struct ZigTypeVector {
1250	1234	// The type must be a pointer, integer, or float
1251	1235	ZigType *elem_type;
...	...	@@ -1276,7 +1260,8 @@ enum ZigTypeId {
1276	1260	ZigTypeIdBoundFn,
1277	1261	ZigTypeIdArgTuple,
1278	1262	ZigTypeIdOpaque,
1279		ZigTypeIdPromise,
	1263	ZigTypeIdFnFrame,
	1264	ZigTypeIdAnyFrame,
1280	1265	ZigTypeIdVector,
1281	1266	ZigTypeIdEnumLiteral,
1282	1267	};
...	...	@@ -1291,6 +1276,15 @@ struct ZigTypeOpaque {
1291	1276	Buf *bare_name;
1292	1277	};
1293	1278
	1279	struct ZigTypeFnFrame {
	1280	ZigFn *fn;
	1281	ZigType *locals_struct;
	1282	};
	1283
	1284	struct ZigTypeAnyFrame {
	1285	ZigType *result_type; // null if `anyframe` instead of `anyframe->T`
	1286	};
	1287
1294	1288	struct ZigType {
1295	1289	ZigTypeId id;
1296	1290	Buf name;
...	...	@@ -1314,16 +1308,16 @@ struct ZigType {
1314	1308	ZigTypeUnion unionation;
1315	1309	ZigTypeFn fn;
1316	1310	ZigTypeBoundFn bound_fn;
1317		ZigTypePromise promise;
1318	1311	ZigTypeVector vector;
1319	1312	ZigTypeOpaque opaque;
	1313	ZigTypeFnFrame frame;
	1314	ZigTypeAnyFrame any_frame;
1320	1315	} data;
1321	1316
1322	1317	// use these fields to make sure we don't duplicate type table entries for the same type
1323	1318	ZigType *pointer_parent[2]; // [0 - mut, 1 - const]
1324	1319	ZigType *optional_parent;
1325		ZigType *promise_parent;
1326		ZigType *promise_frame_parent;
	1320	ZigType *any_frame_parent;
1327	1321	// If we generate a constant name value for this type, we memoize it here.
1328	1322	// The type of this is array
1329	1323	ConstExprValue *cached_const_name_val;
...	...	@@ -1359,7 +1353,6 @@ struct GlobalExport {
1359	1353	};
1360	1354
1361	1355	struct ZigFn {
1362		CodeGen *codegen;
1363	1356	LLVMValueRef llvm_value;
1364	1357	const char *llvm_name;
1365	1358	AstNode *proto_node;
...	...	@@ -1368,7 +1361,17 @@ struct ZigFn {
1368	1361	Scope *child_scope; // parent is scope for last parameter
1369	1362	ScopeBlock *def_scope; // parent is child_scope
1370	1363	Buf symbol_name;
1371		ZigType *type_entry; // function type
	1364	// This is the function type assuming the function does not suspend.
	1365	// Note that for an async function, this can be shared with non-async functions. So the value here
	1366	// should only be read for things in common between non-async and async function types.
	1367	ZigType *type_entry;
	1368	// For normal functions one could use the type_entry->raw_type_ref and type_entry->raw_di_type.
	1369	// However for functions that suspend, those values could possibly be their non-suspending equivalents.
	1370	// So these values should be preferred.
	1371	LLVMTypeRef raw_type_ref;
	1372	ZigLLVMDIType *raw_di_type;
	1373
	1374	ZigType *frame_type;
1372	1375	// in the case of normal functions this is the implicit return type
1373	1376	// in the case of async functions this is the implicit return type according to the
1374	1377	// zig source code, not according to zig ir
...	...	@@ -1379,6 +1382,7 @@ struct ZigFn {
1379	1382	size_t prealloc_backward_branch_quota;
1380	1383	AstNode **param_source_nodes;
1381	1384	Buf **param_names;
	1385	IrInstruction *err_code_spill;
1382	1386
1383	1387	AstNode *fn_no_inline_set_node;
1384	1388	AstNode *fn_static_eval_set_node;
...	...	@@ -1390,8 +1394,11 @@ struct ZigFn {
1390	1394	AstNode *set_alignstack_node;
1391	1395
1392	1396	AstNode *set_cold_node;
	1397	const AstNode *inferred_async_node;
	1398	ZigFn *inferred_async_fn;
1393	1399
1394	1400	ZigList<GlobalExport> export_list;
	1401	ZigList<IrInstructionCallGen *> call_list;
1395	1402
1396	1403	LLVMValueRef valgrind_client_request_array;
1397	1404
...	...	@@ -1442,8 +1449,6 @@ enum BuiltinFnId {
1442	1449	BuiltinFnIdErrName,
1443	1450	BuiltinFnIdBreakpoint,
1444	1451	BuiltinFnIdReturnAddress,
1445		BuiltinFnIdFrameAddress,
1446		BuiltinFnIdHandle,
1447	1452	BuiltinFnIdEmbedFile,
1448	1453	BuiltinFnIdCmpxchgWeak,
1449	1454	BuiltinFnIdCmpxchgStrong,
...	...	@@ -1499,6 +1504,7 @@ enum BuiltinFnId {
1499	1504	BuiltinFnIdInlineCall,
1500	1505	BuiltinFnIdNoInlineCall,
1501	1506	BuiltinFnIdNewStackCall,
	1507	BuiltinFnIdAsyncCall,
1502	1508	BuiltinFnIdTypeId,
1503	1509	BuiltinFnIdShlExact,
1504	1510	BuiltinFnIdShrExact,
...	...	@@ -1514,6 +1520,10 @@ enum BuiltinFnId {
1514	1520	BuiltinFnIdAtomicLoad,
1515	1521	BuiltinFnIdHasDecl,
1516	1522	BuiltinFnIdUnionInit,
	1523	BuiltinFnIdFrameAddress,
	1524	BuiltinFnIdFrameType,
	1525	BuiltinFnIdFrameHandle,
	1526	BuiltinFnIdFrameSize,
1517	1527	};
1518	1528
1519	1529	struct BuiltinFnEntry {
...	...	@@ -1541,6 +1551,12 @@ enum PanicMsgId {
1541	1551	PanicMsgIdBadEnumValue,
1542	1552	PanicMsgIdFloatToInt,
1543	1553	PanicMsgIdPtrCastNull,
	1554	PanicMsgIdBadResume,
	1555	PanicMsgIdBadAwait,
	1556	PanicMsgIdBadReturn,
	1557	PanicMsgIdResumedAnAwaitingFn,
	1558	PanicMsgIdFrameTooSmall,
	1559	PanicMsgIdResumedFnPendingAwait,
1544	1560
1545	1561	PanicMsgIdCount,
1546	1562	};
...	...	@@ -1701,7 +1717,13 @@ struct CodeGen {
1701	1717	LLVMTargetMachineRef target_machine;
1702	1718	ZigLLVMDIFile *dummy_di_file;
1703	1719	LLVMValueRef cur_ret_ptr;
	1720	LLVMValueRef cur_frame_ptr;
1704	1721	LLVMValueRef cur_fn_val;
	1722	LLVMValueRef cur_async_switch_instr;
	1723	LLVMValueRef cur_async_resume_index_ptr;
	1724	LLVMValueRef cur_async_awaiter_ptr;
	1725	LLVMBasicBlockRef cur_preamble_llvm_block;
	1726	size_t cur_resume_block_count;
1705	1727	LLVMValueRef cur_err_ret_trace_val_arg;
1706	1728	LLVMValueRef cur_err_ret_trace_val_stack;
1707	1729	LLVMValueRef memcpy_fn_val;
...	...	@@ -1709,28 +1731,16 @@ struct CodeGen {
1709	1731	LLVMValueRef trap_fn_val;
1710	1732	LLVMValueRef return_address_fn_val;
1711	1733	LLVMValueRef frame_address_fn_val;
1712		LLVMValueRef coro_destroy_fn_val;
1713		LLVMValueRef coro_id_fn_val;
1714		LLVMValueRef coro_alloc_fn_val;
1715		LLVMValueRef coro_size_fn_val;
1716		LLVMValueRef coro_begin_fn_val;
1717		LLVMValueRef coro_suspend_fn_val;
1718		LLVMValueRef coro_end_fn_val;
1719		LLVMValueRef coro_free_fn_val;
1720		LLVMValueRef coro_resume_fn_val;
1721		LLVMValueRef coro_save_fn_val;
1722		LLVMValueRef coro_promise_fn_val;
1723		LLVMValueRef coro_alloc_helper_fn_val;
1724		LLVMValueRef coro_frame_fn_val;
1725		LLVMValueRef merge_err_ret_traces_fn_val;
1726	1734	LLVMValueRef add_error_return_trace_addr_fn_val;
1727	1735	LLVMValueRef stacksave_fn_val;
1728	1736	LLVMValueRef stackrestore_fn_val;
1729	1737	LLVMValueRef write_register_fn_val;
	1738	LLVMValueRef merge_err_ret_traces_fn_val;
1730	1739	LLVMValueRef sp_md_node;
1731	1740	LLVMValueRef err_name_table;
1732	1741	LLVMValueRef safety_crash_err_fn;
1733	1742	LLVMValueRef return_err_fn;
	1743	LLVMTypeRef anyframe_fn_type;
1734	1744
1735	1745	// reminder: hash tables must be initialized before use
1736	1746	HashMap<Buf , ZigType , buf_hash, buf_eql_buf> import_table;
...	...	@@ -1797,12 +1807,12 @@ struct CodeGen {
1797	1807	ZigType *entry_var;
1798	1808	ZigType *entry_global_error_set;
1799	1809	ZigType *entry_arg_tuple;
1800		ZigType *entry_promise;
1801	1810	ZigType *entry_enum_literal;
	1811	ZigType *entry_any_frame;
1802	1812	} builtin_types;
	1813
1803	1814	ZigType *align_amt_type;
1804	1815	ZigType *stack_trace_type;
1805		ZigType *ptr_to_stack_trace_type;
1806	1816	ZigType *err_tag_type;
1807	1817	ZigType *test_fn_type;
1808	1818
...	...	@@ -1938,6 +1948,7 @@ struct ZigVar {
1938	1948	ZigType *var_type;
1939	1949	LLVMValueRef value_ref;
1940	1950	IrInstruction *is_comptime;
	1951	IrInstruction *ptr_instruction;
1941	1952	// which node is the declaration of the variable
1942	1953	AstNode *decl_node;
1943	1954	ZigLLVMDILocalVariable *di_loc_var;
...	...	@@ -1985,7 +1996,6 @@ enum ScopeId {
1985	1996	ScopeIdSuspend,
1986	1997	ScopeIdFnDef,
1987	1998	ScopeIdCompTime,
1988		ScopeIdCoroPrelude,
1989	1999	ScopeIdRuntime,
1990	2000	};
1991	2001
...	...	@@ -2109,7 +2119,6 @@ struct ScopeRuntime {
2109	2119	struct ScopeSuspend {
2110	2120	Scope base;
2111	2121
2112		IrBasicBlock *resume_block;
2113	2122	bool reported_err;
2114	2123	};
2115	2124
...	...	@@ -2128,12 +2137,6 @@ struct ScopeFnDef {
2128	2137	ZigFn *fn_entry;
2129	2138	};
2130	2139
2131		// This scope is created to indicate that the code in the scope
2132		// is auto-generated coroutine prelude stuff.
2133		struct ScopeCoroPrelude {
2134		Scope base;
2135		};
2136
2137	2140	// synchronized with code in define_builtin_compile_vars
2138	2141	enum AtomicOrder {
2139	2142	AtomicOrderUnordered,
...	...	@@ -2231,7 +2234,7 @@ enum IrInstructionId {
2231	2234	IrInstructionIdSetRuntimeSafety,
2232	2235	IrInstructionIdSetFloatMode,
2233	2236	IrInstructionIdArrayType,
2234		IrInstructionIdPromiseType,
	2237	IrInstructionIdAnyFrameType,
2235	2238	IrInstructionIdSliceType,
2236	2239	IrInstructionIdGlobalAsm,
2237	2240	IrInstructionIdAsm,
...	...	@@ -2278,7 +2281,10 @@ enum IrInstructionId {
2278	2281	IrInstructionIdBreakpoint,
2279	2282	IrInstructionIdReturnAddress,
2280	2283	IrInstructionIdFrameAddress,
2281		IrInstructionIdHandle,
	2284	IrInstructionIdFrameHandle,
	2285	IrInstructionIdFrameType,
	2286	IrInstructionIdFrameSizeSrc,
	2287	IrInstructionIdFrameSizeGen,
2282	2288	IrInstructionIdAlignOf,
2283	2289	IrInstructionIdOverflowOp,
2284	2290	IrInstructionIdTestErrSrc,
...	...	@@ -2321,35 +2327,16 @@ enum IrInstructionId {
2321	2327	IrInstructionIdImplicitCast,
2322	2328	IrInstructionIdResolveResult,
2323	2329	IrInstructionIdResetResult,
2324		IrInstructionIdResultPtr,
2325	2330	IrInstructionIdOpaqueType,
2326	2331	IrInstructionIdSetAlignStack,
2327	2332	IrInstructionIdArgType,
2328	2333	IrInstructionIdExport,
2329	2334	IrInstructionIdErrorReturnTrace,
2330	2335	IrInstructionIdErrorUnion,
2331		IrInstructionIdCancel,
2332		IrInstructionIdGetImplicitAllocator,
2333		IrInstructionIdCoroId,
2334		IrInstructionIdCoroAlloc,
2335		IrInstructionIdCoroSize,
2336		IrInstructionIdCoroBegin,
2337		IrInstructionIdCoroAllocFail,
2338		IrInstructionIdCoroSuspend,
2339		IrInstructionIdCoroEnd,
2340		IrInstructionIdCoroFree,
2341		IrInstructionIdCoroResume,
2342		IrInstructionIdCoroSave,
2343		IrInstructionIdCoroPromise,
2344		IrInstructionIdCoroAllocHelper,
2345	2336	IrInstructionIdAtomicRmw,
2346	2337	IrInstructionIdAtomicLoad,
2347		IrInstructionIdPromiseResultType,
2348		IrInstructionIdAwaitBookkeeping,
2349	2338	IrInstructionIdSaveErrRetAddr,
2350	2339	IrInstructionIdAddImplicitReturnType,
2351		IrInstructionIdMergeErrRetTraces,
2352		IrInstructionIdMarkErrRetTracePtr,
2353	2340	IrInstructionIdErrSetCast,
2354	2341	IrInstructionIdToBytes,
2355	2342	IrInstructionIdFromBytes,
...	...	@@ -2365,6 +2352,13 @@ enum IrInstructionId {
2365	2352	IrInstructionIdEndExpr,
2366	2353	IrInstructionIdPtrOfArrayToSlice,
2367	2354	IrInstructionIdUnionInitNamedField,
	2355	IrInstructionIdSuspendBegin,
	2356	IrInstructionIdSuspendFinish,
	2357	IrInstructionIdAwaitSrc,
	2358	IrInstructionIdAwaitGen,
	2359	IrInstructionIdResume,
	2360	IrInstructionIdSpillBegin,
	2361	IrInstructionIdSpillEnd,
2368	2362	};
2369	2363
2370	2364	struct IrInstruction {
...	...	@@ -2607,7 +2601,6 @@ struct IrInstructionCallSrc {
2607	2601	IrInstruction **args;
2608	2602	ResultLoc *result_loc;
2609	2603
2610		IrInstruction *async_allocator;
2611	2604	IrInstruction *new_stack;
2612	2605	FnInline fn_inline;
2613	2606	bool is_async;
...	...	@@ -2622,8 +2615,8 @@ struct IrInstructionCallGen {
2622	2615	size_t arg_count;
2623	2616	IrInstruction **args;
2624	2617	IrInstruction *result_loc;
	2618	IrInstruction *frame_result_loc;
2625	2619
2626		IrInstruction *async_allocator;
2627	2620	IrInstruction *new_stack;
2628	2621	FnInline fn_inline;
2629	2622	bool is_async;
...	...	@@ -2639,7 +2632,7 @@ struct IrInstructionConst {
2639	2632	struct IrInstructionReturn {
2640	2633	IrInstruction base;
2641	2634
2642		IrInstruction *value;
	2635	IrInstruction *operand;
2643	2636	};
2644	2637
2645	2638	enum CastOp {
...	...	@@ -2744,7 +2737,7 @@ struct IrInstructionPtrType {
2744	2737	bool is_allow_zero;
2745	2738	};
2746	2739
2747		struct IrInstructionPromiseType {
	2740	struct IrInstructionAnyFrameType {
2748	2741	IrInstruction base;
2749	2742
2750	2743	IrInstruction *payload_type;
...	...	@@ -3084,8 +3077,26 @@ struct IrInstructionFrameAddress {
3084	3077	IrInstruction base;
3085	3078	};
3086	3079
3087		struct IrInstructionHandle {
	3080	struct IrInstructionFrameHandle {
	3081	IrInstruction base;
	3082	};
	3083
	3084	struct IrInstructionFrameType {
	3085	IrInstruction base;
	3086
	3087	IrInstruction *fn;
	3088	};
	3089
	3090	struct IrInstructionFrameSizeSrc {
	3091	IrInstruction base;
	3092
	3093	IrInstruction *fn;
	3094	};
	3095
	3096	struct IrInstructionFrameSizeGen {
3088	3097	IrInstruction base;
	3098
	3099	IrInstruction *fn;
3089	3100	};
3090	3101
3091	3102	enum IrOverflowOp {
...	...	@@ -3127,6 +3138,7 @@ struct IrInstructionTestErrSrc {
3127	3138	IrInstruction base;
3128	3139
3129	3140	bool resolve_err_set;
	3141	bool base_ptr_is_payload;
3130	3142	IrInstruction *base_ptr;
3131	3143	};
3132	3144
...	...	@@ -3179,7 +3191,6 @@ struct IrInstructionFnProto {
3179	3191	IrInstruction **param_types;
3180	3192	IrInstruction *align_value;
3181	3193	IrInstruction *return_type;
3182		IrInstruction *async_allocator_type_value;
3183	3194	bool is_var_args;
3184	3195	};
3185	3196
...	...	@@ -3409,95 +3420,6 @@ struct IrInstructionErrorUnion {
3409	3420	IrInstruction *payload;
3410	3421	};
3411	3422
3412		struct IrInstructionCancel {
3413		IrInstruction base;
3414
3415		IrInstruction *target;
3416		};
3417
3418		enum ImplicitAllocatorId {
3419		ImplicitAllocatorIdArg,
3420		ImplicitAllocatorIdLocalVar,
3421		};
3422
3423		struct IrInstructionGetImplicitAllocator {
3424		IrInstruction base;
3425
3426		ImplicitAllocatorId id;
3427		};
3428
3429		struct IrInstructionCoroId {
3430		IrInstruction base;
3431
3432		IrInstruction *promise_ptr;
3433		};
3434
3435		struct IrInstructionCoroAlloc {
3436		IrInstruction base;
3437
3438		IrInstruction *coro_id;
3439		};
3440
3441		struct IrInstructionCoroSize {
3442		IrInstruction base;
3443		};
3444
3445		struct IrInstructionCoroBegin {
3446		IrInstruction base;
3447
3448		IrInstruction *coro_id;
3449		IrInstruction *coro_mem_ptr;
3450		};
3451
3452		struct IrInstructionCoroAllocFail {
3453		IrInstruction base;
3454
3455		IrInstruction *err_val;
3456		};
3457
3458		struct IrInstructionCoroSuspend {
3459		IrInstruction base;
3460
3461		IrInstruction *save_point;
3462		IrInstruction *is_final;
3463		};
3464
3465		struct IrInstructionCoroEnd {
3466		IrInstruction base;
3467		};
3468
3469		struct IrInstructionCoroFree {
3470		IrInstruction base;
3471
3472		IrInstruction *coro_id;
3473		IrInstruction *coro_handle;
3474		};
3475
3476		struct IrInstructionCoroResume {
3477		IrInstruction base;
3478
3479		IrInstruction *awaiter_handle;
3480		};
3481
3482		struct IrInstructionCoroSave {
3483		IrInstruction base;
3484
3485		IrInstruction *coro_handle;
3486		};
3487
3488		struct IrInstructionCoroPromise {
3489		IrInstruction base;
3490
3491		IrInstruction *coro_handle;
3492		};
3493
3494		struct IrInstructionCoroAllocHelper {
3495		IrInstruction base;
3496
3497		IrInstruction *realloc_fn;
3498		IrInstruction *coro_size;
3499		};
3500
3501	3423	struct IrInstructionAtomicRmw {
3502	3424	IrInstruction base;
3503	3425
...	...	@@ -3519,18 +3441,6 @@ struct IrInstructionAtomicLoad {
3519	3441	AtomicOrder resolved_ordering;
3520	3442	};
3521	3443
3522		struct IrInstructionPromiseResultType {
3523		IrInstruction base;
3524
3525		IrInstruction *promise_type;
3526		};
3527
3528		struct IrInstructionAwaitBookkeeping {
3529		IrInstruction base;
3530
3531		IrInstruction *promise_result_type;
3532		};
3533
3534	3444	struct IrInstructionSaveErrRetAddr {
3535	3445	IrInstruction base;
3536	3446	};
...	...	@@ -3541,20 +3451,6 @@ struct IrInstructionAddImplicitReturnType {
3541	3451	IrInstruction *value;
3542	3452	};
3543	3453
3544		struct IrInstructionMergeErrRetTraces {
3545		IrInstruction base;
3546
3547		IrInstruction *coro_promise_ptr;
3548		IrInstruction *src_err_ret_trace_ptr;
3549		IrInstruction *dest_err_ret_trace_ptr;
3550		};
3551
3552		struct IrInstructionMarkErrRetTracePtr {
3553		IrInstruction base;
3554
3555		IrInstruction *err_ret_trace_ptr;
3556		};
3557
3558	3454	// For float ops which take a single argument
3559	3455	struct IrInstructionFloatOp {
3560	3456	IrInstruction base;
...	...	@@ -3645,6 +3541,7 @@ struct IrInstructionAllocaGen {
3645	3541
3646	3542	uint32_t align;
3647	3543	const char *name_hint;
	3544	size_t field_index;
3648	3545	};
3649	3546
3650	3547	struct IrInstructionEndExpr {
...	...	@@ -3692,6 +3589,56 @@ struct IrInstructionPtrOfArrayToSlice {
3692	3589	IrInstruction *result_loc;
3693	3590	};
3694	3591
	3592	struct IrInstructionSuspendBegin {
	3593	IrInstruction base;
	3594
	3595	LLVMBasicBlockRef resume_bb;
	3596	};
	3597
	3598	struct IrInstructionSuspendFinish {
	3599	IrInstruction base;
	3600
	3601	IrInstructionSuspendBegin *begin;
	3602	};
	3603
	3604	struct IrInstructionAwaitSrc {
	3605	IrInstruction base;
	3606
	3607	IrInstruction *frame;
	3608	ResultLoc *result_loc;
	3609	};
	3610
	3611	struct IrInstructionAwaitGen {
	3612	IrInstruction base;
	3613
	3614	IrInstruction *frame;
	3615	IrInstruction *result_loc;
	3616	};
	3617
	3618	struct IrInstructionResume {
	3619	IrInstruction base;
	3620
	3621	IrInstruction *frame;
	3622	};
	3623
	3624	enum SpillId {
	3625	SpillIdInvalid,
	3626	SpillIdRetErrCode,
	3627	};
	3628
	3629	struct IrInstructionSpillBegin {
	3630	IrInstruction base;
	3631
	3632	SpillId spill_id;
	3633	IrInstruction *operand;
	3634	};
	3635
	3636	struct IrInstructionSpillEnd {
	3637	IrInstruction base;
	3638
	3639	IrInstructionSpillBegin *begin;
	3640	};
	3641
3695	3642	enum ResultLocId {
3696	3643	ResultLocIdInvalid,
3697	3644	ResultLocIdNone,
...	...	@@ -3775,20 +3722,16 @@ static const size_t maybe_null_index = 1;
3775	3722	static const size_t err_union_payload_index = 0;
3776	3723	static const size_t err_union_err_index = 1;
3777	3724
3778		// TODO call graph analysis to find out what this number needs to be for every function
3779		// MUST BE A POWER OF TWO.
3780		static const size_t stack_trace_ptr_count = 32;
3781
3782		// these belong to the async function
3783		#define RETURN_ADDRESSES_FIELD_NAME "return_addresses"
3784		#define ERR_RET_TRACE_FIELD_NAME "err_ret_trace"
3785		#define RESULT_FIELD_NAME "result"
3786		#define ASYNC_REALLOC_FIELD_NAME "reallocFn"
3787		#define ASYNC_SHRINK_FIELD_NAME "shrinkFn"
3788		#define ATOMIC_STATE_FIELD_NAME "atomic_state"
3789		// these point to data belonging to the awaiter
3790		#define ERR_RET_TRACE_PTR_FIELD_NAME "err_ret_trace_ptr"
3791		#define RESULT_PTR_FIELD_NAME "result_ptr"
	3725	// label (grep this): [fn_frame_struct_layout]
	3726	static const size_t frame_fn_ptr_index = 0;
	3727	static const size_t frame_resume_index = 1;
	3728	static const size_t frame_awaiter_index = 2;
	3729	static const size_t frame_ret_start = 3;
	3730
	3731	// TODO https://github.com/ziglang/zig/issues/3056
	3732	// We require this to be a power of 2 so that we can use shifting rather than
	3733	// remainder division.
	3734	static const size_t stack_trace_ptr_count = 32; // Must be a power of 2.
3792	3735
3793	3736	#define NAMESPACE_SEP_CHAR '.'
3794	3737	#define NAMESPACE_SEP_STR "."
...	...	@@ -3811,11 +3754,13 @@ enum FnWalkId {
3811	3754
3812	3755	struct FnWalkAttrs {
3813	3756	ZigFn *fn;
	3757	LLVMValueRef llvm_fn;
3814	3758	unsigned gen_i;
3815	3759	};
3816	3760
3817	3761	struct FnWalkCall {
3818	3762	ZigList<LLVMValueRef> *gen_param_values;
	3763	ZigList<ZigType > gen_param_types;
3819	3764	IrInstructionCallGen *inst;
3820	3765	bool is_var_args;
3821	3766	};

src/analyze.cpp+873-238

...	...	@@ -7,6 +7,7 @@
7	7
8	8	#include "analyze.hpp"
9	9	#include "ast_render.hpp"
	10	#include "codegen.hpp"
10	11	#include "config.h"
11	12	#include "error.hpp"
12	13	#include "ir.hpp"
...	...	@@ -31,6 +32,11 @@ static void resolve_llvm_types(CodeGen g, ZigType type, ResolveStatus wanted_r
31	32	static void preview_use_decl(CodeGen g, TldUsingNamespace using_namespace, ScopeDecls *dest_decls_scope);
32	33	static void resolve_use_decl(CodeGen g, TldUsingNamespace tld_using_namespace, ScopeDecls *dest_decls_scope);
33	34
	35	// nullptr means not analyzed yet; this one means currently being analyzed
	36	static const AstNode inferred_async_checking = reinterpret_cast<AstNode >(0x1);
	37	// this one means analyzed and it's not async
	38	static const AstNode inferred_async_none = reinterpret_cast<AstNode >(0x2);
	39
34	40	static bool is_top_level_struct(ZigType *import) {
35	41	return import->id == ZigTypeIdStruct && import->data.structure.root_struct != nullptr;
36	42	}
...	...	@@ -56,14 +62,14 @@ ErrorMsg add_token_error(CodeGen g, ZigType owner, Token token, Buf *msg) {
56	62	return err;
57	63	}
58	64
59		ErrorMsg add_node_error(CodeGen g, AstNode node, Buf msg) {
	65	ErrorMsg add_node_error(CodeGen g, const AstNode node, Buf msg) {
60	66	Token fake_token;
61	67	fake_token.start_line = node->line;
62	68	fake_token.start_column = node->column;
63	69	return add_token_error(g, node->owner, &fake_token, msg);
64	70	}
65	71
66		ErrorMsg add_error_note(CodeGen g, ErrorMsg parent_msg, AstNode node, Buf *msg) {
	72	ErrorMsg add_error_note(CodeGen g, ErrorMsg parent_msg, const AstNode node, Buf *msg) {
67	73	Token fake_token;
68	74	fake_token.start_line = node->line;
69	75	fake_token.start_column = node->column;
...	...	@@ -188,12 +194,6 @@ Scope create_comptime_scope(CodeGen g, AstNode node, Scope parent) {
188	194	return &scope->base;
189	195	}
190	196
191		Scope create_coro_prelude_scope(CodeGen g, AstNode node, Scope parent) {
192		ScopeCoroPrelude *scope = allocate<ScopeCoroPrelude>(1);
193		init_scope(g, &scope->base, ScopeIdCoroPrelude, node, parent);
194		return &scope->base;
195		}
196
197	197	ZigType get_scope_import(Scope scope) {
198	198	while (scope) {
199	199	if (scope->id == ScopeIdDecls) {
...	...	@@ -234,6 +234,8 @@ AstNode type_decl_node(ZigType type_entry) {
234	234	return type_entry->data.enumeration.decl_node;
235	235	case ZigTypeIdUnion:
236	236	return type_entry->data.unionation.decl_node;
	237	case ZigTypeIdFnFrame:
	238	return type_entry->data.frame.fn->proto_node;
237	239	case ZigTypeIdOpaque:
238	240	case ZigTypeIdMetaType:
239	241	case ZigTypeIdVoid:
...	...	@@ -254,8 +256,8 @@ AstNode type_decl_node(ZigType type_entry) {
254	256	case ZigTypeIdFn:
255	257	case ZigTypeIdBoundFn:
256	258	case ZigTypeIdArgTuple:
257		case ZigTypeIdPromise:
258	259	case ZigTypeIdVector:
	260	case ZigTypeIdAnyFrame:
259	261	return nullptr;
260	262	}
261	263	zig_unreachable();
...	...	@@ -269,6 +271,20 @@ bool type_is_resolved(ZigType *type_entry, ResolveStatus status) {
269	271	return type_entry->data.structure.resolve_status >= status;
270	272	case ZigTypeIdUnion:
271	273	return type_entry->data.unionation.resolve_status >= status;
	274	case ZigTypeIdFnFrame:
	275	switch (status) {
	276	case ResolveStatusInvalid:
	277	zig_unreachable();
	278	case ResolveStatusUnstarted:
	279	case ResolveStatusZeroBitsKnown:
	280	return true;
	281	case ResolveStatusAlignmentKnown:
	282	case ResolveStatusSizeKnown:
	283	return type_entry->data.frame.locals_struct != nullptr;
	284	case ResolveStatusLLVMFwdDecl:
	285	case ResolveStatusLLVMFull:
	286	return type_entry->llvm_type != nullptr;
	287	}
272	288	case ZigTypeIdEnum:
273	289	switch (status) {
274	290	case ResolveStatusUnstarted:
...	...	@@ -307,8 +323,8 @@ bool type_is_resolved(ZigType *type_entry, ResolveStatus status) {
307	323	case ZigTypeIdFn:
308	324	case ZigTypeIdBoundFn:
309	325	case ZigTypeIdArgTuple:
310		case ZigTypeIdPromise:
311	326	case ZigTypeIdVector:
	327	case ZigTypeIdAnyFrame:
312	328	return true;
313	329	}
314	330	zig_unreachable();
...	...	@@ -341,27 +357,27 @@ ZigType get_smallest_unsigned_int_type(CodeGen g, uint64_t x) {
341	357	return get_int_type(g, false, bits_needed_for_unsigned(x));
342	358	}
343	359
344		ZigType get_promise_type(CodeGen g, ZigType *result_type) {
345		if (result_type != nullptr && result_type->promise_parent != nullptr) {
346		return result_type->promise_parent;
347		} else if (result_type == nullptr && g->builtin_types.entry_promise != nullptr) {
348		return g->builtin_types.entry_promise;
	360	ZigType get_any_frame_type(CodeGen g, ZigType *result_type) {
	361	if (result_type != nullptr && result_type->any_frame_parent != nullptr) {
	362	return result_type->any_frame_parent;
	363	} else if (result_type == nullptr && g->builtin_types.entry_any_frame != nullptr) {
	364	return g->builtin_types.entry_any_frame;
349	365	}
350	366
351		ZigType *entry = new_type_table_entry(ZigTypeIdPromise);
	367	ZigType *entry = new_type_table_entry(ZigTypeIdAnyFrame);
352	368	entry->abi_size = g->builtin_types.entry_usize->abi_size;
353	369	entry->size_in_bits = g->builtin_types.entry_usize->size_in_bits;
354	370	entry->abi_align = g->builtin_types.entry_usize->abi_align;
355		entry->data.promise.result_type = result_type;
356		buf_init_from_str(&entry->name, "promise");
	371	entry->data.any_frame.result_type = result_type;
	372	buf_init_from_str(&entry->name, "anyframe");
357	373	if (result_type != nullptr) {
358	374	buf_appendf(&entry->name, "->%s", buf_ptr(&result_type->name));
359	375	}
360	376
361	377	if (result_type != nullptr) {
362		result_type->promise_parent = entry;
	378	result_type->any_frame_parent = entry;
363	379	} else if (result_type == nullptr) {
364		g->builtin_types.entry_promise = entry;
	380	g->builtin_types.entry_any_frame = entry;
365	381	}
366	382	return entry;
367	383	}
...	...	@@ -378,6 +394,25 @@ static const char *ptr_len_to_star_str(PtrLen ptr_len) {
378	394	zig_unreachable();
379	395	}
380	396
	397	ZigType get_fn_frame_type(CodeGen g, ZigFn *fn) {
	398	if (fn->frame_type != nullptr) {
	399	return fn->frame_type;
	400	}
	401
	402	ZigType *entry = new_type_table_entry(ZigTypeIdFnFrame);
	403	buf_resize(&entry->name, 0);
	404	buf_appendf(&entry->name, "@Frame(%s)", buf_ptr(&fn->symbol_name));
	405
	406	entry->data.frame.fn = fn;
	407
	408	// Async function frames are always non-zero bits because they always have a resume index.
	409	entry->abi_size = SIZE_MAX;
	410	entry->size_in_bits = SIZE_MAX;
	411
	412	fn->frame_type = entry;
	413	return entry;
	414	}
	415
381	416	ZigType get_pointer_to_type_extra(CodeGen g, ZigType *child_type, bool is_const,
382	417	bool is_volatile, PtrLen ptr_len, uint32_t byte_alignment,
383	418	uint32_t bit_offset_in_host, uint32_t host_int_bytes, bool allow_zero)
...	...	@@ -490,42 +525,6 @@ ZigType get_pointer_to_type(CodeGen g, ZigType *child_type, bool is_const) {
490	525	return get_pointer_to_type_extra(g, child_type, is_const, false, PtrLenSingle, 0, 0, 0, false);
491	526	}
492	527
493		ZigType get_promise_frame_type(CodeGen g, ZigType *return_type) {
494		if (return_type->promise_frame_parent != nullptr) {
495		return return_type->promise_frame_parent;
496		}
497
498		ZigType *atomic_state_type = g->builtin_types.entry_usize;
499		ZigType *result_ptr_type = get_pointer_to_type(g, return_type, false);
500
501		ZigList<const char *> field_names = {};
502		field_names.append(ATOMIC_STATE_FIELD_NAME);
503		field_names.append(RESULT_FIELD_NAME);
504		field_names.append(RESULT_PTR_FIELD_NAME);
505		if (g->have_err_ret_tracing) {
506		field_names.append(ERR_RET_TRACE_PTR_FIELD_NAME);
507		field_names.append(ERR_RET_TRACE_FIELD_NAME);
508		field_names.append(RETURN_ADDRESSES_FIELD_NAME);
509		}
510
511		ZigList<ZigType *> field_types = {};
512		field_types.append(atomic_state_type);
513		field_types.append(return_type);
514		field_types.append(result_ptr_type);
515		if (g->have_err_ret_tracing) {
516		field_types.append(get_ptr_to_stack_trace_type(g));
517		field_types.append(g->stack_trace_type);
518		field_types.append(get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count));
519		}
520
521		assert(field_names.length == field_types.length);
522		Buf *name = buf_sprintf("AsyncFramePromise(%s)", buf_ptr(&return_type->name));
523		ZigType *entry = get_struct_type(g, buf_ptr(name), field_names.items, field_types.items, field_names.length);
524
525		return_type->promise_frame_parent = entry;
526		return entry;
527		}
528
529	528	ZigType get_optional_type(CodeGen g, ZigType *child_type) {
530	529	if (child_type->optional_parent != nullptr) {
531	530	return child_type->optional_parent;
...	...	@@ -631,6 +630,7 @@ ZigType get_error_union_type(CodeGen g, ZigType err_set_type, ZigType payloa
631	630	size_t field2_offset = next_field_offset(0, entry->abi_align, field_sizes[0], field_aligns[1]);
632	631	entry->abi_size = next_field_offset(field2_offset, entry->abi_align, field_sizes[1], entry->abi_align);
633	632	entry->size_in_bits = entry->abi_size * 8;
	633	entry->data.error_union.pad_bytes = entry->abi_size - (field2_offset + field_sizes[1]);
634	634	}
635	635
636	636	g->type_table.put(type_id, entry);
...	...	@@ -828,17 +828,15 @@ bool calling_convention_allows_zig_types(CallingConvention cc) {
828	828	zig_unreachable();
829	829	}
830	830
831		ZigType get_ptr_to_stack_trace_type(CodeGen g) {
	831	ZigType get_stack_trace_type(CodeGen g) {
832	832	if (g->stack_trace_type == nullptr) {
833	833	ConstExprValue *stack_trace_type_val = get_builtin_value(g, "StackTrace");
834	834	assert(stack_trace_type_val->type->id == ZigTypeIdMetaType);
835	835
836	836	g->stack_trace_type = stack_trace_type_val->data.x_type;
837	837	assertNoError(type_resolve(g, g->stack_trace_type, ResolveStatusZeroBitsKnown));
838
839		g->ptr_to_stack_trace_type = get_pointer_to_type(g, g->stack_trace_type, false);
840	838	}
841		return g->ptr_to_stack_trace_type;
	839	return g->stack_trace_type;
842	840	}
843	841
844	842	bool want_first_arg_sret(CodeGen g, FnTypeId fn_type_id) {
...	...	@@ -879,13 +877,8 @@ ZigType get_fn_type(CodeGen g, FnTypeId *fn_type_id) {
879	877
880	878	// populate the name of the type
881	879	buf_resize(&fn_type->name, 0);
882		if (fn_type->data.fn.fn_type_id.cc == CallingConventionAsync) {
883		assert(fn_type_id->async_allocator_type != nullptr);
884		buf_appendf(&fn_type->name, "async<%s> ", buf_ptr(&fn_type_id->async_allocator_type->name));
885		} else {
886		const char *cc_str = calling_convention_fn_type_str(fn_type->data.fn.fn_type_id.cc);
887		buf_appendf(&fn_type->name, "%s", cc_str);
888		}
	880	const char *cc_str = calling_convention_fn_type_str(fn_type->data.fn.fn_type_id.cc);
	881	buf_appendf(&fn_type->name, "%s", cc_str);
889	882	buf_appendf(&fn_type->name, "fn(");
890	883	for (size_t i = 0; i < fn_type_id->param_count; i += 1) {
891	884	FnTypeParamInfo *param_info = &fn_type_id->param_info[i];
...	...	@@ -998,14 +991,8 @@ ZigType analyze_type_expr(CodeGen g, Scope scope, AstNode node) {
998	991	ZigType get_generic_fn_type(CodeGen g, FnTypeId *fn_type_id) {
999	992	ZigType *fn_type = new_type_table_entry(ZigTypeIdFn);
1000	993	buf_resize(&fn_type->name, 0);
1001		if (fn_type->data.fn.fn_type_id.cc == CallingConventionAsync) {
1002		const char *async_allocator_type_str = (fn_type->data.fn.fn_type_id.async_allocator_type == nullptr) ?
1003		"var" : buf_ptr(&fn_type_id->async_allocator_type->name);
1004		buf_appendf(&fn_type->name, "async(%s) ", async_allocator_type_str);
1005		} else {
1006		const char *cc_str = calling_convention_fn_type_str(fn_type->data.fn.fn_type_id.cc);
1007		buf_appendf(&fn_type->name, "%s", cc_str);
1008		}
	994	const char *cc_str = calling_convention_fn_type_str(fn_type->data.fn.fn_type_id.cc);
	995	buf_appendf(&fn_type->name, "%s", cc_str);
1009	996	buf_appendf(&fn_type->name, "fn(");
1010	997	size_t i = 0;
1011	998	for (; i < fn_type_id->next_param_index; i += 1) {
...	...	@@ -1119,7 +1106,8 @@ static Error emit_error_unless_type_allowed_in_packed_struct(CodeGen *g, ZigType
1119	1106	case ZigTypeIdBoundFn:
1120	1107	case ZigTypeIdArgTuple:
1121	1108	case ZigTypeIdOpaque:
1122		case ZigTypeIdPromise:
	1109	case ZigTypeIdFnFrame:
	1110	case ZigTypeIdAnyFrame:
1123	1111	add_node_error(g, source_node,
1124	1112	buf_sprintf("type '%s' not allowed in packed struct; no guaranteed in-memory representation",
1125	1113	buf_ptr(&type_entry->name)));
...	...	@@ -1207,8 +1195,9 @@ bool type_allowed_in_extern(CodeGen g, ZigType type_entry) {
1207	1195	case ZigTypeIdErrorSet:
1208	1196	case ZigTypeIdBoundFn:
1209	1197	case ZigTypeIdArgTuple:
1210		case ZigTypeIdPromise:
1211	1198	case ZigTypeIdVoid:
	1199	case ZigTypeIdFnFrame:
	1200	case ZigTypeIdAnyFrame:
1212	1201	return false;
1213	1202	case ZigTypeIdOpaque:
1214	1203	case ZigTypeIdUnreachable:
...	...	@@ -1378,8 +1367,9 @@ static ZigType analyze_fn_type(CodeGen g, AstNode proto_node, Scope child_sc
1378	1367	case ZigTypeIdEnum:
1379	1368	case ZigTypeIdUnion:
1380	1369	case ZigTypeIdFn:
1381		case ZigTypeIdPromise:
1382	1370	case ZigTypeIdVector:
	1371	case ZigTypeIdFnFrame:
	1372	case ZigTypeIdAnyFrame:
1383	1373	switch (type_requires_comptime(g, type_entry)) {
1384	1374	case ReqCompTimeNo:
1385	1375	break;
...	...	@@ -1474,8 +1464,9 @@ static ZigType analyze_fn_type(CodeGen g, AstNode proto_node, Scope child_sc
1474	1464	case ZigTypeIdEnum:
1475	1465	case ZigTypeIdUnion:
1476	1466	case ZigTypeIdFn:
1477		case ZigTypeIdPromise:
1478	1467	case ZigTypeIdVector:
	1468	case ZigTypeIdFnFrame:
	1469	case ZigTypeIdAnyFrame:
1479	1470	switch (type_requires_comptime(g, fn_type_id.return_type)) {
1480	1471	case ReqCompTimeInvalid:
1481	1472	return g->builtin_types.entry_invalid;
...	...	@@ -1487,16 +1478,6 @@ static ZigType analyze_fn_type(CodeGen g, AstNode proto_node, Scope child_sc
1487	1478	break;
1488	1479	}
1489	1480
1490		if (fn_type_id.cc == CallingConventionAsync) {
1491		if (fn_proto->async_allocator_type == nullptr) {
1492		return get_generic_fn_type(g, &fn_type_id);
1493		}
1494		fn_type_id.async_allocator_type = analyze_type_expr(g, child_scope, fn_proto->async_allocator_type);
1495		if (type_is_invalid(fn_type_id.async_allocator_type)) {
1496		return g->builtin_types.entry_invalid;
1497		}
1498		}
1499
1500	1481	return get_fn_type(g, &fn_type_id);
1501	1482	}
1502	1483
...	...	@@ -1516,9 +1497,14 @@ bool type_is_invalid(ZigType *type_entry) {
1516	1497	zig_unreachable();
1517	1498	}
1518	1499
	1500	struct SrcField {
	1501	const char *name;
	1502	ZigType *ty;
	1503	unsigned align;
	1504	};
1519	1505
1520		ZigType get_struct_type(CodeGen g, const char type_name, const char field_names[],
1521		ZigType *field_types[], size_t field_count)
	1506	static ZigType get_struct_type(CodeGen g, const char *type_name, SrcField fields[], size_t field_count,
	1507	unsigned min_abi_align)
1522	1508	{
1523	1509	ZigType *struct_type = new_type_table_entry(ZigTypeIdStruct);
1524	1510
...	...	@@ -1530,22 +1516,20 @@ ZigType get_struct_type(CodeGen g, const char type_name, const char field_na
1530	1516	struct_type->data.structure.fields = allocate<TypeStructField>(field_count);
1531	1517	struct_type->data.structure.fields_by_name.init(field_count);
1532	1518
1533		size_t abi_align = 0;
	1519	size_t abi_align = min_abi_align;
1534	1520	for (size_t i = 0; i < field_count; i += 1) {
1535	1521	TypeStructField *field = &struct_type->data.structure.fields[i];
1536		field->name = buf_create_from_str(field_names[i]);
1537		field->type_entry = field_types[i];
	1522	field->name = buf_create_from_str(fields[i].name);
	1523	field->type_entry = fields[i].ty;
1538	1524	field->src_index = i;
	1525	field->align = fields[i].align;
1539	1526
1540	1527	if (type_has_bits(field->type_entry)) {
1541	1528	assert(type_is_resolved(field->type_entry, ResolveStatusSizeKnown));
1542		if (field->type_entry->abi_align > abi_align) {
1543		abi_align = field->type_entry->abi_align;
	1529	unsigned field_abi_align = max(field->align, field->type_entry->abi_align);
	1530	if (field_abi_align > abi_align) {
	1531	abi_align = field_abi_align;
1544	1532	}
1545		field->gen_index = struct_type->data.structure.gen_field_count;
1546		struct_type->data.structure.gen_field_count += 1;
1547		} else {
1548		field->gen_index = SIZE_MAX;
1549	1533	}
1550	1534
1551	1535	auto prev_entry = struct_type->data.structure.fields_by_name.put_unique(field->name, field);
...	...	@@ -1555,17 +1539,24 @@ ZigType get_struct_type(CodeGen g, const char type_name, const char field_na
1555	1539	size_t next_offset = 0;
1556	1540	for (size_t i = 0; i < field_count; i += 1) {
1557	1541	TypeStructField *field = &struct_type->data.structure.fields[i];
1558		if (field->gen_index == SIZE_MAX)
	1542	if (!type_has_bits(field->type_entry))
1559	1543	continue;
	1544
1560	1545	field->offset = next_offset;
	1546
	1547	// find the next non-zero-byte field for offset calculations
1561	1548	size_t next_src_field_index = i + 1;
1562	1549	for (; next_src_field_index < field_count; next_src_field_index += 1) {
1563		if (struct_type->data.structure.fields[next_src_field_index].gen_index != SIZE_MAX) {
	1550	if (type_has_bits(struct_type->data.structure.fields[next_src_field_index].type_entry))
1564	1551	break;
1565		}
1566	1552	}
1567		size_t next_abi_align = (next_src_field_index == field_count) ?
1568		abi_align : struct_type->data.structure.fields[next_src_field_index].type_entry->abi_align;
	1553	size_t next_abi_align;
	1554	if (next_src_field_index == field_count) {
	1555	next_abi_align = abi_align;
	1556	} else {
	1557	next_abi_align = max(fields[next_src_field_index].align,
	1558	struct_type->data.structure.fields[next_src_field_index].type_entry->abi_align);
	1559	}
1569	1560	next_offset = next_field_offset(next_offset, abi_align, field->type_entry->abi_size, next_abi_align);
1570	1561	}
1571	1562
...	...	@@ -2653,7 +2644,6 @@ ZigFn create_fn_raw(CodeGen g, FnInline inline_value) {
2653	2644
2654	2645	fn_entry->prealloc_backward_branch_quota = default_backward_branch_quota;
2655	2646
2656		fn_entry->codegen = g;
2657	2647	fn_entry->analyzed_executable.backward_branch_count = &fn_entry->prealloc_bbc;
2658	2648	fn_entry->analyzed_executable.backward_branch_quota = &fn_entry->prealloc_backward_branch_quota;
2659	2649	fn_entry->analyzed_executable.fn_entry = fn_entry;
...	...	@@ -2781,6 +2771,7 @@ static void resolve_decl_fn(CodeGen g, TldFn tld_fn) {
2781	2771	}
2782	2772	}
2783	2773	} else {
	2774	fn_table_entry->inferred_async_node = inferred_async_none;
2784	2775	g->external_prototypes.put_unique(tld_fn->base.name, &tld_fn->base);
2785	2776	}
2786	2777
...	...	@@ -2802,6 +2793,13 @@ static void resolve_decl_fn(CodeGen g, TldFn tld_fn) {
2802	2793	g->fn_defs.append(fn_table_entry);
2803	2794	}
2804	2795
	2796	// if the calling convention implies that it cannot be async, we save that for later
	2797	// and leave the value to be nullptr to indicate that we have not emitted possible
	2798	// compile errors for improperly calling async functions.
	2799	if (fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync) {
	2800	fn_table_entry->inferred_async_node = fn_table_entry->proto_node;
	2801	}
	2802
2805	2803	if (scope_is_root_decls(tld_fn->base.parent_scope) &&
2806	2804	(import == g->root_import \|\| import->data.structure.root_struct->package == g->panic_package))
2807	2805	{
...	...	@@ -3035,12 +3033,11 @@ void scan_decls(CodeGen g, ScopeDecls decls_scope, AstNode *node) {
3035	3033	case NodeTypeIfErrorExpr:
3036	3034	case NodeTypeIfOptional:
3037	3035	case NodeTypeErrorSetDecl:
3038		case NodeTypeCancel:
3039	3036	case NodeTypeResume:
3040	3037	case NodeTypeAwaitExpr:
3041	3038	case NodeTypeSuspend:
3042		case NodeTypePromiseType:
3043	3039	case NodeTypeEnumLiteral:
	3040	case NodeTypeAnyFrameType:
3044	3041	zig_unreachable();
3045	3042	}
3046	3043	}
...	...	@@ -3091,8 +3088,9 @@ ZigType validate_var_type(CodeGen g, AstNode source_node, ZigType type_entry
3091	3088	case ZigTypeIdUnion:
3092	3089	case ZigTypeIdFn:
3093	3090	case ZigTypeIdBoundFn:
3094		case ZigTypeIdPromise:
3095	3091	case ZigTypeIdVector:
	3092	case ZigTypeIdFnFrame:
	3093	case ZigTypeIdAnyFrame:
3096	3094	return type_entry;
3097	3095	}
3098	3096	zig_unreachable();
...	...	@@ -3592,8 +3590,9 @@ bool is_container(ZigType *type_entry) {
3592	3590	case ZigTypeIdBoundFn:
3593	3591	case ZigTypeIdArgTuple:
3594	3592	case ZigTypeIdOpaque:
3595		case ZigTypeIdPromise:
3596	3593	case ZigTypeIdVector:
	3594	case ZigTypeIdFnFrame:
	3595	case ZigTypeIdAnyFrame:
3597	3596	return false;
3598	3597	}
3599	3598	zig_unreachable();
...	...	@@ -3649,8 +3648,9 @@ Error resolve_container_type(CodeGen g, ZigType type_entry) {
3649	3648	case ZigTypeIdInvalid:
3650	3649	case ZigTypeIdArgTuple:
3651	3650	case ZigTypeIdOpaque:
3652		case ZigTypeIdPromise:
3653	3651	case ZigTypeIdVector:
	3652	case ZigTypeIdFnFrame:
	3653	case ZigTypeIdAnyFrame:
3654	3654	zig_unreachable();
3655	3655	}
3656	3656	zig_unreachable();
...	...	@@ -3659,13 +3659,13 @@ Error resolve_container_type(CodeGen g, ZigType type_entry) {
3659	3659	ZigType get_src_ptr_type(ZigType type) {
3660	3660	if (type->id == ZigTypeIdPointer) return type;
3661	3661	if (type->id == ZigTypeIdFn) return type;
3662		if (type->id == ZigTypeIdPromise) return type;
	3662	if (type->id == ZigTypeIdAnyFrame) return type;
3663	3663	if (type->id == ZigTypeIdOptional) {
3664	3664	if (type->data.maybe.child_type->id == ZigTypeIdPointer) {
3665	3665	return type->data.maybe.child_type->data.pointer.allow_zero ? nullptr : type->data.maybe.child_type;
3666	3666	}
3667	3667	if (type->data.maybe.child_type->id == ZigTypeIdFn) return type->data.maybe.child_type;
3668		if (type->data.maybe.child_type->id == ZigTypeIdPromise) return type->data.maybe.child_type;
	3668	if (type->data.maybe.child_type->id == ZigTypeIdAnyFrame) return type->data.maybe.child_type;
3669	3669	}
3670	3670	return nullptr;
3671	3671	}
...	...	@@ -3681,6 +3681,13 @@ bool type_is_nonnull_ptr(ZigType *type) {
3681	3681	return get_codegen_ptr_type(type) == type && !ptr_allows_addr_zero(type);
3682	3682	}
3683	3683
	3684	static uint32_t get_async_frame_align_bytes(CodeGen *g) {
	3685	uint32_t a = g->pointer_size_bytes * 2;
	3686	// promises have at least alignment 8 so that we can have 3 extra bits when doing atomicrmw
	3687	if (a < 8) a = 8;
	3688	return a;
	3689	}
	3690
3684	3691	uint32_t get_ptr_align(CodeGen g, ZigType type) {
3685	3692	ZigType *ptr_type = get_src_ptr_type(type);
3686	3693	if (ptr_type->id == ZigTypeIdPointer) {
...	...	@@ -3692,8 +3699,8 @@ uint32_t get_ptr_align(CodeGen g, ZigType type) {
3692	3699	// when getting the alignment of `?extern fn() void`.
3693	3700	// See http://lists.llvm.org/pipermail/llvm-dev/2018-September/126142.html
3694	3701	return (ptr_type->data.fn.fn_type_id.alignment == 0) ? 1 : ptr_type->data.fn.fn_type_id.alignment;
3695		} else if (ptr_type->id == ZigTypeIdPromise) {
3696		return get_coro_frame_align_bytes(g);
	3702	} else if (ptr_type->id == ZigTypeIdAnyFrame) {
	3703	return get_async_frame_align_bytes(g);
3697	3704	} else {
3698	3705	zig_unreachable();
3699	3706	}
...	...	@@ -3705,7 +3712,7 @@ bool get_ptr_const(ZigType *type) {
3705	3712	return ptr_type->data.pointer.is_const;
3706	3713	} else if (ptr_type->id == ZigTypeIdFn) {
3707	3714	return true;
3708		} else if (ptr_type->id == ZigTypeIdPromise) {
	3715	} else if (ptr_type->id == ZigTypeIdAnyFrame) {
3709	3716	return true;
3710	3717	} else {
3711	3718	zig_unreachable();
...	...	@@ -3780,18 +3787,128 @@ bool resolve_inferred_error_set(CodeGen g, ZigType err_set_type, AstNode *sour
3780	3787	return true;
3781	3788	}
3782	3789
3783		void analyze_fn_ir(CodeGen g, ZigFn fn_table_entry, AstNode *return_type_node) {
3784		ZigType *fn_type = fn_table_entry->type_entry;
	3790	static void resolve_async_fn_frame(CodeGen g, ZigFn fn) {
	3791	ZigType *frame_type = get_fn_frame_type(g, fn);
	3792	Error err;
	3793	if ((err = type_resolve(g, frame_type, ResolveStatusSizeKnown))) {
	3794	fn->anal_state = FnAnalStateInvalid;
	3795	return;
	3796	}
	3797	}
	3798
	3799	bool fn_is_async(ZigFn *fn) {
	3800	assert(fn->inferred_async_node != nullptr);
	3801	assert(fn->inferred_async_node != inferred_async_checking);
	3802	return fn->inferred_async_node != inferred_async_none;
	3803	}
	3804
	3805	static void add_async_error_notes(CodeGen g, ErrorMsg msg, ZigFn *fn) {
	3806	assert(fn->inferred_async_node != nullptr);
	3807	assert(fn->inferred_async_node != inferred_async_checking);
	3808	assert(fn->inferred_async_node != inferred_async_none);
	3809	if (fn->inferred_async_fn != nullptr) {
	3810	ErrorMsg *new_msg = add_error_note(g, msg, fn->inferred_async_node,
	3811	buf_sprintf("async function call here"));
	3812	return add_async_error_notes(g, new_msg, fn->inferred_async_fn);
	3813	} else if (fn->inferred_async_node->type == NodeTypeFnProto) {
	3814	add_error_note(g, msg, fn->inferred_async_node,
	3815	buf_sprintf("async calling convention here"));
	3816	} else if (fn->inferred_async_node->type == NodeTypeSuspend) {
	3817	add_error_note(g, msg, fn->inferred_async_node,
	3818	buf_sprintf("suspends here"));
	3819	} else if (fn->inferred_async_node->type == NodeTypeAwaitExpr) {
	3820	add_error_note(g, msg, fn->inferred_async_node,
	3821	buf_sprintf("await is a suspend point"));
	3822	} else if (fn->inferred_async_node->type == NodeTypeFnCallExpr &&
	3823	fn->inferred_async_node->data.fn_call_expr.is_builtin)
	3824	{
	3825	add_error_note(g, msg, fn->inferred_async_node,
	3826	buf_sprintf("@frame() causes function to be async"));
	3827	} else {
	3828	add_error_note(g, msg, fn->inferred_async_node,
	3829	buf_sprintf("suspends here"));
	3830	}
	3831	}
	3832
	3833	// This function resolves functions being inferred async.
	3834	static void analyze_fn_async(CodeGen g, ZigFn fn, bool resolve_frame) {
	3835	if (fn->inferred_async_node == inferred_async_checking) {
	3836	// TODO call graph cycle detected, disallow the recursion
	3837	fn->inferred_async_node = inferred_async_none;
	3838	return;
	3839	}
	3840	if (fn->inferred_async_node == inferred_async_none) {
	3841	return;
	3842	}
	3843	if (fn->inferred_async_node != nullptr) {
	3844	if (resolve_frame) {
	3845	resolve_async_fn_frame(g, fn);
	3846	}
	3847	return;
	3848	}
	3849	fn->inferred_async_node = inferred_async_checking;
	3850
	3851	bool must_not_be_async = false;
	3852	if (fn->type_entry->data.fn.fn_type_id.cc != CallingConventionUnspecified) {
	3853	must_not_be_async = true;
	3854	fn->inferred_async_node = inferred_async_none;
	3855	}
	3856
	3857	for (size_t i = 0; i < fn->call_list.length; i += 1) {
	3858	IrInstructionCallGen *call = fn->call_list.at(i);
	3859	ZigFn *callee = call->fn_entry;
	3860	if (callee == nullptr) {
	3861	// TODO function pointer call here, could be anything
	3862	continue;
	3863	}
	3864
	3865	if (callee->type_entry->data.fn.fn_type_id.cc != CallingConventionUnspecified)
	3866	continue;
	3867	if (callee->anal_state == FnAnalStateReady) {
	3868	analyze_fn_body(g, callee);
	3869	if (callee->anal_state == FnAnalStateInvalid) {
	3870	fn->anal_state = FnAnalStateInvalid;
	3871	return;
	3872	}
	3873	}
	3874	assert(callee->anal_state == FnAnalStateComplete);
	3875	analyze_fn_async(g, callee, true);
	3876	if (callee->anal_state == FnAnalStateInvalid) {
	3877	fn->anal_state = FnAnalStateInvalid;
	3878	return;
	3879	}
	3880	if (fn_is_async(callee)) {
	3881	fn->inferred_async_node = call->base.source_node;
	3882	fn->inferred_async_fn = callee;
	3883	if (must_not_be_async) {
	3884	ErrorMsg *msg = add_node_error(g, fn->proto_node,
	3885	buf_sprintf("function with calling convention '%s' cannot be async",
	3886	calling_convention_name(fn->type_entry->data.fn.fn_type_id.cc)));
	3887	add_async_error_notes(g, msg, fn);
	3888	fn->anal_state = FnAnalStateInvalid;
	3889	return;
	3890	}
	3891	if (resolve_frame) {
	3892	resolve_async_fn_frame(g, fn);
	3893	}
	3894	return;
	3895	}
	3896	}
	3897	fn->inferred_async_node = inferred_async_none;
	3898	}
	3899
	3900	static void analyze_fn_ir(CodeGen g, ZigFn fn, AstNode *return_type_node) {
	3901	ZigType *fn_type = fn->type_entry;
3785	3902	assert(!fn_type->data.fn.is_generic);
3786	3903	FnTypeId *fn_type_id = &fn_type->data.fn.fn_type_id;
3787	3904
3788		ZigType *block_return_type = ir_analyze(g, &fn_table_entry->ir_executable,
3789		&fn_table_entry->analyzed_executable, fn_type_id->return_type, return_type_node);
3790		fn_table_entry->src_implicit_return_type = block_return_type;
	3905	ZigType *block_return_type = ir_analyze(g, &fn->ir_executable,
	3906	&fn->analyzed_executable, fn_type_id->return_type, return_type_node);
	3907	fn->src_implicit_return_type = block_return_type;
3791	3908
3792		if (type_is_invalid(block_return_type) \|\| fn_table_entry->analyzed_executable.invalid) {
	3909	if (type_is_invalid(block_return_type) \|\| fn->analyzed_executable.invalid) {
3793	3910	assert(g->errors.length > 0);
3794		fn_table_entry->anal_state = FnAnalStateInvalid;
	3911	fn->anal_state = FnAnalStateInvalid;
3795	3912	return;
3796	3913	}
3797	3914
...	...	@@ -3799,20 +3916,20 @@ void analyze_fn_ir(CodeGen g, ZigFn fn_table_entry, AstNode *return_type_node)
3799	3916	ZigType *return_err_set_type = fn_type_id->return_type->data.error_union.err_set_type;
3800	3917	if (return_err_set_type->data.error_set.infer_fn != nullptr) {
3801	3918	ZigType *inferred_err_set_type;
3802		if (fn_table_entry->src_implicit_return_type->id == ZigTypeIdErrorSet) {
3803		inferred_err_set_type = fn_table_entry->src_implicit_return_type;
3804		} else if (fn_table_entry->src_implicit_return_type->id == ZigTypeIdErrorUnion) {
3805		inferred_err_set_type = fn_table_entry->src_implicit_return_type->data.error_union.err_set_type;
	3919	if (fn->src_implicit_return_type->id == ZigTypeIdErrorSet) {
	3920	inferred_err_set_type = fn->src_implicit_return_type;
	3921	} else if (fn->src_implicit_return_type->id == ZigTypeIdErrorUnion) {
	3922	inferred_err_set_type = fn->src_implicit_return_type->data.error_union.err_set_type;
3806	3923	} else {
3807	3924	add_node_error(g, return_type_node,
3808	3925	buf_sprintf("function with inferred error set must return at least one possible error"));
3809		fn_table_entry->anal_state = FnAnalStateInvalid;
	3926	fn->anal_state = FnAnalStateInvalid;
3810	3927	return;
3811	3928	}
3812	3929
3813	3930	if (inferred_err_set_type->data.error_set.infer_fn != nullptr) {
3814	3931	if (!resolve_inferred_error_set(g, inferred_err_set_type, return_type_node)) {
3815		fn_table_entry->anal_state = FnAnalStateInvalid;
	3932	fn->anal_state = FnAnalStateInvalid;
3816	3933	return;
3817	3934	}
3818	3935	}
...	...	@@ -3832,13 +3949,25 @@ void analyze_fn_ir(CodeGen g, ZigFn fn_table_entry, AstNode *return_type_node)
3832	3949	}
3833	3950	}
3834	3951
	3952	CallingConvention cc = fn->type_entry->data.fn.fn_type_id.cc;
	3953	if (cc != CallingConventionUnspecified && cc != CallingConventionAsync &&
	3954	fn->inferred_async_node != nullptr &&
	3955	fn->inferred_async_node != inferred_async_checking &&
	3956	fn->inferred_async_node != inferred_async_none)
	3957	{
	3958	ErrorMsg *msg = add_node_error(g, fn->proto_node,
	3959	buf_sprintf("function with calling convention '%s' cannot be async",
	3960	calling_convention_name(cc)));
	3961	add_async_error_notes(g, msg, fn);
	3962	fn->anal_state = FnAnalStateInvalid;
	3963	}
	3964
3835	3965	if (g->verbose_ir) {
3836		fprintf(stderr, "fn %s() { // (analyzed)\n", buf_ptr(&fn_table_entry->symbol_name));
3837		ir_print(g, stderr, &fn_table_entry->analyzed_executable, 4);
	3966	fprintf(stderr, "fn %s() { // (analyzed)\n", buf_ptr(&fn->symbol_name));
	3967	ir_print(g, stderr, &fn->analyzed_executable, 4);
3838	3968	fprintf(stderr, "}\n");
3839	3969	}
3840
3841		fn_table_entry->anal_state = FnAnalStateComplete;
	3970	fn->anal_state = FnAnalStateComplete;
3842	3971	}
3843	3972
3844	3973	static void analyze_fn_body(CodeGen g, ZigFn fn_table_entry) {
...	...	@@ -4008,6 +4137,16 @@ void semantic_analyze(CodeGen *g) {
4008	4137	analyze_fn_body(g, fn_entry);
4009	4138	}
4010	4139	}
	4140
	4141	if (g->errors.length != 0) {
	4142	return;
	4143	}
	4144
	4145	// second pass over functions for detecting async
	4146	for (g->fn_defs_index = 0; g->fn_defs_index < g->fn_defs.length; g->fn_defs_index += 1) {
	4147	ZigFn *fn_entry = g->fn_defs.at(g->fn_defs_index);
	4148	analyze_fn_async(g, fn_entry, true);
	4149	}
4011	4150	}
4012	4151
4013	4152	ZigType get_int_type(CodeGen g, bool is_signed, uint32_t size_in_bits) {
...	...	@@ -4103,11 +4242,12 @@ bool handle_is_ptr(ZigType *type_entry) {
4103	4242	case ZigTypeIdErrorSet:
4104	4243	case ZigTypeIdFn:
4105	4244	case ZigTypeIdEnum:
4106		case ZigTypeIdPromise:
4107	4245	case ZigTypeIdVector:
	4246	case ZigTypeIdAnyFrame:
4108	4247	return false;
4109	4248	case ZigTypeIdArray:
4110	4249	case ZigTypeIdStruct:
	4250	case ZigTypeIdFnFrame:
4111	4251	return type_has_bits(type_entry);
4112	4252	case ZigTypeIdErrorUnion:
4113	4253	return type_has_bits(type_entry->data.error_union.payload_type);
...	...	@@ -4143,7 +4283,6 @@ uint32_t fn_type_id_hash(FnTypeId *id) {
4143	4283	result += ((uint32_t)(id->cc)) * (uint32_t)3349388391;
4144	4284	result += id->is_var_args ? (uint32_t)1931444534 : 0;
4145	4285	result += hash_ptr(id->return_type);
4146		result += hash_ptr(id->async_allocator_type);
4147	4286	result += id->alignment * 0xd3b3f3e2;
4148	4287	for (size_t i = 0; i < id->param_count; i += 1) {
4149	4288	FnTypeParamInfo *info = &id->param_info[i];
...	...	@@ -4158,8 +4297,7 @@ bool fn_type_id_eql(FnTypeId a, FnTypeId b) {
4158	4297	a->return_type != b->return_type \|\|
4159	4298	a->is_var_args != b->is_var_args \|\|
4160	4299	a->param_count != b->param_count \|\|
4161		a->alignment != b->alignment \|\|
4162		a->async_allocator_type != b->async_allocator_type)
	4300	a->alignment != b->alignment)
4163	4301	{
4164	4302	return false;
4165	4303	}
...	...	@@ -4321,9 +4459,6 @@ static uint32_t hash_const_val(ConstExprValue *const_val) {
4321	4459	return 3677364617 ^ hash_ptr(const_val->data.x_ptr.data.fn.fn_entry);
4322	4460	case ZigTypeIdPointer:
4323	4461	return hash_const_val_ptr(const_val);
4324		case ZigTypeIdPromise:
4325		// TODO better hashing algorithm
4326		return 223048345;
4327	4462	case ZigTypeIdUndefined:
4328	4463	return 162837799;
4329	4464	case ZigTypeIdNull:
...	...	@@ -4357,6 +4492,12 @@ static uint32_t hash_const_val(ConstExprValue *const_val) {
4357	4492	case ZigTypeIdVector:
4358	4493	// TODO better hashing algorithm
4359	4494	return 3647867726;
	4495	case ZigTypeIdFnFrame:
	4496	// TODO better hashing algorithm
	4497	return 675741936;
	4498	case ZigTypeIdAnyFrame:
	4499	// TODO better hashing algorithm
	4500	return 3747294894;
4360	4501	case ZigTypeIdBoundFn:
4361	4502	case ZigTypeIdInvalid:
4362	4503	case ZigTypeIdUnreachable:
...	...	@@ -4389,7 +4530,7 @@ bool generic_fn_type_id_eql(GenericFnTypeId a, GenericFnTypeId b) {
4389	4530	if (a_val->special != ConstValSpecialRuntime && b_val->special != ConstValSpecialRuntime) {
4390	4531	assert(a_val->special == ConstValSpecialStatic);
4391	4532	assert(b_val->special == ConstValSpecialStatic);
4392		if (!const_values_equal(a->fn_entry->codegen, a_val, b_val)) {
	4533	if (!const_values_equal(a->codegen, a_val, b_val)) {
4393	4534	return false;
4394	4535	}
4395	4536	} else {
...	...	@@ -4419,9 +4560,10 @@ static bool can_mutate_comptime_var_state(ConstExprValue *value) {
4419	4560	case ZigTypeIdBoundFn:
4420	4561	case ZigTypeIdFn:
4421	4562	case ZigTypeIdOpaque:
4422		case ZigTypeIdPromise:
4423	4563	case ZigTypeIdErrorSet:
4424	4564	case ZigTypeIdEnum:
	4565	case ZigTypeIdFnFrame:
	4566	case ZigTypeIdAnyFrame:
4425	4567	return false;
4426	4568
4427	4569	case ZigTypeIdPointer:
...	...	@@ -4489,11 +4631,12 @@ static bool return_type_is_cacheable(ZigType *return_type) {
4489	4631	case ZigTypeIdBoundFn:
4490	4632	case ZigTypeIdFn:
4491	4633	case ZigTypeIdOpaque:
4492		case ZigTypeIdPromise:
4493	4634	case ZigTypeIdErrorSet:
4494	4635	case ZigTypeIdEnum:
4495	4636	case ZigTypeIdPointer:
4496	4637	case ZigTypeIdVector:
	4638	case ZigTypeIdFnFrame:
	4639	case ZigTypeIdAnyFrame:
4497	4640	return true;
4498	4641
4499	4642	case ZigTypeIdArray:
...	...	@@ -4624,8 +4767,9 @@ OnePossibleValue type_has_one_possible_value(CodeGen g, ZigType type_entry) {
4624	4767	case ZigTypeIdFn:
4625	4768	case ZigTypeIdBool:
4626	4769	case ZigTypeIdFloat:
4627		case ZigTypeIdPromise:
4628	4770	case ZigTypeIdErrorUnion:
	4771	case ZigTypeIdFnFrame:
	4772	case ZigTypeIdAnyFrame:
4629	4773	return OnePossibleValueNo;
4630	4774	case ZigTypeIdUndefined:
4631	4775	case ZigTypeIdNull:
...	...	@@ -4713,7 +4857,8 @@ ReqCompTime type_requires_comptime(CodeGen g, ZigType type_entry) {
4713	4857	case ZigTypeIdFloat:
4714	4858	case ZigTypeIdVoid:
4715	4859	case ZigTypeIdUnreachable:
4716		case ZigTypeIdPromise:
	4860	case ZigTypeIdFnFrame:
	4861	case ZigTypeIdAnyFrame:
4717	4862	return ReqCompTimeNo;
4718	4863	}
4719	4864	zig_unreachable();
...	...	@@ -5032,6 +5177,221 @@ Error ensure_complete_type(CodeGen g, ZigType type_entry) {
5032	5177	return type_resolve(g, type_entry, ResolveStatusSizeKnown);
5033	5178	}
5034	5179
	5180	static ZigType get_async_fn_type(CodeGen g, ZigType *orig_fn_type) {
	5181	if (orig_fn_type->data.fn.fn_type_id.cc == CallingConventionAsync)
	5182	return orig_fn_type;
	5183
	5184	ZigType *fn_type = allocate_nonzero<ZigType>(1);
	5185	fn_type = orig_fn_type;
	5186	fn_type->data.fn.fn_type_id.cc = CallingConventionAsync;
	5187	fn_type->llvm_type = nullptr;
	5188	fn_type->llvm_di_type = nullptr;
	5189
	5190	return fn_type;
	5191	}
	5192
	5193	static Error resolve_async_frame(CodeGen g, ZigType frame_type) {
	5194	Error err;
	5195
	5196	if (frame_type->data.frame.locals_struct != nullptr)
	5197	return ErrorNone;
	5198
	5199	ZigFn *fn = frame_type->data.frame.fn;
	5200	switch (fn->anal_state) {
	5201	case FnAnalStateInvalid:
	5202	return ErrorSemanticAnalyzeFail;
	5203	case FnAnalStateComplete:
	5204	break;
	5205	case FnAnalStateReady:
	5206	analyze_fn_body(g, fn);
	5207	if (fn->anal_state == FnAnalStateInvalid)
	5208	return ErrorSemanticAnalyzeFail;
	5209	break;
	5210	case FnAnalStateProbing: {
	5211	ErrorMsg *msg = add_node_error(g, fn->proto_node,
	5212	buf_sprintf("cannot resolve '%s': function not fully analyzed yet",
	5213	buf_ptr(&frame_type->name)));
	5214	ir_add_analysis_trace(fn->ir_executable.analysis, msg,
	5215	buf_sprintf("depends on its own frame here"));
	5216	return ErrorSemanticAnalyzeFail;
	5217	}
	5218	}
	5219	analyze_fn_async(g, fn, false);
	5220	if (fn->anal_state == FnAnalStateInvalid)
	5221	return ErrorSemanticAnalyzeFail;
	5222
	5223	if (!fn_is_async(fn)) {
	5224	ZigType *fn_type = fn->type_entry;
	5225	FnTypeId *fn_type_id = &fn_type->data.fn.fn_type_id;
	5226	ZigType *ptr_return_type = get_pointer_to_type(g, fn_type_id->return_type, false);
	5227
	5228	// label (grep this): [fn_frame_struct_layout]
	5229	ZigList<SrcField> fields = {};
	5230
	5231	fields.append({"@fn_ptr", g->builtin_types.entry_usize, 0});
	5232	fields.append({"@resume_index", g->builtin_types.entry_usize, 0});
	5233	fields.append({"@awaiter", g->builtin_types.entry_usize, 0});
	5234
	5235	fields.append({"@result_ptr_callee", ptr_return_type, 0});
	5236	fields.append({"@result_ptr_awaiter", ptr_return_type, 0});
	5237	fields.append({"@result", fn_type_id->return_type, 0});
	5238
	5239	if (codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type)) {
	5240	ZigType *ptr_to_stack_trace_type = get_pointer_to_type(g, get_stack_trace_type(g), false);
	5241	fields.append({"@ptr_stack_trace_callee", ptr_to_stack_trace_type, 0});
	5242	fields.append({"@ptr_stack_trace_awaiter", ptr_to_stack_trace_type, 0});
	5243
	5244	fields.append({"@stack_trace", get_stack_trace_type(g), 0});
	5245	fields.append({"@instruction_addresses",
	5246	get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count), 0});
	5247	}
	5248
	5249	frame_type->data.frame.locals_struct = get_struct_type(g, buf_ptr(&frame_type->name),
	5250	fields.items, fields.length, target_fn_align(g->zig_target));
	5251	frame_type->abi_size = frame_type->data.frame.locals_struct->abi_size;
	5252	frame_type->abi_align = frame_type->data.frame.locals_struct->abi_align;
	5253	frame_type->size_in_bits = frame_type->data.frame.locals_struct->size_in_bits;
	5254
	5255	return ErrorNone;
	5256	}
	5257
	5258	ZigType *fn_type = get_async_fn_type(g, fn->type_entry);
	5259
	5260	if (fn->analyzed_executable.need_err_code_spill) {
	5261	IrInstructionAllocaGen *alloca_gen = allocate<IrInstructionAllocaGen>(1);
	5262	alloca_gen->base.id = IrInstructionIdAllocaGen;
	5263	alloca_gen->base.source_node = fn->proto_node;
	5264	alloca_gen->base.scope = fn->child_scope;
	5265	alloca_gen->base.value.type = get_pointer_to_type(g, g->builtin_types.entry_global_error_set, false);
	5266	alloca_gen->base.ref_count = 1;
	5267	alloca_gen->name_hint = "";
	5268	fn->alloca_gen_list.append(alloca_gen);
	5269	fn->err_code_spill = &alloca_gen->base;
	5270	}
	5271
	5272	for (size_t i = 0; i < fn->call_list.length; i += 1) {
	5273	IrInstructionCallGen *call = fn->call_list.at(i);
	5274	ZigFn *callee = call->fn_entry;
	5275	if (callee == nullptr) {
	5276	add_node_error(g, call->base.source_node,
	5277	buf_sprintf("function is not comptime-known; @asyncCall required"));
	5278	return ErrorSemanticAnalyzeFail;
	5279	}
	5280	if (callee->body_node == nullptr) {
	5281	continue;
	5282	}
	5283	if (callee->anal_state == FnAnalStateProbing) {
	5284	ErrorMsg *msg = add_node_error(g, fn->proto_node,
	5285	buf_sprintf("unable to determine async function frame of '%s'", buf_ptr(&fn->symbol_name)));
	5286	ErrorMsg *note = add_error_note(g, msg, call->base.source_node,
	5287	buf_sprintf("analysis of function '%s' depends on the frame", buf_ptr(&callee->symbol_name)));
	5288	ir_add_analysis_trace(callee->ir_executable.analysis, note,
	5289	buf_sprintf("depends on the frame here"));
	5290	return ErrorSemanticAnalyzeFail;
	5291	}
	5292
	5293	analyze_fn_body(g, callee);
	5294	if (callee->anal_state == FnAnalStateInvalid) {
	5295	frame_type->data.frame.locals_struct = g->builtin_types.entry_invalid;
	5296	return ErrorSemanticAnalyzeFail;
	5297	}
	5298	analyze_fn_async(g, callee, true);
	5299	if (!fn_is_async(callee))
	5300	continue;
	5301
	5302	ZigType *callee_frame_type = get_fn_frame_type(g, callee);
	5303
	5304	IrInstructionAllocaGen *alloca_gen = allocate<IrInstructionAllocaGen>(1);
	5305	alloca_gen->base.id = IrInstructionIdAllocaGen;
	5306	alloca_gen->base.source_node = call->base.source_node;
	5307	alloca_gen->base.scope = call->base.scope;
	5308	alloca_gen->base.value.type = get_pointer_to_type(g, callee_frame_type, false);
	5309	alloca_gen->base.ref_count = 1;
	5310	alloca_gen->name_hint = "";
	5311	fn->alloca_gen_list.append(alloca_gen);
	5312	call->frame_result_loc = &alloca_gen->base;
	5313	}
	5314	FnTypeId *fn_type_id = &fn_type->data.fn.fn_type_id;
	5315	ZigType *ptr_return_type = get_pointer_to_type(g, fn_type_id->return_type, false);
	5316
	5317	// label (grep this): [fn_frame_struct_layout]
	5318	ZigList<SrcField> fields = {};
	5319
	5320	fields.append({"@fn_ptr", fn_type, 0});
	5321	fields.append({"@resume_index", g->builtin_types.entry_usize, 0});
	5322	fields.append({"@awaiter", g->builtin_types.entry_usize, 0});
	5323
	5324	fields.append({"@result_ptr_callee", ptr_return_type, 0});
	5325	fields.append({"@result_ptr_awaiter", ptr_return_type, 0});
	5326	fields.append({"@result", fn_type_id->return_type, 0});
	5327
	5328	if (codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type)) {
	5329	ZigType *ptr_stack_trace_type = get_pointer_to_type(g, get_stack_trace_type(g), false);
	5330	fields.append({"@ptr_stack_trace_callee", ptr_stack_trace_type, 0});
	5331	fields.append({"@ptr_stack_trace_awaiter", ptr_stack_trace_type, 0});
	5332	}
	5333
	5334	for (size_t arg_i = 0; arg_i < fn_type_id->param_count; arg_i += 1) {
	5335	FnTypeParamInfo *param_info = &fn_type_id->param_info[arg_i];
	5336	AstNode *param_decl_node = get_param_decl_node(fn, arg_i);
	5337	Buf *param_name;
	5338	bool is_var_args = param_decl_node && param_decl_node->data.param_decl.is_var_args;
	5339	if (param_decl_node && !is_var_args) {
	5340	param_name = param_decl_node->data.param_decl.name;
	5341	} else {
	5342	param_name = buf_sprintf("@arg%" ZIG_PRI_usize, arg_i);
	5343	}
	5344	ZigType *param_type = param_info->type;
	5345
	5346	fields.append({buf_ptr(param_name), param_type, 0});
	5347	}
	5348
	5349	if (codegen_fn_has_err_ret_tracing_stack(g, fn, true)) {
	5350	fields.append({"@stack_trace", get_stack_trace_type(g), 0});
	5351	fields.append({"@instruction_addresses",
	5352	get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count), 0});
	5353	}
	5354
	5355	for (size_t alloca_i = 0; alloca_i < fn->alloca_gen_list.length; alloca_i += 1) {
	5356	IrInstructionAllocaGen *instruction = fn->alloca_gen_list.at(alloca_i);
	5357	instruction->field_index = SIZE_MAX;
	5358	ZigType *ptr_type = instruction->base.value.type;
	5359	assert(ptr_type->id == ZigTypeIdPointer);
	5360	ZigType *child_type = ptr_type->data.pointer.child_type;
	5361	if (!type_has_bits(child_type))
	5362	continue;
	5363	if (instruction->base.ref_count == 0)
	5364	continue;
	5365	if (instruction->base.value.special != ConstValSpecialRuntime) {
	5366	if (const_ptr_pointee(nullptr, g, &instruction->base.value, nullptr)->special !=
	5367	ConstValSpecialRuntime)
	5368	{
	5369	continue;
	5370	}
	5371	}
	5372	if ((err = type_resolve(g, child_type, ResolveStatusSizeKnown))) {
	5373	return err;
	5374	}
	5375	const char *name;
	5376	if (*instruction->name_hint == 0) {
	5377	name = buf_ptr(buf_sprintf("@local%" ZIG_PRI_usize, alloca_i));
	5378	} else {
	5379	name = buf_ptr(buf_sprintf("%s.%" ZIG_PRI_usize, instruction->name_hint, alloca_i));
	5380	}
	5381	instruction->field_index = fields.length;
	5382
	5383	fields.append({name, child_type, instruction->align});
	5384	}
	5385
	5386
	5387	frame_type->data.frame.locals_struct = get_struct_type(g, buf_ptr(&frame_type->name),
	5388	fields.items, fields.length, target_fn_align(g->zig_target));
	5389	frame_type->abi_size = frame_type->data.frame.locals_struct->abi_size;
	5390	frame_type->abi_align = frame_type->data.frame.locals_struct->abi_align;
	5391	frame_type->size_in_bits = frame_type->data.frame.locals_struct->size_in_bits;
	5392	return ErrorNone;
	5393	}
	5394
5035	5395	Error type_resolve(CodeGen g, ZigType ty, ResolveStatus status) {
5036	5396	if (type_is_invalid(ty))
5037	5397	return ErrorSemanticAnalyzeFail;
...	...	@@ -5056,6 +5416,8 @@ Error type_resolve(CodeGen g, ZigType ty, ResolveStatus status) {
5056	5416	return resolve_enum_zero_bits(g, ty);
5057	5417	} else if (ty->id == ZigTypeIdUnion) {
5058	5418	return resolve_union_alignment(g, ty);
	5419	} else if (ty->id == ZigTypeIdFnFrame) {
	5420	return resolve_async_frame(g, ty);
5059	5421	}
5060	5422	return ErrorNone;
5061	5423	case ResolveStatusSizeKnown:
...	...	@@ -5065,6 +5427,8 @@ Error type_resolve(CodeGen g, ZigType ty, ResolveStatus status) {
5065	5427	return resolve_enum_zero_bits(g, ty);
5066	5428	} else if (ty->id == ZigTypeIdUnion) {
5067	5429	return resolve_union_type(g, ty);
	5430	} else if (ty->id == ZigTypeIdFnFrame) {
	5431	return resolve_async_frame(g, ty);
5068	5432	}
5069	5433	return ErrorNone;
5070	5434	case ResolveStatusLLVMFwdDecl:
...	...	@@ -5259,6 +5623,10 @@ bool const_values_equal(CodeGen g, ConstExprValue a, ConstExprValue *b) {
5259	5623	return false;
5260	5624	}
5261	5625	return true;
	5626	case ZigTypeIdFnFrame:
	5627	zig_panic("TODO");
	5628	case ZigTypeIdAnyFrame:
	5629	zig_panic("TODO");
5262	5630	case ZigTypeIdUndefined:
5263	5631	zig_panic("TODO");
5264	5632	case ZigTypeIdNull:
...	...	@@ -5279,7 +5647,6 @@ bool const_values_equal(CodeGen g, ConstExprValue a, ConstExprValue *b) {
5279	5647	case ZigTypeIdBoundFn:
5280	5648	case ZigTypeIdInvalid:
5281	5649	case ZigTypeIdUnreachable:
5282		case ZigTypeIdPromise:
5283	5650	zig_unreachable();
5284	5651	}
5285	5652	zig_unreachable();
...	...	@@ -5612,8 +5979,14 @@ void render_const_value(CodeGen g, Buf buf, ConstExprValue *const_val) {
5612	5979	buf_appendf(buf, "(args value)");
5613	5980	return;
5614	5981	}
5615		case ZigTypeIdPromise:
5616		zig_unreachable();
	5982	case ZigTypeIdFnFrame:
	5983	buf_appendf(buf, "(TODO: async function frame value)");
	5984	return;
	5985
	5986	case ZigTypeIdAnyFrame:
	5987	buf_appendf(buf, "(TODO: anyframe value)");
	5988	return;
	5989
5617	5990	}
5618	5991	zig_unreachable();
5619	5992	}
...	...	@@ -5627,6 +6000,15 @@ ZigType make_int_type(CodeGen g, bool is_signed, uint32_t size_in_bits) {
5627	6000	entry->llvm_type = LLVMIntType(size_in_bits);
5628	6001	entry->abi_size = LLVMABISizeOfType(g->target_data_ref, entry->llvm_type);
5629	6002	entry->abi_align = LLVMABIAlignmentOfType(g->target_data_ref, entry->llvm_type);
	6003
	6004	if (size_in_bits >= 128) {
	6005	// Override the incorrect alignment reported by LLVM. Clang does this as well.
	6006	// On x86_64 there are some instructions like CMPXCHG16B which require this.
	6007	// On all targets, integers 128 bits and above have ABI alignment of 16.
	6008	// See: https://github.com/ziglang/zig/issues/2987
	6009	assert(entry->abi_align == 8); // if this trips we can remove the workaround
	6010	entry->abi_align = 16;
	6011	}
5630	6012	}
5631	6013
5632	6014	const char u_or_i = is_signed ? 'i' : 'u';
...	...	@@ -5660,7 +6042,8 @@ uint32_t type_id_hash(TypeId x) {
5660	6042	case ZigTypeIdFn:
5661	6043	case ZigTypeIdBoundFn:
5662	6044	case ZigTypeIdArgTuple:
5663		case ZigTypeIdPromise:
	6045	case ZigTypeIdFnFrame:
	6046	case ZigTypeIdAnyFrame:
5664	6047	zig_unreachable();
5665	6048	case ZigTypeIdErrorUnion:
5666	6049	return hash_ptr(x.data.error_union.err_set_type) ^ hash_ptr(x.data.error_union.payload_type);
...	...	@@ -5702,7 +6085,6 @@ bool type_id_eql(TypeId a, TypeId b) {
5702	6085	case ZigTypeIdUndefined:
5703	6086	case ZigTypeIdNull:
5704	6087	case ZigTypeIdOptional:
5705		case ZigTypeIdPromise:
5706	6088	case ZigTypeIdErrorSet:
5707	6089	case ZigTypeIdEnum:
5708	6090	case ZigTypeIdUnion:
...	...	@@ -5710,6 +6092,8 @@ bool type_id_eql(TypeId a, TypeId b) {
5710	6092	case ZigTypeIdBoundFn:
5711	6093	case ZigTypeIdArgTuple:
5712	6094	case ZigTypeIdOpaque:
	6095	case ZigTypeIdFnFrame:
	6096	case ZigTypeIdAnyFrame:
5713	6097	zig_unreachable();
5714	6098	case ZigTypeIdErrorUnion:
5715	6099	return a.data.error_union.err_set_type == b.data.error_union.err_set_type &&
...	...	@@ -5875,7 +6259,8 @@ static const ZigTypeId all_type_ids[] = {
5875	6259	ZigTypeIdBoundFn,
5876	6260	ZigTypeIdArgTuple,
5877	6261	ZigTypeIdOpaque,
5878		ZigTypeIdPromise,
	6262	ZigTypeIdFnFrame,
	6263	ZigTypeIdAnyFrame,
5879	6264	ZigTypeIdVector,
5880	6265	ZigTypeIdEnumLiteral,
5881	6266	};
...	...	@@ -5939,12 +6324,14 @@ size_t type_id_index(ZigType *entry) {
5939	6324	return 20;
5940	6325	case ZigTypeIdOpaque:
5941	6326	return 21;
5942		case ZigTypeIdPromise:
	6327	case ZigTypeIdFnFrame:
5943	6328	return 22;
5944		case ZigTypeIdVector:
	6329	case ZigTypeIdAnyFrame:
5945	6330	return 23;
5946		case ZigTypeIdEnumLiteral:
	6331	case ZigTypeIdVector:
5947	6332	return 24;
	6333	case ZigTypeIdEnumLiteral:
	6334	return 25;
5948	6335	}
5949	6336	zig_unreachable();
5950	6337	}
...	...	@@ -5999,10 +6386,12 @@ const char *type_id_name(ZigTypeId id) {
5999	6386	return "ArgTuple";
6000	6387	case ZigTypeIdOpaque:
6001	6388	return "Opaque";
6002		case ZigTypeIdPromise:
6003		return "Promise";
6004	6389	case ZigTypeIdVector:
6005	6390	return "Vector";
	6391	case ZigTypeIdFnFrame:
	6392	return "Frame";
	6393	case ZigTypeIdAnyFrame:
	6394	return "AnyFrame";
6006	6395	}
6007	6396	zig_unreachable();
6008	6397	}
...	...	@@ -6067,19 +6456,12 @@ bool type_is_global_error_set(ZigType *err_set_type) {
6067	6456	return err_set_type->data.error_set.err_count == UINT32_MAX;
6068	6457	}
6069	6458
6070		uint32_t get_coro_frame_align_bytes(CodeGen *g) {
6071		uint32_t a = g->pointer_size_bytes * 2;
6072		// promises have at least alignment 8 so that we can have 3 extra bits when doing atomicrmw
6073		if (a < 8) a = 8;
6074		return a;
6075		}
6076
6077	6459	bool type_can_fail(ZigType *type_entry) {
6078	6460	return type_entry->id == ZigTypeIdErrorUnion \|\| type_entry->id == ZigTypeIdErrorSet;
6079	6461	}
6080	6462
6081	6463	bool fn_type_can_fail(FnTypeId *fn_type_id) {
6082		return type_can_fail(fn_type_id->return_type) \|\| fn_type_id->cc == CallingConventionAsync;
	6464	return type_can_fail(fn_type_id->return_type);
6083	6465	}
6084	6466
6085	6467	// ErrorNone - result pointer has the type
...	...	@@ -6449,7 +6831,9 @@ static void resolve_llvm_types_slice(CodeGen g, ZigType type, ResolveStatus wa
6449	6831	type->data.structure.resolve_status = ResolveStatusLLVMFull;
6450	6832	}
6451	6833
6452		static void resolve_llvm_types_struct(CodeGen g, ZigType struct_type, ResolveStatus wanted_resolve_status) {
	6834	static void resolve_llvm_types_struct(CodeGen g, ZigType struct_type, ResolveStatus wanted_resolve_status,
	6835	ZigType *async_frame_type)
	6836	{
6453	6837	assert(struct_type->id == ZigTypeIdStruct);
6454	6838	assert(struct_type->data.structure.resolve_status != ResolveStatusInvalid);
6455	6839	assert(struct_type->data.structure.resolve_status >= ResolveStatusSizeKnown);
...	...	@@ -6486,10 +6870,9 @@ static void resolve_llvm_types_struct(CodeGen g, ZigType struct_type, ResolveS
6486	6870	}
6487	6871
6488	6872	size_t field_count = struct_type->data.structure.src_field_count;
6489		size_t gen_field_count = struct_type->data.structure.gen_field_count;
6490		LLVMTypeRef *element_types = allocate<LLVMTypeRef>(gen_field_count);
	6873	// Every field could potentially have a generated padding field after it.
	6874	LLVMTypeRef element_types = allocate<LLVMTypeRef>(field_count 2);
6491	6875
6492		size_t gen_field_index = 0;
6493	6876	bool packed = (struct_type->data.structure.layout == ContainerLayoutPacked);
6494	6877	size_t packed_bits_offset = 0;
6495	6878	size_t first_packed_bits_offset_misalign = SIZE_MAX;
...	...	@@ -6497,20 +6880,36 @@ static void resolve_llvm_types_struct(CodeGen g, ZigType struct_type, ResolveS
6497	6880
6498	6881	// trigger all the recursive get_llvm_type calls
6499	6882	for (size_t i = 0; i < field_count; i += 1) {
6500		TypeStructField *type_struct_field = &struct_type->data.structure.fields[i];
6501		ZigType *field_type = type_struct_field->type_entry;
	6883	TypeStructField *field = &struct_type->data.structure.fields[i];
	6884	ZigType *field_type = field->type_entry;
6502	6885	if (!type_has_bits(field_type))
6503	6886	continue;
6504	6887	(void)get_llvm_type(g, field_type);
6505	6888	if (struct_type->data.structure.resolve_status >= wanted_resolve_status) return;
6506	6889	}
6507	6890
6508		for (size_t i = 0; i < field_count; i += 1) {
6509		TypeStructField *type_struct_field = &struct_type->data.structure.fields[i];
6510		ZigType *field_type = type_struct_field->type_entry;
	6891	size_t gen_field_index = 0;
6511	6892
	6893	// Calculate what LLVM thinks the ABI align of the struct will be. We do this to avoid
	6894	// inserting padding bytes where LLVM would do it automatically.
	6895	size_t llvm_struct_abi_align = 0;
	6896	for (size_t i = 0; i < field_count; i += 1) {
	6897	ZigType *field_type = struct_type->data.structure.fields[i].type_entry;
6512	6898	if (!type_has_bits(field_type))
6513	6899	continue;
	6900	LLVMTypeRef field_llvm_type = get_llvm_type(g, field_type);
	6901	size_t llvm_field_abi_align = LLVMABIAlignmentOfType(g->target_data_ref, field_llvm_type);
	6902	llvm_struct_abi_align = max(llvm_struct_abi_align, llvm_field_abi_align);
	6903	}
	6904
	6905	for (size_t i = 0; i < field_count; i += 1) {
	6906	TypeStructField *field = &struct_type->data.structure.fields[i];
	6907	ZigType *field_type = field->type_entry;
	6908
	6909	if (!type_has_bits(field_type)) {
	6910	field->gen_index = SIZE_MAX;
	6911	continue;
	6912	}
6514	6913
6515	6914	if (packed) {
6516	6915	size_t field_size_in_bits = type_size_bits(g, field_type);
...	...	@@ -6537,12 +6936,61 @@ static void resolve_llvm_types_struct(CodeGen g, ZigType struct_type, ResolveS
6537	6936	}
6538	6937	packed_bits_offset = next_packed_bits_offset;
6539	6938	} else {
6540		element_types[gen_field_index] = get_llvm_type(g, field_type);
6541
	6939	LLVMTypeRef llvm_type;
	6940	if (i == 0 && async_frame_type != nullptr) {
	6941	assert(async_frame_type->id == ZigTypeIdFnFrame);
	6942	assert(field_type->id == ZigTypeIdFn);
	6943	resolve_llvm_types_fn(g, async_frame_type->data.frame.fn);
	6944	llvm_type = LLVMPointerType(async_frame_type->data.frame.fn->raw_type_ref, 0);
	6945	} else {
	6946	llvm_type = get_llvm_type(g, field_type);
	6947	}
	6948	element_types[gen_field_index] = llvm_type;
	6949	field->gen_index = gen_field_index;
6542	6950	gen_field_index += 1;
	6951
	6952	// find the next non-zero-byte field for offset calculations
	6953	size_t next_src_field_index = i + 1;
	6954	for (; next_src_field_index < field_count; next_src_field_index += 1) {
	6955	if (type_has_bits(struct_type->data.structure.fields[next_src_field_index].type_entry))
	6956	break;
	6957	}
	6958	size_t next_abi_align;
	6959	if (next_src_field_index == field_count) {
	6960	next_abi_align = struct_type->abi_align;
	6961	} else {
	6962	if (struct_type->data.structure.fields[next_src_field_index].align == 0) {
	6963	next_abi_align = struct_type->data.structure.fields[next_src_field_index].type_entry->abi_align;
	6964	} else {
	6965	next_abi_align = struct_type->data.structure.fields[next_src_field_index].align;
	6966	}
	6967	}
	6968	size_t llvm_next_abi_align = (next_src_field_index == field_count) ?
	6969	llvm_struct_abi_align :
	6970	LLVMABIAlignmentOfType(g->target_data_ref,
	6971	get_llvm_type(g, struct_type->data.structure.fields[next_src_field_index].type_entry));
	6972
	6973	size_t next_offset = next_field_offset(field->offset, struct_type->abi_align,
	6974	field_type->abi_size, next_abi_align);
	6975	size_t llvm_next_offset = next_field_offset(field->offset, llvm_struct_abi_align,
	6976	LLVMABISizeOfType(g->target_data_ref, llvm_type), llvm_next_abi_align);
	6977
	6978	assert(next_offset >= llvm_next_offset);
	6979	if (next_offset > llvm_next_offset) {
	6980	size_t pad_bytes = next_offset - (field->offset + field_type->abi_size);
	6981	if (pad_bytes != 0) {
	6982	LLVMTypeRef pad_llvm_type = LLVMArrayType(LLVMInt8Type(), pad_bytes);
	6983	element_types[gen_field_index] = pad_llvm_type;
	6984	gen_field_index += 1;
	6985	}
	6986	}
6543	6987	}
6544	6988	debug_field_count += 1;
6545	6989	}
	6990	if (!packed) {
	6991	struct_type->data.structure.gen_field_count = gen_field_index;
	6992	}
	6993
6546	6994	if (first_packed_bits_offset_misalign != SIZE_MAX) {
6547	6995	size_t full_bit_count = packed_bits_offset - first_packed_bits_offset_misalign;
6548	6996	size_t full_abi_size = get_abi_size_bytes(full_bit_count, g->pointer_size_bytes);
...	...	@@ -6551,19 +6999,20 @@ static void resolve_llvm_types_struct(CodeGen g, ZigType struct_type, ResolveS
6551	6999	}
6552	7000
6553	7001	if (type_has_bits(struct_type)) {
6554		LLVMStructSetBody(struct_type->llvm_type, element_types, (unsigned)gen_field_count, packed);
	7002	LLVMStructSetBody(struct_type->llvm_type, element_types,
	7003	(unsigned)struct_type->data.structure.gen_field_count, packed);
6555	7004	}
6556	7005
6557	7006	ZigLLVMDIType *di_element_types = allocate<ZigLLVMDIType>(debug_field_count);
6558	7007	size_t debug_field_index = 0;
6559	7008	for (size_t i = 0; i < field_count; i += 1) {
6560		TypeStructField *type_struct_field = &struct_type->data.structure.fields[i];
6561		size_t gen_field_index = type_struct_field->gen_index;
	7009	TypeStructField *field = &struct_type->data.structure.fields[i];
	7010	size_t gen_field_index = field->gen_index;
6562	7011	if (gen_field_index == SIZE_MAX) {
6563	7012	continue;
6564	7013	}
6565	7014
6566		ZigType *field_type = type_struct_field->type_entry;
	7015	ZigType *field_type = field->type_entry;
6567	7016
6568	7017	// if the field is a function, actually the debug info should be a pointer.
6569	7018	ZigLLVMDIType *field_di_type;
...	...	@@ -6581,13 +7030,13 @@ static void resolve_llvm_types_struct(CodeGen g, ZigType struct_type, ResolveS
6581	7030	uint64_t debug_align_in_bits;
6582	7031	uint64_t debug_offset_in_bits;
6583	7032	if (packed) {
6584		debug_size_in_bits = type_struct_field->type_entry->size_in_bits;
6585		debug_align_in_bits = 8 * type_struct_field->type_entry->abi_align;
6586		debug_offset_in_bits = 8 * type_struct_field->offset + type_struct_field->bit_offset_in_host;
	7033	debug_size_in_bits = field->type_entry->size_in_bits;
	7034	debug_align_in_bits = 8 * field->type_entry->abi_align;
	7035	debug_offset_in_bits = 8 * field->offset + field->bit_offset_in_host;
6587	7036	} else {
6588	7037	debug_size_in_bits = 8 * get_store_size_bytes(field_type->size_in_bits);
6589	7038	debug_align_in_bits = 8 * field_type->abi_align;
6590		debug_offset_in_bits = 8 * type_struct_field->offset;
	7039	debug_offset_in_bits = 8 * field->offset;
6591	7040	}
6592	7041	unsigned line;
6593	7042	if (decl_node != nullptr) {
...	...	@@ -6597,7 +7046,7 @@ static void resolve_llvm_types_struct(CodeGen g, ZigType struct_type, ResolveS
6597	7046	line = 0;
6598	7047	}
6599	7048	di_element_types[debug_field_index] = ZigLLVMCreateDebugMemberType(g->dbuilder,
6600		ZigLLVMTypeToScope(struct_type->llvm_di_type), buf_ptr(type_struct_field->name),
	7049	ZigLLVMTypeToScope(struct_type->llvm_di_type), buf_ptr(field->name),
6601	7050	di_file, line,
6602	7051	debug_size_in_bits,
6603	7052	debug_align_in_bits,
...	...	@@ -6838,7 +7287,7 @@ static void resolve_llvm_types_union(CodeGen g, ZigType union_type, ResolveSta
6838	7287	union_type->data.unionation.resolve_status = ResolveStatusLLVMFull;
6839	7288	}
6840	7289
6841		static void resolve_llvm_types_pointer(CodeGen g, ZigType type) {
	7290	static void resolve_llvm_types_pointer(CodeGen g, ZigType type, ResolveStatus wanted_resolve_status) {
6842	7291	if (type->llvm_di_type != nullptr) return;
6843	7292
6844	7293	if (!type_has_bits(type)) {
...	...	@@ -6867,7 +7316,7 @@ static void resolve_llvm_types_pointer(CodeGen g, ZigType type) {
6867	7316	uint64_t debug_align_in_bits = 8*type->abi_align;
6868	7317	type->llvm_di_type = ZigLLVMCreateDebugPointerType(g->dbuilder, elem_type->llvm_di_type,
6869	7318	debug_size_in_bits, debug_align_in_bits, buf_ptr(&type->name));
6870		assertNoError(type_resolve(g, elem_type, ResolveStatusLLVMFull));
	7319	assertNoError(type_resolve(g, elem_type, wanted_resolve_status));
6871	7320	} else {
6872	7321	ZigType host_int_type = get_int_type(g, false, type->data.pointer.host_int_bytes 8);
6873	7322	LLVMTypeRef host_int_llvm_type = get_llvm_type(g, host_int_type);
...	...	@@ -6993,10 +7442,17 @@ static void resolve_llvm_types_error_union(CodeGen g, ZigType type) {
6993	7442	} else {
6994	7443	LLVMTypeRef err_set_llvm_type = get_llvm_type(g, err_set_type);
6995	7444	LLVMTypeRef payload_llvm_type = get_llvm_type(g, payload_type);
6996		LLVMTypeRef elem_types[2];
	7445	LLVMTypeRef elem_types[3];
6997	7446	elem_types[err_union_err_index] = err_set_llvm_type;
6998	7447	elem_types[err_union_payload_index] = payload_llvm_type;
	7448
6999	7449	type->llvm_type = LLVMStructType(elem_types, 2, false);
	7450	if (LLVMABISizeOfType(g->target_data_ref, type->llvm_type) != type->abi_size) {
	7451	// we need to do our own padding
	7452	type->data.error_union.pad_llvm_type = LLVMArrayType(LLVMInt8Type(), type->data.error_union.pad_bytes);
	7453	elem_types[2] = type->data.error_union.pad_llvm_type;
	7454	type->llvm_type = LLVMStructType(elem_types, 3, false);
	7455	}
7000	7456
7001	7457	ZigLLVMDIScope *compile_unit_scope = ZigLLVMCompileUnitToScope(g->compile_unit);
7002	7458	ZigLLVMDIFile *di_file = nullptr;
...	...	@@ -7068,7 +7524,7 @@ static void resolve_llvm_types_array(CodeGen g, ZigType type) {
7068	7524	debug_align_in_bits, get_llvm_di_type(g, elem_type), (int)type->data.array.len);
7069	7525	}
7070	7526
7071		static void resolve_llvm_types_fn(CodeGen g, ZigType fn_type) {
	7527	static void resolve_llvm_types_fn_type(CodeGen g, ZigType fn_type) {
7072	7528	if (fn_type->llvm_di_type != nullptr) return;
7073	7529
7074	7530	FnTypeId *fn_type_id = &fn_type->data.fn.fn_type_id;
...	...	@@ -7085,67 +7541,73 @@ static void resolve_llvm_types_fn(CodeGen g, ZigType fn_type) {
7085	7541	// +1 for maybe first argument the error return trace
7086	7542	// +2 for maybe arguments async allocator and error code pointer
7087	7543	ZigList<ZigLLVMDIType *> param_di_types = {};
7088		param_di_types.append(get_llvm_di_type(g, fn_type_id->return_type));
7089	7544	ZigType *gen_return_type;
7090	7545	if (is_async) {
7091		gen_return_type = get_pointer_to_type(g, g->builtin_types.entry_u8, false);
	7546	gen_return_type = g->builtin_types.entry_void;
	7547	param_di_types.append(get_llvm_di_type(g, gen_return_type));
7092	7548	} else if (!type_has_bits(fn_type_id->return_type)) {
7093	7549	gen_return_type = g->builtin_types.entry_void;
	7550	param_di_types.append(get_llvm_di_type(g, gen_return_type));
7094	7551	} else if (first_arg_return) {
	7552	gen_return_type = g->builtin_types.entry_void;
	7553	param_di_types.append(get_llvm_di_type(g, gen_return_type));
7095	7554	ZigType *gen_type = get_pointer_to_type(g, fn_type_id->return_type, false);
7096	7555	gen_param_types.append(get_llvm_type(g, gen_type));
7097	7556	param_di_types.append(get_llvm_di_type(g, gen_type));
7098		gen_return_type = g->builtin_types.entry_void;
7099	7557	} else {
7100	7558	gen_return_type = fn_type_id->return_type;
	7559	param_di_types.append(get_llvm_di_type(g, gen_return_type));
7101	7560	}
7102	7561	fn_type->data.fn.gen_return_type = gen_return_type;
7103	7562
7104		if (prefix_arg_error_return_trace) {
7105		ZigType *gen_type = get_ptr_to_stack_trace_type(g);
	7563	if (prefix_arg_error_return_trace && !is_async) {
	7564	ZigType *gen_type = get_pointer_to_type(g, get_stack_trace_type(g), false);
7106	7565	gen_param_types.append(get_llvm_type(g, gen_type));
7107	7566	param_di_types.append(get_llvm_di_type(g, gen_type));
7108	7567	}
7109	7568	if (is_async) {
7110		{
7111		// async allocator param
7112		ZigType *gen_type = fn_type_id->async_allocator_type;
7113		gen_param_types.append(get_llvm_type(g, gen_type));
7114		param_di_types.append(get_llvm_di_type(g, gen_type));
7115		}
	7569	fn_type->data.fn.gen_param_info = allocate<FnGenParamInfo>(2);
7116	7570
7117		{
7118		// error code pointer
7119		ZigType *gen_type = get_pointer_to_type(g, g->builtin_types.entry_global_error_set, false);
7120		gen_param_types.append(get_llvm_type(g, gen_type));
7121		param_di_types.append(get_llvm_di_type(g, gen_type));
7122		}
7123		}
	7571	ZigType *frame_type = get_any_frame_type(g, fn_type_id->return_type);
	7572	gen_param_types.append(get_llvm_type(g, frame_type));
	7573	param_di_types.append(get_llvm_di_type(g, frame_type));
7124	7574
7125		fn_type->data.fn.gen_param_info = allocate<FnGenParamInfo>(fn_type_id->param_count);
7126		for (size_t i = 0; i < fn_type_id->param_count; i += 1) {
7127		FnTypeParamInfo *src_param_info = &fn_type->data.fn.fn_type_id.param_info[i];
7128		ZigType *type_entry = src_param_info->type;
7129		FnGenParamInfo *gen_param_info = &fn_type->data.fn.gen_param_info[i];
	7575	fn_type->data.fn.gen_param_info[0].src_index = 0;
	7576	fn_type->data.fn.gen_param_info[0].gen_index = 0;
	7577	fn_type->data.fn.gen_param_info[0].type = frame_type;
7130	7578
7131		gen_param_info->src_index = i;
7132		gen_param_info->gen_index = SIZE_MAX;
	7579	gen_param_types.append(get_llvm_type(g, g->builtin_types.entry_usize));
	7580	param_di_types.append(get_llvm_di_type(g, g->builtin_types.entry_usize));
7133	7581
7134		if (is_c_abi \|\| !type_has_bits(type_entry))
7135		continue;
	7582	fn_type->data.fn.gen_param_info[1].src_index = 1;
	7583	fn_type->data.fn.gen_param_info[1].gen_index = 1;
	7584	fn_type->data.fn.gen_param_info[1].type = g->builtin_types.entry_usize;
	7585	} else {
	7586	fn_type->data.fn.gen_param_info = allocate<FnGenParamInfo>(fn_type_id->param_count);
	7587	for (size_t i = 0; i < fn_type_id->param_count; i += 1) {
	7588	FnTypeParamInfo *src_param_info = &fn_type->data.fn.fn_type_id.param_info[i];
	7589	ZigType *type_entry = src_param_info->type;
	7590	FnGenParamInfo *gen_param_info = &fn_type->data.fn.gen_param_info[i];
7136	7591
7137		ZigType *gen_type;
7138		if (handle_is_ptr(type_entry)) {
7139		gen_type = get_pointer_to_type(g, type_entry, true);
7140		gen_param_info->is_byval = true;
7141		} else {
7142		gen_type = type_entry;
7143		}
7144		gen_param_info->gen_index = gen_param_types.length;
7145		gen_param_info->type = gen_type;
7146		gen_param_types.append(get_llvm_type(g, gen_type));
	7592	gen_param_info->src_index = i;
	7593	gen_param_info->gen_index = SIZE_MAX;
7147	7594
7148		param_di_types.append(get_llvm_di_type(g, gen_type));
	7595	if (is_c_abi \|\| !type_has_bits(type_entry))
	7596	continue;
	7597
	7598	ZigType *gen_type;
	7599	if (handle_is_ptr(type_entry)) {
	7600	gen_type = get_pointer_to_type(g, type_entry, true);
	7601	gen_param_info->is_byval = true;
	7602	} else {
	7603	gen_type = type_entry;
	7604	}
	7605	gen_param_info->gen_index = gen_param_types.length;
	7606	gen_param_info->type = gen_type;
	7607	gen_param_types.append(get_llvm_type(g, gen_type));
	7608
	7609	param_di_types.append(get_llvm_di_type(g, gen_type));
	7610	}
7149	7611	}
7150	7612
7151	7613	if (is_c_abi) {
...	...	@@ -7161,6 +7623,7 @@ static void resolve_llvm_types_fn(CodeGen g, ZigType fn_type) {
7161	7623	for (size_t i = 0; i < gen_param_types.length; i += 1) {
7162	7624	assert(gen_param_types.items[i] != nullptr);
7163	7625	}
	7626
7164	7627	fn_type->data.fn.raw_type_ref = LLVMFunctionType(get_llvm_type(g, gen_return_type),
7165	7628	gen_param_types.items, (unsigned int)gen_param_types.length, fn_type_id->is_var_args);
7166	7629	fn_type->llvm_type = LLVMPointerType(fn_type->data.fn.raw_type_ref, 0);
...	...	@@ -7170,6 +7633,40 @@ static void resolve_llvm_types_fn(CodeGen g, ZigType fn_type) {
7170	7633	LLVMABIAlignmentOfType(g->target_data_ref, fn_type->llvm_type), "");
7171	7634	}
7172	7635
	7636	void resolve_llvm_types_fn(CodeGen g, ZigFn fn) {
	7637	Error err;
	7638	if (fn->raw_di_type != nullptr) return;
	7639
	7640	ZigType *fn_type = fn->type_entry;
	7641	if (!fn_is_async(fn)) {
	7642	resolve_llvm_types_fn_type(g, fn_type);
	7643	fn->raw_type_ref = fn_type->data.fn.raw_type_ref;
	7644	fn->raw_di_type = fn_type->data.fn.raw_di_type;
	7645	return;
	7646	}
	7647
	7648	ZigType *gen_return_type = g->builtin_types.entry_void;
	7649	ZigList<ZigLLVMDIType *> param_di_types = {};
	7650	ZigList<LLVMTypeRef> gen_param_types = {};
	7651	// first "parameter" is return value
	7652	param_di_types.append(get_llvm_di_type(g, gen_return_type));
	7653
	7654	ZigType *frame_type = get_fn_frame_type(g, fn);
	7655	ZigType *ptr_type = get_pointer_to_type(g, frame_type, false);
	7656	if ((err = type_resolve(g, ptr_type, ResolveStatusLLVMFwdDecl)))
	7657	zig_unreachable();
	7658	gen_param_types.append(ptr_type->llvm_type);
	7659	param_di_types.append(ptr_type->llvm_di_type);
	7660
	7661	// this parameter is used to pass the result pointer when await completes
	7662	gen_param_types.append(get_llvm_type(g, g->builtin_types.entry_usize));
	7663	param_di_types.append(get_llvm_di_type(g, g->builtin_types.entry_usize));
	7664
	7665	fn->raw_type_ref = LLVMFunctionType(get_llvm_type(g, gen_return_type),
	7666	gen_param_types.items, gen_param_types.length, false);
	7667	fn->raw_di_type = ZigLLVMCreateSubroutineType(g->dbuilder, param_di_types.items, (int)param_di_types.length, 0);
	7668	}
	7669
7173	7670	static void resolve_llvm_types_anyerror(CodeGen *g) {
7174	7671	ZigType *entry = g->builtin_types.entry_global_error_set;
7175	7672	entry->llvm_type = get_llvm_type(g, g->err_tag_type);
...	...	@@ -7194,6 +7691,147 @@ static void resolve_llvm_types_anyerror(CodeGen *g) {
7194	7691	get_llvm_di_type(g, g->err_tag_type), "");
7195	7692	}
7196	7693
	7694	static void resolve_llvm_types_async_frame(CodeGen g, ZigType frame_type, ResolveStatus wanted_resolve_status) {
	7695	ZigType *passed_frame_type = fn_is_async(frame_type->data.frame.fn) ? frame_type : nullptr;
	7696	resolve_llvm_types_struct(g, frame_type->data.frame.locals_struct, wanted_resolve_status, passed_frame_type);
	7697	frame_type->llvm_type = frame_type->data.frame.locals_struct->llvm_type;
	7698	frame_type->llvm_di_type = frame_type->data.frame.locals_struct->llvm_di_type;
	7699	}
	7700
	7701	static void resolve_llvm_types_any_frame(CodeGen g, ZigType any_frame_type, ResolveStatus wanted_resolve_status) {
	7702	if (any_frame_type->llvm_di_type != nullptr) return;
	7703
	7704	Buf *name = buf_sprintf("(%s header)", buf_ptr(&any_frame_type->name));
	7705	LLVMTypeRef frame_header_type = LLVMStructCreateNamed(LLVMGetGlobalContext(), buf_ptr(name));
	7706	any_frame_type->llvm_type = LLVMPointerType(frame_header_type, 0);
	7707
	7708	unsigned dwarf_kind = ZigLLVMTag_DW_structure_type();
	7709	ZigLLVMDIFile *di_file = nullptr;
	7710	ZigLLVMDIScope *di_scope = ZigLLVMCompileUnitToScope(g->compile_unit);
	7711	unsigned line = 0;
	7712	ZigLLVMDIType *frame_header_di_type = ZigLLVMCreateReplaceableCompositeType(g->dbuilder,
	7713	dwarf_kind, buf_ptr(name), di_scope, di_file, line);
	7714	any_frame_type->llvm_di_type = ZigLLVMCreateDebugPointerType(g->dbuilder, frame_header_di_type,
	7715	8g->pointer_size_bytes, 8g->builtin_types.entry_usize->abi_align, buf_ptr(&any_frame_type->name));
	7716
	7717	LLVMTypeRef llvm_void = LLVMVoidType();
	7718	LLVMTypeRef arg_types[] = {any_frame_type->llvm_type, g->builtin_types.entry_usize->llvm_type};
	7719	LLVMTypeRef fn_type = LLVMFunctionType(llvm_void, arg_types, 2, false);
	7720	LLVMTypeRef usize_type_ref = get_llvm_type(g, g->builtin_types.entry_usize);
	7721	ZigLLVMDIType *usize_di_type = get_llvm_di_type(g, g->builtin_types.entry_usize);
	7722	ZigLLVMDIScope *compile_unit_scope = ZigLLVMCompileUnitToScope(g->compile_unit);
	7723
	7724	ZigType *result_type = any_frame_type->data.any_frame.result_type;
	7725	ZigType *ptr_result_type = (result_type == nullptr) ? nullptr : get_pointer_to_type(g, result_type, false);
	7726	LLVMTypeRef ptr_fn_llvm_type = LLVMPointerType(fn_type, 0);
	7727	if (result_type == nullptr) {
	7728	g->anyframe_fn_type = ptr_fn_llvm_type;
	7729	}
	7730
	7731	ZigList<LLVMTypeRef> field_types = {};
	7732	ZigList<ZigLLVMDIType *> di_element_types = {};
	7733
	7734	// label (grep this): [fn_frame_struct_layout]
	7735	field_types.append(ptr_fn_llvm_type); // fn_ptr
	7736	field_types.append(usize_type_ref); // resume_index
	7737	field_types.append(usize_type_ref); // awaiter
	7738
	7739	bool have_result_type = result_type != nullptr && type_has_bits(result_type);
	7740	if (have_result_type) {
	7741	field_types.append(get_llvm_type(g, ptr_result_type)); // result_ptr_callee
	7742	field_types.append(get_llvm_type(g, ptr_result_type)); // result_ptr_awaiter
	7743	field_types.append(get_llvm_type(g, result_type)); // result
	7744	if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) {
	7745	ZigType *ptr_stack_trace = get_pointer_to_type(g, get_stack_trace_type(g), false);
	7746	field_types.append(get_llvm_type(g, ptr_stack_trace)); // ptr_stack_trace_callee
	7747	field_types.append(get_llvm_type(g, ptr_stack_trace)); // ptr_stack_trace_awaiter
	7748	}
	7749	}
	7750	LLVMStructSetBody(frame_header_type, field_types.items, field_types.length, false);
	7751
	7752	di_element_types.append(
	7753	ZigLLVMCreateDebugMemberType(g->dbuilder,
	7754	ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "fn_ptr",
	7755	di_file, line,
	7756	8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7757	8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7758	8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, di_element_types.length),
	7759	ZigLLVM_DIFlags_Zero, usize_di_type));
	7760	di_element_types.append(
	7761	ZigLLVMCreateDebugMemberType(g->dbuilder,
	7762	ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "resume_index",
	7763	di_file, line,
	7764	8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7765	8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7766	8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, di_element_types.length),
	7767	ZigLLVM_DIFlags_Zero, usize_di_type));
	7768	di_element_types.append(
	7769	ZigLLVMCreateDebugMemberType(g->dbuilder,
	7770	ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "awaiter",
	7771	di_file, line,
	7772	8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7773	8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7774	8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, di_element_types.length),
	7775	ZigLLVM_DIFlags_Zero, usize_di_type));
	7776
	7777	if (have_result_type) {
	7778	di_element_types.append(
	7779	ZigLLVMCreateDebugMemberType(g->dbuilder,
	7780	ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "result_ptr_callee",
	7781	di_file, line,
	7782	8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7783	8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7784	8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, di_element_types.length),
	7785	ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, ptr_result_type)));
	7786	di_element_types.append(
	7787	ZigLLVMCreateDebugMemberType(g->dbuilder,
	7788	ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "result_ptr_awaiter",
	7789	di_file, line,
	7790	8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7791	8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7792	8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, di_element_types.length),
	7793	ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, ptr_result_type)));
	7794	di_element_types.append(
	7795	ZigLLVMCreateDebugMemberType(g->dbuilder,
	7796	ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "result",
	7797	di_file, line,
	7798	8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7799	8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7800	8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, di_element_types.length),
	7801	ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, result_type)));
	7802
	7803	if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) {
	7804	ZigType *ptr_stack_trace = get_pointer_to_type(g, get_stack_trace_type(g), false);
	7805	di_element_types.append(
	7806	ZigLLVMCreateDebugMemberType(g->dbuilder,
	7807	ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "ptr_stack_trace_callee",
	7808	di_file, line,
	7809	8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7810	8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7811	8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, di_element_types.length),
	7812	ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, ptr_stack_trace)));
	7813	di_element_types.append(
	7814	ZigLLVMCreateDebugMemberType(g->dbuilder,
	7815	ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "ptr_stack_trace_awaiter",
	7816	di_file, line,
	7817	8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7818	8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
	7819	8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, di_element_types.length),
	7820	ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, ptr_stack_trace)));
	7821	}
	7822	};
	7823
	7824	ZigLLVMDIType *replacement_di_type = ZigLLVMCreateDebugStructType(g->dbuilder,
	7825	compile_unit_scope, buf_ptr(name),
	7826	di_file, line,
	7827	8*LLVMABISizeOfType(g->target_data_ref, frame_header_type),
	7828	8*LLVMABIAlignmentOfType(g->target_data_ref, frame_header_type),
	7829	ZigLLVM_DIFlags_Zero,
	7830	nullptr, di_element_types.items, di_element_types.length, 0, nullptr, "");
	7831
	7832	ZigLLVMReplaceTemporary(g->dbuilder, frame_header_di_type, replacement_di_type);
	7833	}
	7834
7197	7835	static void resolve_llvm_types(CodeGen g, ZigType type, ResolveStatus wanted_resolve_status) {
7198	7836	assert(type->id == ZigTypeIdOpaque \|\| type_is_resolved(type, ResolveStatusSizeKnown));
7199	7837	assert(wanted_resolve_status > ResolveStatusSizeKnown);
...	...	@@ -7219,20 +7857,13 @@ static void resolve_llvm_types(CodeGen g, ZigType type, ResolveStatus wanted_r
7219	7857	if (type->data.structure.is_slice)
7220	7858	return resolve_llvm_types_slice(g, type, wanted_resolve_status);
7221	7859	else
7222		return resolve_llvm_types_struct(g, type, wanted_resolve_status);
	7860	return resolve_llvm_types_struct(g, type, wanted_resolve_status, nullptr);
7223	7861	case ZigTypeIdEnum:
7224	7862	return resolve_llvm_types_enum(g, type);
7225	7863	case ZigTypeIdUnion:
7226	7864	return resolve_llvm_types_union(g, type, wanted_resolve_status);
7227	7865	case ZigTypeIdPointer:
7228		return resolve_llvm_types_pointer(g, type);
7229		case ZigTypeIdPromise: {
7230		if (type->llvm_di_type != nullptr) return;
7231		ZigType *u8_ptr_type = get_pointer_to_type(g, g->builtin_types.entry_u8, false);
7232		type->llvm_type = get_llvm_type(g, u8_ptr_type);
7233		type->llvm_di_type = get_llvm_di_type(g, u8_ptr_type);
7234		return;
7235		}
	7866	return resolve_llvm_types_pointer(g, type, wanted_resolve_status);
7236	7867	case ZigTypeIdInt:
7237	7868	return resolve_llvm_types_integer(g, type);
7238	7869	case ZigTypeIdOptional:
...	...	@@ -7242,7 +7873,7 @@ static void resolve_llvm_types(CodeGen g, ZigType type, ResolveStatus wanted_r
7242	7873	case ZigTypeIdArray:
7243	7874	return resolve_llvm_types_array(g, type);
7244	7875	case ZigTypeIdFn:
7245		return resolve_llvm_types_fn(g, type);
	7876	return resolve_llvm_types_fn_type(g, type);
7246	7877	case ZigTypeIdErrorSet: {
7247	7878	if (type->llvm_di_type != nullptr) return;
7248	7879
...	...	@@ -7261,14 +7892,18 @@ static void resolve_llvm_types(CodeGen g, ZigType type, ResolveStatus wanted_r
7261	7892	type->abi_align, get_llvm_di_type(g, type->data.vector.elem_type), type->data.vector.len);
7262	7893	return;
7263	7894	}
	7895	case ZigTypeIdFnFrame:
	7896	return resolve_llvm_types_async_frame(g, type, wanted_resolve_status);
	7897	case ZigTypeIdAnyFrame:
	7898	return resolve_llvm_types_any_frame(g, type, wanted_resolve_status);
7264	7899	}
7265	7900	zig_unreachable();
7266	7901	}
7267	7902
7268	7903	LLVMTypeRef get_llvm_type(CodeGen g, ZigType type) {
7269	7904	assertNoError(type_resolve(g, type, ResolveStatusLLVMFull));
7270		assert(type->abi_size == 0 \|\| type->abi_size == LLVMABISizeOfType(g->target_data_ref, type->llvm_type));
7271		assert(type->abi_align == 0 \|\| type->abi_align == LLVMABIAlignmentOfType(g->target_data_ref, type->llvm_type));
	7905	assert(type->abi_size == 0 \|\| type->abi_size >= LLVMABISizeOfType(g->target_data_ref, type->llvm_type));
	7906	assert(type->abi_align == 0 \|\| type->abi_align >= LLVMABIAlignmentOfType(g->target_data_ref, type->llvm_type));
7272	7907	return type->llvm_type;
7273	7908	}
7274	7909

src/analyze.hpp+8-10

...	...	@@ -11,11 +11,12 @@
11	11	#include "all_types.hpp"
12	12
13	13	void semantic_analyze(CodeGen *g);
14		ErrorMsg add_node_error(CodeGen g, AstNode node, Buf msg);
	14	ErrorMsg add_node_error(CodeGen g, const AstNode node, Buf msg);
15	15	ErrorMsg add_token_error(CodeGen g, ZigType owner, Token token, Buf *msg);
16		ErrorMsg add_error_note(CodeGen g, ErrorMsg parent_msg, AstNode node, Buf *msg);
	16	ErrorMsg add_error_note(CodeGen g, ErrorMsg parent_msg, const AstNode node, Buf *msg);
17	17	void emit_error_notes_for_ref_stack(CodeGen g, ErrorMsg msg);
18	18	ZigType *new_type_table_entry(ZigTypeId id);
	19	ZigType get_fn_frame_type(CodeGen g, ZigFn *fn);
19	20	ZigType get_pointer_to_type(CodeGen g, ZigType *child_type, bool is_const);
20	21	ZigType get_pointer_to_type_extra(CodeGen g, ZigType *child_type, bool is_const,
21	22	bool is_volatile, PtrLen ptr_len,
...	...	@@ -37,11 +38,8 @@ ZigType get_smallest_unsigned_int_type(CodeGen g, uint64_t x);
37	38	ZigType get_error_union_type(CodeGen g, ZigType err_set_type, ZigType payload_type);
38	39	ZigType get_bound_fn_type(CodeGen g, ZigFn *fn_entry);
39	40	ZigType get_opaque_type(CodeGen g, Scope scope, AstNode source_node, const char full_name, Buf bare_name);
40		ZigType get_struct_type(CodeGen g, const char type_name, const char field_names[],
41		ZigType *field_types[], size_t field_count);
42		ZigType get_promise_type(CodeGen g, ZigType *result_type);
43		ZigType get_promise_frame_type(CodeGen g, ZigType *return_type);
44	41	ZigType get_test_fn_type(CodeGen g);
	42	ZigType get_any_frame_type(CodeGen g, ZigType *result_type);
45	43	bool handle_is_ptr(ZigType *type_entry);
46	44
47	45	bool type_has_bits(ZigType *type_entry);
...	...	@@ -106,7 +104,6 @@ void eval_min_max_value(CodeGen g, ZigType type_entry, ConstExprValue *const_v
106	104	void eval_min_max_value_int(CodeGen g, ZigType int_type, BigInt *bigint, bool is_max);
107	105
108	106	void render_const_value(CodeGen g, Buf buf, ConstExprValue *const_val);
109		void analyze_fn_ir(CodeGen g, ZigFn fn_table_entry, AstNode *return_type_node);
110	107
111	108	ScopeBlock create_block_scope(CodeGen g, AstNode node, Scope parent);
112	109	ScopeDefer create_defer_scope(CodeGen g, AstNode node, Scope parent);
...	...	@@ -117,7 +114,6 @@ ScopeLoop create_loop_scope(CodeGen g, AstNode node, Scope parent);
117	114	ScopeSuspend create_suspend_scope(CodeGen g, AstNode node, Scope parent);
118	115	ScopeFnDef create_fndef_scope(CodeGen g, AstNode node, Scope parent, ZigFn *fn_entry);
119	116	Scope create_comptime_scope(CodeGen g, AstNode node, Scope parent);
120		Scope create_coro_prelude_scope(CodeGen g, AstNode node, Scope parent);
121	117	Scope create_runtime_scope(CodeGen g, AstNode node, Scope parent, IrInstruction *is_comptime);
122	118
123	119	void init_const_str_lit(CodeGen g, ConstExprValue const_val, Buf *str);
...	...	@@ -199,12 +195,11 @@ void add_var_export(CodeGen g, ZigVar fn_table_entry, Buf *symbol_name, Global
199	195
200	196
201	197	ConstExprValue get_builtin_value(CodeGen codegen, const char *name);
202		ZigType get_ptr_to_stack_trace_type(CodeGen g);
	198	ZigType get_stack_trace_type(CodeGen g);
203	199	bool resolve_inferred_error_set(CodeGen g, ZigType err_set_type, AstNode *source_node);
204	200
205	201	ZigType get_auto_err_set_type(CodeGen g, ZigFn *fn_entry);
206	202
207		uint32_t get_coro_frame_align_bytes(CodeGen *g);
208	203	bool fn_type_can_fail(FnTypeId *fn_type_id);
209	204	bool type_can_fail(ZigType *type_entry);
210	205	bool fn_eval_cacheable(Scope scope, ZigType return_type);
...	...	@@ -251,4 +246,7 @@ void src_assert(bool ok, AstNode *source_node);
251	246	bool is_container(ZigType *type_entry);
252	247	ConstExprValue analyze_const_value(CodeGen g, Scope scope, AstNode node, ZigType type_entry, Buf type_name);
253	248
	249	void resolve_llvm_types_fn(CodeGen g, ZigFn fn);
	250	bool fn_is_async(ZigFn *fn);
	251
254	252	#endif

src/ast_render.cpp+19-26

...	...	@@ -249,18 +249,16 @@ static const char *node_type_str(NodeType node_type) {
249	249	return "IfOptional";
250	250	case NodeTypeErrorSetDecl:
251	251	return "ErrorSetDecl";
252		case NodeTypeCancel:
253		return "Cancel";
254	252	case NodeTypeResume:
255	253	return "Resume";
256	254	case NodeTypeAwaitExpr:
257	255	return "AwaitExpr";
258	256	case NodeTypeSuspend:
259	257	return "Suspend";
260		case NodeTypePromiseType:
261		return "PromiseType";
262	258	case NodeTypePointerType:
263	259	return "PointerType";
	260	case NodeTypeAnyFrameType:
	261	return "AnyFrameType";
264	262	case NodeTypeEnumLiteral:
265	263	return "EnumLiteral";
266	264	}
...	...	@@ -699,13 +697,7 @@ static void render_node_extra(AstRender ar, AstNode node, bool grouped) {
699	697	fprintf(ar->f, "@");
700	698	}
701	699	if (node->data.fn_call_expr.is_async) {
702		fprintf(ar->f, "async");
703		if (node->data.fn_call_expr.async_allocator != nullptr) {
704		fprintf(ar->f, "<");
705		render_node_extra(ar, node->data.fn_call_expr.async_allocator, true);
706		fprintf(ar->f, ">");
707		}
708		fprintf(ar->f, " ");
	700	fprintf(ar->f, "async ");
709	701	}
710	702	AstNode *fn_ref_node = node->data.fn_call_expr.fn_ref_expr;
711	703	bool grouped = (fn_ref_node->type != NodeTypePrefixOpExpr && fn_ref_node->type != NodeTypePointerType);
...	...	@@ -862,15 +854,14 @@ static void render_node_extra(AstRender ar, AstNode node, bool grouped) {
862	854	render_node_ungrouped(ar, node->data.inferred_array_type.child_type);
863	855	break;
864	856	}
865		case NodeTypePromiseType:
866		{
867		fprintf(ar->f, "promise");
868		if (node->data.promise_type.payload_type != nullptr) {
869		fprintf(ar->f, "->");
870		render_node_grouped(ar, node->data.promise_type.payload_type);
871		}
872		break;
	857	case NodeTypeAnyFrameType: {
	858	fprintf(ar->f, "anyframe");
	859	if (node->data.anyframe_type.payload_type != nullptr) {
	860	fprintf(ar->f, "->");
	861	render_node_grouped(ar, node->data.anyframe_type.payload_type);
873	862	}
	863	break;
	864	}
874	865	case NodeTypeErrorType:
875	866	fprintf(ar->f, "anyerror");
876	867	break;
...	...	@@ -1143,12 +1134,6 @@ static void render_node_extra(AstRender ar, AstNode node, bool grouped) {
1143	1134	fprintf(ar->f, "}");
1144	1135	break;
1145	1136	}
1146		case NodeTypeCancel:
1147		{
1148		fprintf(ar->f, "cancel ");
1149		render_node_grouped(ar, node->data.cancel_expr.expr);
1150		break;
1151		}
1152	1137	case NodeTypeResume:
1153	1138	{
1154	1139	fprintf(ar->f, "resume ");
...	...	@@ -1163,9 +1148,11 @@ static void render_node_extra(AstRender ar, AstNode node, bool grouped) {
1163	1148	}
1164	1149	case NodeTypeSuspend:
1165	1150	{
1166		fprintf(ar->f, "suspend");
1167	1151	if (node->data.suspend.block != nullptr) {
	1152	fprintf(ar->f, "suspend ");
1168	1153	render_node_grouped(ar, node->data.suspend.block);
	1154	} else {
	1155	fprintf(ar->f, "suspend\n");
1169	1156	}
1170	1157	break;
1171	1158	}
...	...	@@ -1191,3 +1178,9 @@ void ast_render(FILE f, AstNode node, int indent_size) {
1191	1178
1192	1179	render_node_grouped(&ar, node);
1193	1180	}
	1181
	1182	void AstNode::src() {
	1183	fprintf(stderr, "%s:%" ZIG_PRI_usize ":%" ZIG_PRI_usize "\n",
	1184	buf_ptr(this->owner->data.structure.root_struct->path),
	1185	this->line + 1, this->column + 1);
	1186	}

src/codegen.cpp+1232-920

...	...	@@ -24,6 +24,12 @@
24	24	#include <stdio.h>
25	25	#include <errno.h>
26	26
	27	enum ResumeId {
	28	ResumeIdManual,
	29	ResumeIdReturn,
	30	ResumeIdCall,
	31	};
	32
27	33	static void init_darwin_native(CodeGen *g) {
28	34	char *osx_target = getenv("MACOSX_DEPLOYMENT_TARGET");
29	35	char *ios_target = getenv("IPHONEOS_DEPLOYMENT_TARGET");
...	...	@@ -297,12 +303,42 @@ static LLVMLinkage to_llvm_linkage(GlobalLinkageId id) {
297	303	zig_unreachable();
298	304	}
299	305
	306	// label (grep this): [fn_frame_struct_layout]
	307	static uint32_t frame_index_trace_arg(CodeGen g, ZigType return_type) {
	308	// [0] *ReturnType (callee's)
	309	// [1] *ReturnType (awaiter's)
	310	// [2] ReturnType
	311	uint32_t return_field_count = type_has_bits(return_type) ? 3 : 0;
	312	return frame_ret_start + return_field_count;
	313	}
	314
	315	// label (grep this): [fn_frame_struct_layout]
	316	static uint32_t frame_index_arg(CodeGen g, ZigType return_type) {
	317	bool have_stack_trace = codegen_fn_has_err_ret_tracing_arg(g, return_type);
	318	// [0] *StackTrace (callee's)
	319	// [1] *StackTrace (awaiter's)
	320	uint32_t trace_field_count = have_stack_trace ? 2 : 0;
	321	return frame_index_trace_arg(g, return_type) + trace_field_count;
	322	}
	323
	324	// label (grep this): [fn_frame_struct_layout]
	325	static uint32_t frame_index_trace_stack(CodeGen g, FnTypeId fn_type_id) {
	326	uint32_t result = frame_index_arg(g, fn_type_id->return_type);
	327	for (size_t i = 0; i < fn_type_id->param_count; i += 1) {
	328	if (type_has_bits(fn_type_id->param_info->type)) {
	329	result += 1;
	330	}
	331	}
	332	return result;
	333	}
	334
	335
300	336	static uint32_t get_err_ret_trace_arg_index(CodeGen g, ZigFn fn_table_entry) {
301	337	if (!g->have_err_ret_tracing) {
302	338	return UINT32_MAX;
303	339	}
304		if (fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync) {
305		return 0;
	340	if (fn_is_async(fn_table_entry)) {
	341	return UINT32_MAX;
306	342	}
307	343	ZigType *fn_type = fn_table_entry->type_entry;
308	344	if (!fn_type_can_fail(&fn_type->data.fn.fn_type_id)) {
...	...	@@ -343,27 +379,28 @@ static bool cc_want_sret_attr(CallingConvention cc) {
343	379	zig_unreachable();
344	380	}
345	381
346		static LLVMValueRef fn_llvm_value(CodeGen g, ZigFn fn_table_entry) {
347		if (fn_table_entry->llvm_value)
348		return fn_table_entry->llvm_value;
	382	static bool codegen_have_frame_pointer(CodeGen *g) {
	383	return g->build_mode == BuildModeDebug;
	384	}
349	385
350		Buf *unmangled_name = &fn_table_entry->symbol_name;
	386	static LLVMValueRef make_fn_llvm_value(CodeGen g, ZigFn fn) {
	387	Buf *unmangled_name = &fn->symbol_name;
351	388	Buf *symbol_name;
352	389	GlobalLinkageId linkage;
353		if (fn_table_entry->body_node == nullptr) {
	390	if (fn->body_node == nullptr) {
354	391	symbol_name = unmangled_name;
355	392	linkage = GlobalLinkageIdStrong;
356		} else if (fn_table_entry->export_list.length == 0) {
	393	} else if (fn->export_list.length == 0) {
357	394	symbol_name = get_mangled_name(g, unmangled_name, false);
358	395	linkage = GlobalLinkageIdInternal;
359	396	} else {
360		GlobalExport *fn_export = &fn_table_entry->export_list.items[0];
	397	GlobalExport *fn_export = &fn->export_list.items[0];
361	398	symbol_name = &fn_export->name;
362	399	linkage = fn_export->linkage;
363	400	}
364	401
365	402	bool external_linkage = linkage != GlobalLinkageIdInternal;
366		CallingConvention cc = fn_table_entry->type_entry->data.fn.fn_type_id.cc;
	403	CallingConvention cc = fn->type_entry->data.fn.fn_type_id.cc;
367	404	if (cc == CallingConventionStdcall && external_linkage &&
368	405	g->zig_target->arch == ZigLLVM_x86)
369	406	{
...	...	@@ -371,130 +408,125 @@ static LLVMValueRef fn_llvm_value(CodeGen g, ZigFn fn_table_entry) {
371	408	symbol_name = buf_sprintf("\x01_%s", buf_ptr(symbol_name));
372	409	}
373	410
	411	bool is_async = fn_is_async(fn);
374	412
375		ZigType *fn_type = fn_table_entry->type_entry;
	413
	414	ZigType *fn_type = fn->type_entry;
376	415	// Make the raw_type_ref populated
377		(void)get_llvm_type(g, fn_type);
378		LLVMTypeRef fn_llvm_type = fn_type->data.fn.raw_type_ref;
379		if (fn_table_entry->body_node == nullptr) {
	416	resolve_llvm_types_fn(g, fn);
	417	LLVMTypeRef fn_llvm_type = fn->raw_type_ref;
	418	LLVMValueRef llvm_fn = nullptr;
	419	if (fn->body_node == nullptr) {
380	420	LLVMValueRef existing_llvm_fn = LLVMGetNamedFunction(g->module, buf_ptr(symbol_name));
381	421	if (existing_llvm_fn) {
382		fn_table_entry->llvm_value = LLVMConstBitCast(existing_llvm_fn, LLVMPointerType(fn_llvm_type, 0));
383		return fn_table_entry->llvm_value;
	422	return LLVMConstBitCast(existing_llvm_fn, LLVMPointerType(fn_llvm_type, 0));
384	423	} else {
385	424	auto entry = g->exported_symbol_names.maybe_get(symbol_name);
386	425	if (entry == nullptr) {
387		fn_table_entry->llvm_value = LLVMAddFunction(g->module, buf_ptr(symbol_name), fn_llvm_type);
	426	llvm_fn = LLVMAddFunction(g->module, buf_ptr(symbol_name), fn_llvm_type);
388	427
389	428	if (target_is_wasm(g->zig_target)) {
390		assert(fn_table_entry->proto_node->type == NodeTypeFnProto);
391		AstNodeFnProto *fn_proto = &fn_table_entry->proto_node->data.fn_proto;
	429	assert(fn->proto_node->type == NodeTypeFnProto);
	430	AstNodeFnProto *fn_proto = &fn->proto_node->data.fn_proto;
392	431	if (fn_proto-> is_extern && fn_proto->lib_name != nullptr ) {
393		addLLVMFnAttrStr(fn_table_entry->llvm_value, "wasm-import-module", buf_ptr(fn_proto->lib_name));
	432	addLLVMFnAttrStr(llvm_fn, "wasm-import-module", buf_ptr(fn_proto->lib_name));
394	433	}
395	434	}
396	435	} else {
397	436	assert(entry->value->id == TldIdFn);
398	437	TldFn tld_fn = reinterpret_cast<TldFn >(entry->value);
399	438	// Make the raw_type_ref populated
400		(void)get_llvm_type(g, tld_fn->fn_entry->type_entry);
	439	resolve_llvm_types_fn(g, tld_fn->fn_entry);
401	440	tld_fn->fn_entry->llvm_value = LLVMAddFunction(g->module, buf_ptr(symbol_name),
402		tld_fn->fn_entry->type_entry->data.fn.raw_type_ref);
403		fn_table_entry->llvm_value = LLVMConstBitCast(tld_fn->fn_entry->llvm_value,
404		LLVMPointerType(fn_llvm_type, 0));
405		return fn_table_entry->llvm_value;
	441	tld_fn->fn_entry->raw_type_ref);
	442	llvm_fn = LLVMConstBitCast(tld_fn->fn_entry->llvm_value, LLVMPointerType(fn_llvm_type, 0));
	443	return llvm_fn;
406	444	}
407	445	}
408	446	} else {
409		if (fn_table_entry->llvm_value == nullptr) {
410		fn_table_entry->llvm_value = LLVMAddFunction(g->module, buf_ptr(symbol_name), fn_llvm_type);
	447	if (llvm_fn == nullptr) {
	448	llvm_fn = LLVMAddFunction(g->module, buf_ptr(symbol_name), fn_llvm_type);
411	449	}
412	450
413		for (size_t i = 1; i < fn_table_entry->export_list.length; i += 1) {
414		GlobalExport *fn_export = &fn_table_entry->export_list.items[i];
415		LLVMAddAlias(g->module, LLVMTypeOf(fn_table_entry->llvm_value),
416		fn_table_entry->llvm_value, buf_ptr(&fn_export->name));
	451	for (size_t i = 1; i < fn->export_list.length; i += 1) {
	452	GlobalExport *fn_export = &fn->export_list.items[i];
	453	LLVMAddAlias(g->module, LLVMTypeOf(llvm_fn), llvm_fn, buf_ptr(&fn_export->name));
417	454	}
418	455	}
419		fn_table_entry->llvm_name = strdup(LLVMGetValueName(fn_table_entry->llvm_value));
420	456
421		switch (fn_table_entry->fn_inline) {
	457	switch (fn->fn_inline) {
422	458	case FnInlineAlways:
423		addLLVMFnAttr(fn_table_entry->llvm_value, "alwaysinline");
424		g->inline_fns.append(fn_table_entry);
	459	addLLVMFnAttr(llvm_fn, "alwaysinline");
	460	g->inline_fns.append(fn);
425	461	break;
426	462	case FnInlineNever:
427		addLLVMFnAttr(fn_table_entry->llvm_value, "noinline");
	463	addLLVMFnAttr(llvm_fn, "noinline");
428	464	break;
429	465	case FnInlineAuto:
430		if (fn_table_entry->alignstack_value != 0) {
431		addLLVMFnAttr(fn_table_entry->llvm_value, "noinline");
	466	if (fn->alignstack_value != 0) {
	467	addLLVMFnAttr(llvm_fn, "noinline");
432	468	}
433	469	break;
434	470	}
435	471
436	472	if (cc == CallingConventionNaked) {
437		addLLVMFnAttr(fn_table_entry->llvm_value, "naked");
	473	addLLVMFnAttr(llvm_fn, "naked");
438	474	} else {
439		LLVMSetFunctionCallConv(fn_table_entry->llvm_value, get_llvm_cc(g, fn_type->data.fn.fn_type_id.cc));
440		}
441		if (cc == CallingConventionAsync) {
442		addLLVMFnAttr(fn_table_entry->llvm_value, "optnone");
443		addLLVMFnAttr(fn_table_entry->llvm_value, "noinline");
	475	LLVMSetFunctionCallConv(llvm_fn, get_llvm_cc(g, fn_type->data.fn.fn_type_id.cc));
444	476	}
445	477
446		bool want_cold = fn_table_entry->is_cold \|\| cc == CallingConventionCold;
	478	bool want_cold = fn->is_cold \|\| cc == CallingConventionCold;
447	479	if (want_cold) {
448		ZigLLVMAddFunctionAttrCold(fn_table_entry->llvm_value);
	480	ZigLLVMAddFunctionAttrCold(llvm_fn);
449	481	}
450	482
451	483
452		LLVMSetLinkage(fn_table_entry->llvm_value, to_llvm_linkage(linkage));
	484	LLVMSetLinkage(llvm_fn, to_llvm_linkage(linkage));
453	485
454	486	if (linkage == GlobalLinkageIdInternal) {
455		LLVMSetUnnamedAddr(fn_table_entry->llvm_value, true);
	487	LLVMSetUnnamedAddr(llvm_fn, true);
456	488	}
457	489
458	490	ZigType *return_type = fn_type->data.fn.fn_type_id.return_type;
459	491	if (return_type->id == ZigTypeIdUnreachable) {
460		addLLVMFnAttr(fn_table_entry->llvm_value, "noreturn");
	492	addLLVMFnAttr(llvm_fn, "noreturn");
461	493	}
462	494
463		if (fn_table_entry->body_node != nullptr) {
464		maybe_export_dll(g, fn_table_entry->llvm_value, linkage);
	495	if (fn->body_node != nullptr) {
	496	maybe_export_dll(g, llvm_fn, linkage);
465	497
466	498	bool want_fn_safety = g->build_mode != BuildModeFastRelease &&
467	499	g->build_mode != BuildModeSmallRelease &&
468		!fn_table_entry->def_scope->safety_off;
	500	!fn->def_scope->safety_off;
469	501	if (want_fn_safety) {
470	502	if (g->libc_link_lib != nullptr) {
471		addLLVMFnAttr(fn_table_entry->llvm_value, "sspstrong");
472		addLLVMFnAttrStr(fn_table_entry->llvm_value, "stack-protector-buffer-size", "4");
	503	addLLVMFnAttr(llvm_fn, "sspstrong");
	504	addLLVMFnAttrStr(llvm_fn, "stack-protector-buffer-size", "4");
473	505	}
474	506	}
475		if (g->have_stack_probing && !fn_table_entry->def_scope->safety_off) {
476		addLLVMFnAttrStr(fn_table_entry->llvm_value, "probe-stack", "__zig_probe_stack");
	507	if (g->have_stack_probing && !fn->def_scope->safety_off) {
	508	addLLVMFnAttrStr(llvm_fn, "probe-stack", "__zig_probe_stack");
477	509	}
478	510	} else {
479		maybe_import_dll(g, fn_table_entry->llvm_value, linkage);
	511	maybe_import_dll(g, llvm_fn, linkage);
480	512	}
481	513
482		if (fn_table_entry->alignstack_value != 0) {
483		addLLVMFnAttrInt(fn_table_entry->llvm_value, "alignstack", fn_table_entry->alignstack_value);
	514	if (fn->alignstack_value != 0) {
	515	addLLVMFnAttrInt(llvm_fn, "alignstack", fn->alignstack_value);
484	516	}
485	517
486		addLLVMFnAttr(fn_table_entry->llvm_value, "nounwind");
487		add_uwtable_attr(g, fn_table_entry->llvm_value);
488		addLLVMFnAttr(fn_table_entry->llvm_value, "nobuiltin");
489		if (g->build_mode == BuildModeDebug && fn_table_entry->fn_inline != FnInlineAlways) {
490		ZigLLVMAddFunctionAttr(fn_table_entry->llvm_value, "no-frame-pointer-elim", "true");
491		ZigLLVMAddFunctionAttr(fn_table_entry->llvm_value, "no-frame-pointer-elim-non-leaf", nullptr);
	518	addLLVMFnAttr(llvm_fn, "nounwind");
	519	add_uwtable_attr(g, llvm_fn);
	520	addLLVMFnAttr(llvm_fn, "nobuiltin");
	521	if (codegen_have_frame_pointer(g) && fn->fn_inline != FnInlineAlways) {
	522	ZigLLVMAddFunctionAttr(llvm_fn, "no-frame-pointer-elim", "true");
	523	ZigLLVMAddFunctionAttr(llvm_fn, "no-frame-pointer-elim-non-leaf", nullptr);
492	524	}
493		if (fn_table_entry->section_name) {
494		LLVMSetSection(fn_table_entry->llvm_value, buf_ptr(fn_table_entry->section_name));
	525	if (fn->section_name) {
	526	LLVMSetSection(llvm_fn, buf_ptr(fn->section_name));
495	527	}
496		if (fn_table_entry->align_bytes > 0) {
497		LLVMSetAlignment(fn_table_entry->llvm_value, (unsigned)fn_table_entry->align_bytes);
	528	if (fn->align_bytes > 0) {
	529	LLVMSetAlignment(llvm_fn, (unsigned)fn->align_bytes);
498	530	} else {
499	531	// We'd like to set the best alignment for the function here, but on Darwin LLVM gives
500	532	// "Cannot getTypeInfo() on a type that is unsized!" assertion failure when calling
...	...	@@ -502,36 +534,50 @@ static LLVMValueRef fn_llvm_value(CodeGen g, ZigFn fn_table_entry) {
502	534	// use the ABI alignment, which is fine.
503	535	}
504	536
505		unsigned init_gen_i = 0;
506		if (!type_has_bits(return_type)) {
507		// nothing to do
508		} else if (type_is_nonnull_ptr(return_type)) {
509		addLLVMAttr(fn_table_entry->llvm_value, 0, "nonnull");
510		} else if (want_first_arg_sret(g, &fn_type->data.fn.fn_type_id)) {
511		// Sret pointers must not be address 0
512		addLLVMArgAttr(fn_table_entry->llvm_value, 0, "nonnull");
513		addLLVMArgAttr(fn_table_entry->llvm_value, 0, "sret");
514		if (cc_want_sret_attr(cc)) {
515		addLLVMArgAttr(fn_table_entry->llvm_value, 0, "noalias");
	537	if (is_async) {
	538	addLLVMArgAttr(llvm_fn, 0, "nonnull");
	539	} else {
	540	unsigned init_gen_i = 0;
	541	if (!type_has_bits(return_type)) {
	542	// nothing to do
	543	} else if (type_is_nonnull_ptr(return_type)) {
	544	addLLVMAttr(llvm_fn, 0, "nonnull");
	545	} else if (want_first_arg_sret(g, &fn_type->data.fn.fn_type_id)) {
	546	// Sret pointers must not be address 0
	547	addLLVMArgAttr(llvm_fn, 0, "nonnull");
	548	addLLVMArgAttr(llvm_fn, 0, "sret");
	549	if (cc_want_sret_attr(cc)) {
	550	addLLVMArgAttr(llvm_fn, 0, "noalias");
	551	}
	552	init_gen_i = 1;
516	553	}
517		init_gen_i = 1;
518		}
519	554
520		// set parameter attributes
521		FnWalk fn_walk = {};
522		fn_walk.id = FnWalkIdAttrs;
523		fn_walk.data.attrs.fn = fn_table_entry;
524		fn_walk.data.attrs.gen_i = init_gen_i;
525		walk_function_params(g, fn_type, &fn_walk);
	555	// set parameter attributes
	556	FnWalk fn_walk = {};
	557	fn_walk.id = FnWalkIdAttrs;
	558	fn_walk.data.attrs.fn = fn;
	559	fn_walk.data.attrs.llvm_fn = llvm_fn;
	560	fn_walk.data.attrs.gen_i = init_gen_i;
	561	walk_function_params(g, fn_type, &fn_walk);
526	562
527		uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn_table_entry);
528		if (err_ret_trace_arg_index != UINT32_MAX) {
529		// Error return trace memory is in the stack, which is impossible to be at address 0
530		// on any architecture.
531		addLLVMArgAttr(fn_table_entry->llvm_value, (unsigned)err_ret_trace_arg_index, "nonnull");
	563	uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn);
	564	if (err_ret_trace_arg_index != UINT32_MAX) {
	565	// Error return trace memory is in the stack, which is impossible to be at address 0
	566	// on any architecture.
	567	addLLVMArgAttr(llvm_fn, (unsigned)err_ret_trace_arg_index, "nonnull");
	568	}
532	569	}
533	570
534		return fn_table_entry->llvm_value;
	571	return llvm_fn;
	572	}
	573
	574	static LLVMValueRef fn_llvm_value(CodeGen g, ZigFn fn) {
	575	if (fn->llvm_value)
	576	return fn->llvm_value;
	577
	578	fn->llvm_value = make_fn_llvm_value(g, fn);
	579	fn->llvm_name = strdup(LLVMGetValueName(fn->llvm_value));
	580	return fn->llvm_value;
535	581	}
536	582
537	583	static ZigLLVMDIScope get_di_scope(CodeGen g, Scope *scope) {
...	...	@@ -559,10 +605,11 @@ static ZigLLVMDIScope get_di_scope(CodeGen g, Scope *scope) {
559	605	unsigned flags = ZigLLVM_DIFlags_StaticMember;
560	606	ZigLLVMDIScope *fn_di_scope = get_di_scope(g, scope->parent);
561	607	assert(fn_di_scope != nullptr);
	608	assert(fn_table_entry->raw_di_type != nullptr);
562	609	ZigLLVMDISubprogram *subprogram = ZigLLVMCreateFunction(g->dbuilder,
563	610	fn_di_scope, buf_ptr(&fn_table_entry->symbol_name), "",
564	611	import->data.structure.root_struct->di_file, line_number,
565		fn_table_entry->type_entry->data.fn.raw_di_type, is_internal_linkage,
	612	fn_table_entry->raw_di_type, is_internal_linkage,
566	613	is_definition, scope_line, flags, is_optimized, nullptr);
567	614
568	615	scope->di_scope = ZigLLVMSubprogramToScope(subprogram);
...	...	@@ -597,7 +644,6 @@ static ZigLLVMDIScope get_di_scope(CodeGen g, Scope *scope) {
597	644	case ScopeIdLoop:
598	645	case ScopeIdSuspend:
599	646	case ScopeIdCompTime:
600		case ScopeIdCoroPrelude:
601	647	case ScopeIdRuntime:
602	648	return get_di_scope(g, scope->parent);
603	649	}
...	...	@@ -798,9 +844,8 @@ static bool ir_want_fast_math(CodeGen g, IrInstruction instruction) {
798	844	return false;
799	845	}
800	846
801		static bool ir_want_runtime_safety(CodeGen g, IrInstruction instruction) {
	847	static bool ir_want_runtime_safety_scope(CodeGen g, Scope scope) {
802	848	// TODO memoize
803		Scope *scope = instruction->scope;
804	849	while (scope) {
805	850	if (scope->id == ScopeIdBlock) {
806	851	ScopeBlock block_scope = (ScopeBlock )scope;
...	...	@@ -818,6 +863,10 @@ static bool ir_want_runtime_safety(CodeGen g, IrInstruction instruction) {
818	863	g->build_mode != BuildModeSmallRelease);
819	864	}
820	865
	866	static bool ir_want_runtime_safety(CodeGen g, IrInstruction instruction) {
	867	return ir_want_runtime_safety_scope(g, instruction->scope);
	868	}
	869
821	870	static Buf *panic_msg_buf(PanicMsgId msg_id) {
822	871	switch (msg_id) {
823	872	case PanicMsgIdCount:
...	...	@@ -858,6 +907,18 @@ static Buf *panic_msg_buf(PanicMsgId msg_id) {
858	907	return buf_create_from_str("integer part of floating point value out of bounds");
859	908	case PanicMsgIdPtrCastNull:
860	909	return buf_create_from_str("cast causes pointer to be null");
	910	case PanicMsgIdBadResume:
	911	return buf_create_from_str("resumed an async function which already returned");
	912	case PanicMsgIdBadAwait:
	913	return buf_create_from_str("async function awaited twice");
	914	case PanicMsgIdBadReturn:
	915	return buf_create_from_str("async function returned twice");
	916	case PanicMsgIdResumedAnAwaitingFn:
	917	return buf_create_from_str("awaiting function resumed");
	918	case PanicMsgIdFrameTooSmall:
	919	return buf_create_from_str("frame too small");
	920	case PanicMsgIdResumedFnPendingAwait:
	921	return buf_create_from_str("resumed an async function which can only be awaited");
861	922	}
862	923	zig_unreachable();
863	924	}
...	...	@@ -882,13 +943,16 @@ static LLVMValueRef get_panic_msg_ptr_val(CodeGen *g, PanicMsgId msg_id) {
882	943	return LLVMConstBitCast(val->global_refs->llvm_global, LLVMPointerType(get_llvm_type(g, str_type), 0));
883	944	}
884	945
	946	static ZigType ptr_to_stack_trace_type(CodeGen g) {
	947	return get_pointer_to_type(g, get_stack_trace_type(g), false);
	948	}
	949
885	950	static void gen_panic(CodeGen *g, LLVMValueRef msg_arg, LLVMValueRef stack_trace_arg) {
886	951	assert(g->panic_fn != nullptr);
887	952	LLVMValueRef fn_val = fn_llvm_value(g, g->panic_fn);
888	953	LLVMCallConv llvm_cc = get_llvm_cc(g, g->panic_fn->type_entry->data.fn.fn_type_id.cc);
889	954	if (stack_trace_arg == nullptr) {
890		ZigType *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g);
891		stack_trace_arg = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type));
	955	stack_trace_arg = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g)));
892	956	}
893	957	LLVMValueRef args[] = {
894	958	msg_arg,
...	...	@@ -904,14 +968,18 @@ static void gen_safety_crash(CodeGen *g, PanicMsgId msg_id) {
904	968	gen_panic(g, get_panic_msg_ptr_val(g, msg_id), nullptr);
905	969	}
906	970
907		static void gen_assertion(CodeGen g, PanicMsgId msg_id, IrInstruction source_instruction) {
908		if (ir_want_runtime_safety(g, source_instruction)) {
	971	static void gen_assertion_scope(CodeGen g, PanicMsgId msg_id, Scope source_scope) {
	972	if (ir_want_runtime_safety_scope(g, source_scope)) {
909	973	gen_safety_crash(g, msg_id);
910	974	} else {
911	975	LLVMBuildUnreachable(g->builder);
912	976	}
913	977	}
914	978
	979	static void gen_assertion(CodeGen g, PanicMsgId msg_id, IrInstruction source_instruction) {
	980	return gen_assertion_scope(g, msg_id, source_instruction->scope);
	981	}
	982
915	983	static LLVMValueRef get_stacksave_fn_val(CodeGen *g) {
916	984	if (g->stacksave_fn_val)
917	985	return g->stacksave_fn_val;
...	...	@@ -959,177 +1027,6 @@ static LLVMValueRef get_write_register_fn_val(CodeGen *g) {
959	1027	return g->write_register_fn_val;
960	1028	}
961	1029
962		static LLVMValueRef get_coro_destroy_fn_val(CodeGen *g) {
963		if (g->coro_destroy_fn_val)
964		return g->coro_destroy_fn_val;
965
966		LLVMTypeRef param_types[] = {
967		LLVMPointerType(LLVMInt8Type(), 0),
968		};
969		LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 1, false);
970		Buf *name = buf_sprintf("llvm.coro.destroy");
971		g->coro_destroy_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
972		assert(LLVMGetIntrinsicID(g->coro_destroy_fn_val));
973
974		return g->coro_destroy_fn_val;
975		}
976
977		static LLVMValueRef get_coro_id_fn_val(CodeGen *g) {
978		if (g->coro_id_fn_val)
979		return g->coro_id_fn_val;
980
981		LLVMTypeRef param_types[] = {
982		LLVMInt32Type(),
983		LLVMPointerType(LLVMInt8Type(), 0),
984		LLVMPointerType(LLVMInt8Type(), 0),
985		LLVMPointerType(LLVMInt8Type(), 0),
986		};
987		LLVMTypeRef fn_type = LLVMFunctionType(ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), param_types, 4, false);
988		Buf *name = buf_sprintf("llvm.coro.id");
989		g->coro_id_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
990		assert(LLVMGetIntrinsicID(g->coro_id_fn_val));
991
992		return g->coro_id_fn_val;
993		}
994
995		static LLVMValueRef get_coro_alloc_fn_val(CodeGen *g) {
996		if (g->coro_alloc_fn_val)
997		return g->coro_alloc_fn_val;
998
999		LLVMTypeRef param_types[] = {
1000		ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()),
1001		};
1002		LLVMTypeRef fn_type = LLVMFunctionType(LLVMInt1Type(), param_types, 1, false);
1003		Buf *name = buf_sprintf("llvm.coro.alloc");
1004		g->coro_alloc_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
1005		assert(LLVMGetIntrinsicID(g->coro_alloc_fn_val));
1006
1007		return g->coro_alloc_fn_val;
1008		}
1009
1010		static LLVMValueRef get_coro_size_fn_val(CodeGen *g) {
1011		if (g->coro_size_fn_val)
1012		return g->coro_size_fn_val;
1013
1014		LLVMTypeRef fn_type = LLVMFunctionType(g->builtin_types.entry_usize->llvm_type, nullptr, 0, false);
1015		Buf name = buf_sprintf("llvm.coro.size.i%d", g->pointer_size_bytes 8);
1016		g->coro_size_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
1017		assert(LLVMGetIntrinsicID(g->coro_size_fn_val));
1018
1019		return g->coro_size_fn_val;
1020		}
1021
1022		static LLVMValueRef get_coro_begin_fn_val(CodeGen *g) {
1023		if (g->coro_begin_fn_val)
1024		return g->coro_begin_fn_val;
1025
1026		LLVMTypeRef param_types[] = {
1027		ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()),
1028		LLVMPointerType(LLVMInt8Type(), 0),
1029		};
1030		LLVMTypeRef fn_type = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), param_types, 2, false);
1031		Buf *name = buf_sprintf("llvm.coro.begin");
1032		g->coro_begin_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
1033		assert(LLVMGetIntrinsicID(g->coro_begin_fn_val));
1034
1035		return g->coro_begin_fn_val;
1036		}
1037
1038		static LLVMValueRef get_coro_suspend_fn_val(CodeGen *g) {
1039		if (g->coro_suspend_fn_val)
1040		return g->coro_suspend_fn_val;
1041
1042		LLVMTypeRef param_types[] = {
1043		ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()),
1044		LLVMInt1Type(),
1045		};
1046		LLVMTypeRef fn_type = LLVMFunctionType(LLVMInt8Type(), param_types, 2, false);
1047		Buf *name = buf_sprintf("llvm.coro.suspend");
1048		g->coro_suspend_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
1049		assert(LLVMGetIntrinsicID(g->coro_suspend_fn_val));
1050
1051		return g->coro_suspend_fn_val;
1052		}
1053
1054		static LLVMValueRef get_coro_end_fn_val(CodeGen *g) {
1055		if (g->coro_end_fn_val)
1056		return g->coro_end_fn_val;
1057
1058		LLVMTypeRef param_types[] = {
1059		LLVMPointerType(LLVMInt8Type(), 0),
1060		LLVMInt1Type(),
1061		};
1062		LLVMTypeRef fn_type = LLVMFunctionType(LLVMInt1Type(), param_types, 2, false);
1063		Buf *name = buf_sprintf("llvm.coro.end");
1064		g->coro_end_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
1065		assert(LLVMGetIntrinsicID(g->coro_end_fn_val));
1066
1067		return g->coro_end_fn_val;
1068		}
1069
1070		static LLVMValueRef get_coro_free_fn_val(CodeGen *g) {
1071		if (g->coro_free_fn_val)
1072		return g->coro_free_fn_val;
1073
1074		LLVMTypeRef param_types[] = {
1075		ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()),
1076		LLVMPointerType(LLVMInt8Type(), 0),
1077		};
1078		LLVMTypeRef fn_type = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), param_types, 2, false);
1079		Buf *name = buf_sprintf("llvm.coro.free");
1080		g->coro_free_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
1081		assert(LLVMGetIntrinsicID(g->coro_free_fn_val));
1082
1083		return g->coro_free_fn_val;
1084		}
1085
1086		static LLVMValueRef get_coro_resume_fn_val(CodeGen *g) {
1087		if (g->coro_resume_fn_val)
1088		return g->coro_resume_fn_val;
1089
1090		LLVMTypeRef param_types[] = {
1091		LLVMPointerType(LLVMInt8Type(), 0),
1092		};
1093		LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 1, false);
1094		Buf *name = buf_sprintf("llvm.coro.resume");
1095		g->coro_resume_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
1096		assert(LLVMGetIntrinsicID(g->coro_resume_fn_val));
1097
1098		return g->coro_resume_fn_val;
1099		}
1100
1101		static LLVMValueRef get_coro_save_fn_val(CodeGen *g) {
1102		if (g->coro_save_fn_val)
1103		return g->coro_save_fn_val;
1104
1105		LLVMTypeRef param_types[] = {
1106		LLVMPointerType(LLVMInt8Type(), 0),
1107		};
1108		LLVMTypeRef fn_type = LLVMFunctionType(ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), param_types, 1, false);
1109		Buf *name = buf_sprintf("llvm.coro.save");
1110		g->coro_save_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
1111		assert(LLVMGetIntrinsicID(g->coro_save_fn_val));
1112
1113		return g->coro_save_fn_val;
1114		}
1115
1116		static LLVMValueRef get_coro_promise_fn_val(CodeGen *g) {
1117		if (g->coro_promise_fn_val)
1118		return g->coro_promise_fn_val;
1119
1120		LLVMTypeRef param_types[] = {
1121		LLVMPointerType(LLVMInt8Type(), 0),
1122		LLVMInt32Type(),
1123		LLVMInt1Type(),
1124		};
1125		LLVMTypeRef fn_type = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), param_types, 3, false);
1126		Buf *name = buf_sprintf("llvm.coro.promise");
1127		g->coro_promise_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
1128		assert(LLVMGetIntrinsicID(g->coro_promise_fn_val));
1129
1130		return g->coro_promise_fn_val;
1131		}
1132
1133	1030	static LLVMValueRef get_return_address_fn_val(CodeGen *g) {
1134	1031	if (g->return_address_fn_val)
1135	1032	return g->return_address_fn_val;
...	...	@@ -1149,7 +1046,7 @@ static LLVMValueRef get_add_error_return_trace_addr_fn(CodeGen *g) {
1149	1046	return g->add_error_return_trace_addr_fn_val;
1150	1047
1151	1048	LLVMTypeRef arg_types[] = {
1152		get_llvm_type(g, get_ptr_to_stack_trace_type(g)),
	1049	get_llvm_type(g, ptr_to_stack_trace_type(g)),
1153	1050	g->builtin_types.entry_usize->llvm_type,
1154	1051	};
1155	1052	LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 2, false);
...	...	@@ -1164,7 +1061,7 @@ static LLVMValueRef get_add_error_return_trace_addr_fn(CodeGen *g) {
1164	1061	// Error return trace memory is in the stack, which is impossible to be at address 0
1165	1062	// on any architecture.
1166	1063	addLLVMArgAttr(fn_val, (unsigned)0, "nonnull");
1167		if (g->build_mode == BuildModeDebug) {
	1064	if (codegen_have_frame_pointer(g)) {
1168	1065	ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
1169	1066	ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
1170	1067	}
...	...	@@ -1222,140 +1119,6 @@ static LLVMValueRef get_add_error_return_trace_addr_fn(CodeGen *g) {
1222	1119	return fn_val;
1223	1120	}
1224	1121
1225		static LLVMValueRef get_merge_err_ret_traces_fn_val(CodeGen *g) {
1226		if (g->merge_err_ret_traces_fn_val)
1227		return g->merge_err_ret_traces_fn_val;
1228
1229		assert(g->stack_trace_type != nullptr);
1230
1231		LLVMTypeRef param_types[] = {
1232		get_llvm_type(g, get_ptr_to_stack_trace_type(g)),
1233		get_llvm_type(g, get_ptr_to_stack_trace_type(g)),
1234		};
1235		LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), param_types, 2, false);
1236
1237		Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_merge_error_return_traces"), false);
1238		LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref);
1239		LLVMSetLinkage(fn_val, LLVMInternalLinkage);
1240		LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
1241		addLLVMFnAttr(fn_val, "nounwind");
1242		add_uwtable_attr(g, fn_val);
1243		// Error return trace memory is in the stack, which is impossible to be at address 0
1244		// on any architecture.
1245		addLLVMArgAttr(fn_val, (unsigned)0, "nonnull");
1246		addLLVMArgAttr(fn_val, (unsigned)0, "noalias");
1247		addLLVMArgAttr(fn_val, (unsigned)0, "writeonly");
1248		// Error return trace memory is in the stack, which is impossible to be at address 0
1249		// on any architecture.
1250		addLLVMArgAttr(fn_val, (unsigned)1, "nonnull");
1251		addLLVMArgAttr(fn_val, (unsigned)1, "noalias");
1252		addLLVMArgAttr(fn_val, (unsigned)1, "readonly");
1253		if (g->build_mode == BuildModeDebug) {
1254		ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
1255		ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
1256		}
1257
1258		// this is above the ZigLLVMClearCurrentDebugLocation
1259		LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g);
1260
1261		LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
1262		LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
1263		LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
1264		LLVMPositionBuilderAtEnd(g->builder, entry_block);
1265		ZigLLVMClearCurrentDebugLocation(g->builder);
1266
1267		// var frame_index: usize = undefined;
1268		// var frames_left: usize = undefined;
1269		// if (src_stack_trace.index < src_stack_trace.instruction_addresses.len) {
1270		// frame_index = 0;
1271		// frames_left = src_stack_trace.index;
1272		// if (frames_left == 0) return;
1273		// } else {
1274		// frame_index = (src_stack_trace.index + 1) % src_stack_trace.instruction_addresses.len;
1275		// frames_left = src_stack_trace.instruction_addresses.len;
1276		// }
1277		// while (true) {
1278		// __zig_add_err_ret_trace_addr(dest_stack_trace, src_stack_trace.instruction_addresses[frame_index]);
1279		// frames_left -= 1;
1280		// if (frames_left == 0) return;
1281		// frame_index = (frame_index + 1) % src_stack_trace.instruction_addresses.len;
1282		// }
1283		LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return");
1284
1285		LLVMValueRef frame_index_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frame_index");
1286		LLVMValueRef frames_left_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frames_left");
1287
1288		LLVMValueRef dest_stack_trace_ptr = LLVMGetParam(fn_val, 0);
1289		LLVMValueRef src_stack_trace_ptr = LLVMGetParam(fn_val, 1);
1290
1291		size_t src_index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index;
1292		size_t src_addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index;
1293		LLVMValueRef src_index_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr,
1294		(unsigned)src_index_field_index, "");
1295		LLVMValueRef src_addresses_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr,
1296		(unsigned)src_addresses_field_index, "");
1297		ZigType *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry;
1298		size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index;
1299		LLVMValueRef src_ptr_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)ptr_field_index, "");
1300		size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index;
1301		LLVMValueRef src_len_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)len_field_index, "");
1302		LLVMValueRef src_index_val = LLVMBuildLoad(g->builder, src_index_field_ptr, "");
1303		LLVMValueRef src_ptr_val = LLVMBuildLoad(g->builder, src_ptr_field_ptr, "");
1304		LLVMValueRef src_len_val = LLVMBuildLoad(g->builder, src_len_field_ptr, "");
1305		LLVMValueRef no_wrap_bit = LLVMBuildICmp(g->builder, LLVMIntULT, src_index_val, src_len_val, "");
1306		LLVMBasicBlockRef no_wrap_block = LLVMAppendBasicBlock(fn_val, "NoWrap");
1307		LLVMBasicBlockRef yes_wrap_block = LLVMAppendBasicBlock(fn_val, "YesWrap");
1308		LLVMBasicBlockRef loop_block = LLVMAppendBasicBlock(fn_val, "Loop");
1309		LLVMBuildCondBr(g->builder, no_wrap_bit, no_wrap_block, yes_wrap_block);
1310
1311		LLVMPositionBuilderAtEnd(g->builder, no_wrap_block);
1312		LLVMValueRef usize_zero = LLVMConstNull(g->builtin_types.entry_usize->llvm_type);
1313		LLVMBuildStore(g->builder, usize_zero, frame_index_ptr);
1314		LLVMBuildStore(g->builder, src_index_val, frames_left_ptr);
1315		LLVMValueRef frames_left_eq_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_index_val, usize_zero, "");
1316		LLVMBuildCondBr(g->builder, frames_left_eq_zero_bit, return_block, loop_block);
1317
1318		LLVMPositionBuilderAtEnd(g->builder, yes_wrap_block);
1319		LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, 1, false);
1320		LLVMValueRef plus_one = LLVMBuildNUWAdd(g->builder, src_index_val, usize_one, "");
1321		LLVMValueRef mod_len = LLVMBuildURem(g->builder, plus_one, src_len_val, "");
1322		LLVMBuildStore(g->builder, mod_len, frame_index_ptr);
1323		LLVMBuildStore(g->builder, src_len_val, frames_left_ptr);
1324		LLVMBuildBr(g->builder, loop_block);
1325
1326		LLVMPositionBuilderAtEnd(g->builder, loop_block);
1327		LLVMValueRef ptr_index = LLVMBuildLoad(g->builder, frame_index_ptr, "");
1328		LLVMValueRef addr_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr_val, &ptr_index, 1, "");
1329		LLVMValueRef this_addr_val = LLVMBuildLoad(g->builder, addr_ptr, "");
1330		LLVMValueRef args[] = {dest_stack_trace_ptr, this_addr_val};
1331		ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, "");
1332		LLVMValueRef prev_frames_left = LLVMBuildLoad(g->builder, frames_left_ptr, "");
1333		LLVMValueRef new_frames_left = LLVMBuildNUWSub(g->builder, prev_frames_left, usize_one, "");
1334		LLVMValueRef done_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, new_frames_left, usize_zero, "");
1335		LLVMBasicBlockRef continue_block = LLVMAppendBasicBlock(fn_val, "Continue");
1336		LLVMBuildCondBr(g->builder, done_bit, return_block, continue_block);
1337
1338		LLVMPositionBuilderAtEnd(g->builder, return_block);
1339		LLVMBuildRetVoid(g->builder);
1340
1341		LLVMPositionBuilderAtEnd(g->builder, continue_block);
1342		LLVMBuildStore(g->builder, new_frames_left, frames_left_ptr);
1343		LLVMValueRef prev_index = LLVMBuildLoad(g->builder, frame_index_ptr, "");
1344		LLVMValueRef index_plus_one = LLVMBuildNUWAdd(g->builder, prev_index, usize_one, "");
1345		LLVMValueRef index_mod_len = LLVMBuildURem(g->builder, index_plus_one, src_len_val, "");
1346		LLVMBuildStore(g->builder, index_mod_len, frame_index_ptr);
1347		LLVMBuildBr(g->builder, loop_block);
1348
1349		LLVMPositionBuilderAtEnd(g->builder, prev_block);
1350		if (!g->strip_debug_symbols) {
1351		LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
1352		}
1353
1354		g->merge_err_ret_traces_fn_val = fn_val;
1355		return fn_val;
1356
1357		}
1358
1359	1122	static LLVMValueRef get_return_err_fn(CodeGen *g) {
1360	1123	if (g->return_err_fn != nullptr)
1361	1124	return g->return_err_fn;
...	...	@@ -1364,7 +1127,7 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) {
1364	1127
1365	1128	LLVMTypeRef arg_types[] = {
1366	1129	// error return trace pointer
1367		get_llvm_type(g, get_ptr_to_stack_trace_type(g)),
	1130	get_llvm_type(g, ptr_to_stack_trace_type(g)),
1368	1131	};
1369	1132	LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 1, false);
1370	1133
...	...	@@ -1376,10 +1139,7 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) {
1376	1139	LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
1377	1140	addLLVMFnAttr(fn_val, "nounwind");
1378	1141	add_uwtable_attr(g, fn_val);
1379		// Error return trace memory is in the stack, which is impossible to be at address 0
1380		// on any architecture.
1381		addLLVMArgAttr(fn_val, (unsigned)0, "nonnull");
1382		if (g->build_mode == BuildModeDebug) {
	1142	if (codegen_have_frame_pointer(g)) {
1383	1143	ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
1384	1144	ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
1385	1145	}
...	...	@@ -1400,6 +1160,17 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) {
1400	1160	LLVMValueRef return_address_ptr = LLVMBuildCall(g->builder, get_return_address_fn_val(g), &zero, 1, "");
1401	1161	LLVMValueRef return_address = LLVMBuildPtrToInt(g->builder, return_address_ptr, usize_type_ref, "");
1402	1162
	1163	LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return");
	1164	LLVMBasicBlockRef dest_non_null_block = LLVMAppendBasicBlock(fn_val, "DestNonNull");
	1165
	1166	LLVMValueRef null_dest_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, err_ret_trace_ptr,
	1167	LLVMConstNull(LLVMTypeOf(err_ret_trace_ptr)), "");
	1168	LLVMBuildCondBr(g->builder, null_dest_bit, return_block, dest_non_null_block);
	1169
	1170	LLVMPositionBuilderAtEnd(g->builder, return_block);
	1171	LLVMBuildRetVoid(g->builder);
	1172
	1173	LLVMPositionBuilderAtEnd(g->builder, dest_non_null_block);
1403	1174	LLVMValueRef args[] = { err_ret_trace_ptr, return_address };
1404	1175	ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, "");
1405	1176	LLVMBuildRetVoid(g->builder);
...	...	@@ -1434,7 +1205,7 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) {
1434	1205	LLVMTypeRef fn_type_ref;
1435	1206	if (g->have_err_ret_tracing) {
1436	1207	LLVMTypeRef arg_types[] = {
1437		get_llvm_type(g, g->ptr_to_stack_trace_type),
	1208	get_llvm_type(g, get_pointer_to_type(g, get_stack_trace_type(g), false)),
1438	1209	get_llvm_type(g, g->err_tag_type),
1439	1210	};
1440	1211	fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 2, false);
...	...	@@ -1451,7 +1222,7 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) {
1451	1222	LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
1452	1223	addLLVMFnAttr(fn_val, "nounwind");
1453	1224	add_uwtable_attr(g, fn_val);
1454		if (g->build_mode == BuildModeDebug) {
	1225	if (codegen_have_frame_pointer(g)) {
1455	1226	ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
1456	1227	ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
1457	1228	}
...	...	@@ -1543,25 +1314,10 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) {
1543	1314	return fn_val;
1544	1315	}
1545	1316
1546		static bool is_coro_prelude_scope(Scope *scope) {
1547		while (scope != nullptr) {
1548		if (scope->id == ScopeIdCoroPrelude) {
1549		return true;
1550		} else if (scope->id == ScopeIdFnDef) {
1551		break;
1552		}
1553		scope = scope->parent;
1554		}
1555		return false;
1556		}
1557
1558	1317	static LLVMValueRef get_cur_err_ret_trace_val(CodeGen g, Scope scope) {
1559	1318	if (!g->have_err_ret_tracing) {
1560	1319	return nullptr;
1561	1320	}
1562		if (g->cur_fn->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync) {
1563		return is_coro_prelude_scope(scope) ? g->cur_err_ret_trace_val_arg : g->cur_err_ret_trace_val_stack;
1564		}
1565	1321	if (g->cur_err_ret_trace_val_stack != nullptr) {
1566	1322	return g->cur_err_ret_trace_val_stack;
1567	1323	}
...	...	@@ -1574,8 +1330,7 @@ static void gen_safety_crash_for_err(CodeGen g, LLVMValueRef err_val, Scope sc
1574	1330	if (g->have_err_ret_tracing) {
1575	1331	LLVMValueRef err_ret_trace_val = get_cur_err_ret_trace_val(g, scope);
1576	1332	if (err_ret_trace_val == nullptr) {
1577		ZigType *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g);
1578		err_ret_trace_val = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type));
	1333	err_ret_trace_val = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g)));
1579	1334	}
1580	1335	LLVMValueRef args[] = {
1581	1336	err_ret_trace_val,
...	...	@@ -1820,14 +1575,14 @@ static LLVMRealPredicate cmp_op_to_real_predicate(IrBinOp cmp_op) {
1820	1575	}
1821	1576	}
1822	1577
1823		static LLVMValueRef gen_assign_raw(CodeGen g, LLVMValueRef ptr, ZigType ptr_type,
	1578	static void gen_assign_raw(CodeGen g, LLVMValueRef ptr, ZigType ptr_type,
1824	1579	LLVMValueRef value)
1825	1580	{
1826	1581	assert(ptr_type->id == ZigTypeIdPointer);
1827	1582	ZigType *child_type = ptr_type->data.pointer.child_type;
1828	1583
1829	1584	if (!type_has_bits(child_type))
1830		return nullptr;
	1585	return;
1831	1586
1832	1587	if (handle_is_ptr(child_type)) {
1833	1588	assert(LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMPointerTypeKind);
...	...	@@ -1847,13 +1602,13 @@ static LLVMValueRef gen_assign_raw(CodeGen g, LLVMValueRef ptr, ZigType ptr_ty
1847	1602	ZigLLVMBuildMemCpy(g->builder, dest_ptr, align_bytes, src_ptr, align_bytes,
1848	1603	LLVMConstInt(usize->llvm_type, size_bytes, false),
1849	1604	ptr_type->data.pointer.is_volatile);
1850		return nullptr;
	1605	return;
1851	1606	}
1852	1607
1853	1608	uint32_t host_int_bytes = ptr_type->data.pointer.host_int_bytes;
1854	1609	if (host_int_bytes == 0) {
1855	1610	gen_store(g, value, ptr, ptr_type);
1856		return nullptr;
	1611	return;
1857	1612	}
1858	1613
1859	1614	bool big_endian = g->is_big_endian;
...	...	@@ -1883,7 +1638,7 @@ static LLVMValueRef gen_assign_raw(CodeGen g, LLVMValueRef ptr, ZigType ptr_ty
1883	1638	LLVMValueRef ored_value = LLVMBuildOr(g->builder, shifted_value, anded_containing_int, "");
1884	1639
1885	1640	gen_store(g, ored_value, ptr, ptr_type);
1886		return nullptr;
	1641	return;
1887	1642	}
1888	1643
1889	1644	static void gen_var_debug_decl(CodeGen g, ZigVar var) {
...	...	@@ -1967,7 +1722,7 @@ static bool iter_function_params_c_abi(CodeGen g, ZigType fn_type, FnWalk *fn_
1967	1722	param_info = &fn_type->data.fn.fn_type_id.param_info[src_i];
1968	1723	ty = param_info->type;
1969	1724	source_node = fn_walk->data.attrs.fn->proto_node;
1970		llvm_fn = fn_walk->data.attrs.fn->llvm_value;
	1725	llvm_fn = fn_walk->data.attrs.llvm_fn;
1971	1726	break;
1972	1727	case FnWalkIdCall: {
1973	1728	if (src_i >= fn_walk->data.call.inst->arg_count)
...	...	@@ -2149,10 +1904,12 @@ static bool iter_function_params_c_abi(CodeGen g, ZigType fn_type, FnWalk *fn_
2149	1904	}
2150	1905	case FnWalkIdInits: {
2151	1906	clear_debug_source_node(g);
2152		LLVMValueRef arg = LLVMGetParam(llvm_fn, fn_walk->data.inits.gen_i);
2153		LLVMTypeRef ptr_to_int_type_ref = LLVMPointerType(LLVMIntType((unsigned)ty_size * 8), 0);
2154		LLVMValueRef bitcasted = LLVMBuildBitCast(g->builder, var->value_ref, ptr_to_int_type_ref, "");
2155		gen_store_untyped(g, arg, bitcasted, var->align_bytes, false);
	1907	if (!fn_is_async(fn_walk->data.inits.fn)) {
	1908	LLVMValueRef arg = LLVMGetParam(llvm_fn, fn_walk->data.inits.gen_i);
	1909	LLVMTypeRef ptr_to_int_type_ref = LLVMPointerType(LLVMIntType((unsigned)ty_size * 8), 0);
	1910	LLVMValueRef bitcasted = LLVMBuildBitCast(g->builder, var->value_ref, ptr_to_int_type_ref, "");
	1911	gen_store_untyped(g, arg, bitcasted, var->align_bytes, false);
	1912	}
2156	1913	if (var->decl_node) {
2157	1914	gen_var_debug_decl(g, var);
2158	1915	}
...	...	@@ -2201,6 +1958,7 @@ void walk_function_params(CodeGen g, ZigType fn_type, FnWalk *fn_walk) {
2201	1958	LLVMValueRef param_value = ir_llvm_value(g, param_instruction);
2202	1959	assert(param_value);
2203	1960	fn_walk->data.call.gen_param_values->append(param_value);
	1961	fn_walk->data.call.gen_param_types->append(param_type);
2204	1962	}
2205	1963	}
2206	1964	return;
...	...	@@ -2216,7 +1974,7 @@ void walk_function_params(CodeGen g, ZigType fn_type, FnWalk *fn_walk) {
2216	1974
2217	1975	switch (fn_walk->id) {
2218	1976	case FnWalkIdAttrs: {
2219		LLVMValueRef llvm_fn = fn_walk->data.attrs.fn->llvm_value;
	1977	LLVMValueRef llvm_fn = fn_walk->data.attrs.llvm_fn;
2220	1978	bool is_byval = gen_info->is_byval;
2221	1979	FnTypeParamInfo *param_info = &fn_type->data.fn.fn_type_id.param_info[param_i];
2222	1980
...	...	@@ -2245,7 +2003,7 @@ void walk_function_params(CodeGen g, ZigType fn_type, FnWalk *fn_walk) {
2245	2003	assert(variable);
2246	2004	assert(variable->value_ref);
2247	2005
2248		if (!handle_is_ptr(variable->var_type)) {
	2006	if (!handle_is_ptr(variable->var_type) && !fn_is_async(fn_walk->data.inits.fn)) {
2249	2007	clear_debug_source_node(g);
2250	2008	ZigType *fn_type = fn_table_entry->type_entry;
2251	2009	unsigned gen_arg_index = fn_type->data.fn.gen_param_info[variable->src_arg_index].gen_index;
...	...	@@ -2271,48 +2029,357 @@ void walk_function_params(CodeGen g, ZigType fn_type, FnWalk *fn_walk) {
2271	2029	}
2272	2030	}
2273	2031
	2032	static LLVMValueRef get_merge_err_ret_traces_fn_val(CodeGen *g) {
	2033	if (g->merge_err_ret_traces_fn_val)
	2034	return g->merge_err_ret_traces_fn_val;
	2035
	2036	assert(g->stack_trace_type != nullptr);
	2037
	2038	LLVMTypeRef param_types[] = {
	2039	get_llvm_type(g, ptr_to_stack_trace_type(g)),
	2040	get_llvm_type(g, ptr_to_stack_trace_type(g)),
	2041	};
	2042	LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), param_types, 2, false);
	2043
	2044	Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_merge_error_return_traces"), false);
	2045	LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref);
	2046	LLVMSetLinkage(fn_val, LLVMInternalLinkage);
	2047	LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
	2048	addLLVMFnAttr(fn_val, "nounwind");
	2049	add_uwtable_attr(g, fn_val);
	2050	addLLVMArgAttr(fn_val, (unsigned)0, "noalias");
	2051	addLLVMArgAttr(fn_val, (unsigned)0, "writeonly");
	2052
	2053	addLLVMArgAttr(fn_val, (unsigned)1, "noalias");
	2054	addLLVMArgAttr(fn_val, (unsigned)1, "readonly");
	2055	if (g->build_mode == BuildModeDebug) {
	2056	ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
	2057	ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
	2058	}
	2059
	2060	// this is above the ZigLLVMClearCurrentDebugLocation
	2061	LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g);
	2062
	2063	LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
	2064	LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
	2065	LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
	2066	LLVMPositionBuilderAtEnd(g->builder, entry_block);
	2067	ZigLLVMClearCurrentDebugLocation(g->builder);
	2068
	2069	// if (dest_stack_trace == null or src_stack_trace == null) return;
	2070	// var frame_index: usize = undefined;
	2071	// var frames_left: usize = undefined;
	2072	// if (src_stack_trace.index < src_stack_trace.instruction_addresses.len) {
	2073	// frame_index = 0;
	2074	// frames_left = src_stack_trace.index;
	2075	// if (frames_left == 0) return;
	2076	// } else {
	2077	// frame_index = (src_stack_trace.index + 1) % src_stack_trace.instruction_addresses.len;
	2078	// frames_left = src_stack_trace.instruction_addresses.len;
	2079	// }
	2080	// while (true) {
	2081	// __zig_add_err_ret_trace_addr(dest_stack_trace, src_stack_trace.instruction_addresses[frame_index]);
	2082	// frames_left -= 1;
	2083	// if (frames_left == 0) return;
	2084	// frame_index = (frame_index + 1) % src_stack_trace.instruction_addresses.len;
	2085	// }
	2086	LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return");
	2087	LLVMBasicBlockRef non_null_block = LLVMAppendBasicBlock(fn_val, "NonNull");
	2088
	2089	LLVMValueRef frame_index_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frame_index");
	2090	LLVMValueRef frames_left_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frames_left");
	2091
	2092	LLVMValueRef dest_stack_trace_ptr = LLVMGetParam(fn_val, 0);
	2093	LLVMValueRef src_stack_trace_ptr = LLVMGetParam(fn_val, 1);
	2094
	2095	LLVMValueRef null_dest_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, dest_stack_trace_ptr,
	2096	LLVMConstNull(LLVMTypeOf(dest_stack_trace_ptr)), "");
	2097	LLVMValueRef null_src_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_stack_trace_ptr,
	2098	LLVMConstNull(LLVMTypeOf(src_stack_trace_ptr)), "");
	2099	LLVMValueRef null_bit = LLVMBuildOr(g->builder, null_dest_bit, null_src_bit, "");
	2100	LLVMBuildCondBr(g->builder, null_bit, return_block, non_null_block);
	2101
	2102	LLVMPositionBuilderAtEnd(g->builder, non_null_block);
	2103	size_t src_index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index;
	2104	size_t src_addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index;
	2105	LLVMValueRef src_index_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr,
	2106	(unsigned)src_index_field_index, "");
	2107	LLVMValueRef src_addresses_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr,
	2108	(unsigned)src_addresses_field_index, "");
	2109	ZigType *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry;
	2110	size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index;
	2111	LLVMValueRef src_ptr_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)ptr_field_index, "");
	2112	size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index;
	2113	LLVMValueRef src_len_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)len_field_index, "");
	2114	LLVMValueRef src_index_val = LLVMBuildLoad(g->builder, src_index_field_ptr, "");
	2115	LLVMValueRef src_ptr_val = LLVMBuildLoad(g->builder, src_ptr_field_ptr, "");
	2116	LLVMValueRef src_len_val = LLVMBuildLoad(g->builder, src_len_field_ptr, "");
	2117	LLVMValueRef no_wrap_bit = LLVMBuildICmp(g->builder, LLVMIntULT, src_index_val, src_len_val, "");
	2118	LLVMBasicBlockRef no_wrap_block = LLVMAppendBasicBlock(fn_val, "NoWrap");
	2119	LLVMBasicBlockRef yes_wrap_block = LLVMAppendBasicBlock(fn_val, "YesWrap");
	2120	LLVMBasicBlockRef loop_block = LLVMAppendBasicBlock(fn_val, "Loop");
	2121	LLVMBuildCondBr(g->builder, no_wrap_bit, no_wrap_block, yes_wrap_block);
	2122
	2123	LLVMPositionBuilderAtEnd(g->builder, no_wrap_block);
	2124	LLVMValueRef usize_zero = LLVMConstNull(g->builtin_types.entry_usize->llvm_type);
	2125	LLVMBuildStore(g->builder, usize_zero, frame_index_ptr);
	2126	LLVMBuildStore(g->builder, src_index_val, frames_left_ptr);
	2127	LLVMValueRef frames_left_eq_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_index_val, usize_zero, "");
	2128	LLVMBuildCondBr(g->builder, frames_left_eq_zero_bit, return_block, loop_block);
	2129
	2130	LLVMPositionBuilderAtEnd(g->builder, yes_wrap_block);
	2131	LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, 1, false);
	2132	LLVMValueRef plus_one = LLVMBuildNUWAdd(g->builder, src_index_val, usize_one, "");
	2133	LLVMValueRef mod_len = LLVMBuildURem(g->builder, plus_one, src_len_val, "");
	2134	LLVMBuildStore(g->builder, mod_len, frame_index_ptr);
	2135	LLVMBuildStore(g->builder, src_len_val, frames_left_ptr);
	2136	LLVMBuildBr(g->builder, loop_block);
	2137
	2138	LLVMPositionBuilderAtEnd(g->builder, loop_block);
	2139	LLVMValueRef ptr_index = LLVMBuildLoad(g->builder, frame_index_ptr, "");
	2140	LLVMValueRef addr_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr_val, &ptr_index, 1, "");
	2141	LLVMValueRef this_addr_val = LLVMBuildLoad(g->builder, addr_ptr, "");
	2142	LLVMValueRef args[] = {dest_stack_trace_ptr, this_addr_val};
	2143	ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, "");
	2144	LLVMValueRef prev_frames_left = LLVMBuildLoad(g->builder, frames_left_ptr, "");
	2145	LLVMValueRef new_frames_left = LLVMBuildNUWSub(g->builder, prev_frames_left, usize_one, "");
	2146	LLVMValueRef done_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, new_frames_left, usize_zero, "");
	2147	LLVMBasicBlockRef continue_block = LLVMAppendBasicBlock(fn_val, "Continue");
	2148	LLVMBuildCondBr(g->builder, done_bit, return_block, continue_block);
	2149
	2150	LLVMPositionBuilderAtEnd(g->builder, return_block);
	2151	LLVMBuildRetVoid(g->builder);
	2152
	2153	LLVMPositionBuilderAtEnd(g->builder, continue_block);
	2154	LLVMBuildStore(g->builder, new_frames_left, frames_left_ptr);
	2155	LLVMValueRef prev_index = LLVMBuildLoad(g->builder, frame_index_ptr, "");
	2156	LLVMValueRef index_plus_one = LLVMBuildNUWAdd(g->builder, prev_index, usize_one, "");
	2157	LLVMValueRef index_mod_len = LLVMBuildURem(g->builder, index_plus_one, src_len_val, "");
	2158	LLVMBuildStore(g->builder, index_mod_len, frame_index_ptr);
	2159	LLVMBuildBr(g->builder, loop_block);
	2160
	2161	LLVMPositionBuilderAtEnd(g->builder, prev_block);
	2162	if (!g->strip_debug_symbols) {
	2163	LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
	2164	}
	2165
	2166	g->merge_err_ret_traces_fn_val = fn_val;
	2167	return fn_val;
	2168
	2169	}
2274	2170	static LLVMValueRef ir_render_save_err_ret_addr(CodeGen g, IrExecutable executable,
2275	2171	IrInstructionSaveErrRetAddr *save_err_ret_addr_instruction)
2276	2172	{
2277	2173	assert(g->have_err_ret_tracing);
2278	2174
2279	2175	LLVMValueRef return_err_fn = get_return_err_fn(g);
2280		LLVMValueRef args[] = {
2281		get_cur_err_ret_trace_val(g, save_err_ret_addr_instruction->base.scope),
2282		};
2283		LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1,
	2176	LLVMValueRef my_err_trace_val = get_cur_err_ret_trace_val(g, save_err_ret_addr_instruction->base.scope);
	2177	ZigLLVMBuildCall(g->builder, return_err_fn, &my_err_trace_val, 1,
2284	2178	get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
2285		return call_instruction;
	2179
	2180	ZigType *ret_type = g->cur_fn->type_entry->data.fn.fn_type_id.return_type;
	2181	if (fn_is_async(g->cur_fn) && codegen_fn_has_err_ret_tracing_arg(g, ret_type)) {
	2182	LLVMValueRef trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
	2183	frame_index_trace_arg(g, ret_type), "");
	2184	LLVMBuildStore(g->builder, my_err_trace_val, trace_ptr_ptr);
	2185	}
	2186
	2187	return nullptr;
	2188	}
	2189
	2190	static void gen_assert_resume_id(CodeGen g, IrInstruction source_instr, ResumeId resume_id, PanicMsgId msg_id,
	2191	LLVMBasicBlockRef end_bb)
	2192	{
	2193	LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
	2194	LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
	2195	if (end_bb == nullptr) end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume");
	2196	LLVMValueRef expected_value = LLVMConstSub(LLVMConstAllOnes(usize_type_ref),
	2197	LLVMConstInt(usize_type_ref, resume_id, false));
	2198	LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, LLVMGetParam(g->cur_fn_val, 1), expected_value, "");
	2199	LLVMBuildCondBr(g->builder, ok_bit, end_bb, bad_resume_block);
	2200
	2201	LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);
	2202	gen_assertion(g, msg_id, source_instr);
	2203
	2204	LLVMPositionBuilderAtEnd(g->builder, end_bb);
2286	2205	}
2287	2206
2288		static LLVMValueRef ir_render_return(CodeGen g, IrExecutable executable, IrInstructionReturn *return_instruction) {
	2207	static LLVMValueRef gen_resume(CodeGen *g, LLVMValueRef fn_val, LLVMValueRef target_frame_ptr, ResumeId resume_id) {
	2208	LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
	2209	if (fn_val == nullptr) {
	2210	LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_fn_ptr_index, "");
	2211	fn_val = LLVMBuildLoad(g->builder, fn_ptr_ptr, "");
	2212	}
	2213	LLVMValueRef arg_val = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref),
	2214	LLVMConstInt(usize_type_ref, resume_id, false), "");
	2215	LLVMValueRef args[] = {target_frame_ptr, arg_val};
	2216	return ZigLLVMBuildCall(g->builder, fn_val, args, 2, LLVMFastCallConv, ZigLLVM_FnInlineAuto, "");
	2217	}
	2218
	2219	static LLVMBasicBlockRef gen_suspend_begin(CodeGen g, const char name_hint) {
	2220	LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
	2221	LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, name_hint);
	2222	size_t new_block_index = g->cur_resume_block_count;
	2223	g->cur_resume_block_count += 1;
	2224	LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false);
	2225	LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb);
	2226	LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr);
	2227	return resume_bb;
	2228	}
	2229
	2230	static void set_tail_call_if_appropriate(CodeGen *g, LLVMValueRef call_inst) {
	2231	LLVMSetTailCall(call_inst, true);
	2232	}
	2233
	2234	static LLVMValueRef gen_maybe_atomic_op(CodeGen *g, LLVMAtomicRMWBinOp op, LLVMValueRef ptr, LLVMValueRef val,
	2235	LLVMAtomicOrdering order)
	2236	{
	2237	if (g->is_single_threaded) {
	2238	LLVMValueRef loaded = LLVMBuildLoad(g->builder, ptr, "");
	2239	LLVMValueRef modified;
	2240	switch (op) {
	2241	case LLVMAtomicRMWBinOpXchg:
	2242	modified = val;
	2243	break;
	2244	case LLVMAtomicRMWBinOpXor:
	2245	modified = LLVMBuildXor(g->builder, loaded, val, "");
	2246	break;
	2247	default:
	2248	zig_unreachable();
	2249	}
	2250	LLVMBuildStore(g->builder, modified, ptr);
	2251	return loaded;
	2252	} else {
	2253	return LLVMBuildAtomicRMW(g->builder, op, ptr, val, order, false);
	2254	}
	2255	}
	2256
	2257	static void gen_async_return(CodeGen g, IrInstructionReturn instruction) {
	2258	LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
	2259
	2260	ZigType *operand_type = (instruction->operand != nullptr) ? instruction->operand->value.type : nullptr;
	2261	bool operand_has_bits = (operand_type != nullptr) && type_has_bits(operand_type);
	2262	ZigType *ret_type = g->cur_fn->type_entry->data.fn.fn_type_id.return_type;
	2263	bool ret_type_has_bits = type_has_bits(ret_type);
	2264
	2265	if (operand_has_bits && instruction->operand != nullptr) {
	2266	bool need_store = instruction->operand->value.special != ConstValSpecialRuntime \|\| !handle_is_ptr(ret_type);
	2267	if (need_store) {
	2268	// It didn't get written to the result ptr. We do that now.
	2269	ZigType *ret_ptr_type = get_pointer_to_type(g, ret_type, true);
	2270	gen_assign_raw(g, g->cur_ret_ptr, ret_ptr_type, ir_llvm_value(g, instruction->operand));
	2271	}
	2272	}
	2273
	2274	// Whether we tail resume the awaiter, or do an early return, we are done and will not be resumed.
	2275	if (ir_want_runtime_safety(g, &instruction->base)) {
	2276	LLVMValueRef new_resume_index = LLVMConstAllOnes(usize_type_ref);
	2277	LLVMBuildStore(g->builder, new_resume_index, g->cur_async_resume_index_ptr);
	2278	}
	2279
	2280	LLVMValueRef zero = LLVMConstNull(usize_type_ref);
	2281	LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
	2282
	2283	LLVMValueRef prev_val = gen_maybe_atomic_op(g, LLVMAtomicRMWBinOpXor, g->cur_async_awaiter_ptr,
	2284	all_ones, LLVMAtomicOrderingAcquire);
	2285
	2286	LLVMBasicBlockRef bad_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadReturn");
	2287	LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");
	2288	LLVMBasicBlockRef resume_them_block = LLVMAppendBasicBlock(g->cur_fn_val, "ResumeThem");
	2289
	2290	LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, resume_them_block, 2);
	2291
	2292	LLVMAddCase(switch_instr, zero, early_return_block);
	2293	LLVMAddCase(switch_instr, all_ones, bad_return_block);
	2294
	2295	// Something has gone horribly wrong, and this is an invalid second return.
	2296	LLVMPositionBuilderAtEnd(g->builder, bad_return_block);
	2297	gen_assertion(g, PanicMsgIdBadReturn, &instruction->base);
	2298
	2299	// There is no awaiter yet, but we're completely done.
	2300	LLVMPositionBuilderAtEnd(g->builder, early_return_block);
	2301	LLVMBuildRetVoid(g->builder);
	2302
	2303	// We need to resume the caller by tail calling them,
	2304	// but first write through the result pointer and possibly
	2305	// error return trace pointer.
	2306	LLVMPositionBuilderAtEnd(g->builder, resume_them_block);
	2307
	2308	if (ret_type_has_bits) {
	2309	// If the awaiter result pointer is non-null, we need to copy the result to there.
	2310	LLVMBasicBlockRef copy_block = LLVMAppendBasicBlock(g->cur_fn_val, "CopyResult");
	2311	LLVMBasicBlockRef copy_end_block = LLVMAppendBasicBlock(g->cur_fn_val, "CopyResultEnd");
	2312	LLVMValueRef awaiter_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_ret_start + 1, "");
	2313	LLVMValueRef awaiter_ret_ptr = LLVMBuildLoad(g->builder, awaiter_ret_ptr_ptr, "");
	2314	LLVMValueRef zero_ptr = LLVMConstNull(LLVMTypeOf(awaiter_ret_ptr));
	2315	LLVMValueRef need_copy_bit = LLVMBuildICmp(g->builder, LLVMIntNE, awaiter_ret_ptr, zero_ptr, "");
	2316	LLVMBuildCondBr(g->builder, need_copy_bit, copy_block, copy_end_block);
	2317
	2318	LLVMPositionBuilderAtEnd(g->builder, copy_block);
	2319	LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
	2320	LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, awaiter_ret_ptr, ptr_u8, "");
	2321	LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, g->cur_ret_ptr, ptr_u8, "");
	2322	bool is_volatile = false;
	2323	uint32_t abi_align = get_abi_alignment(g, ret_type);
	2324	LLVMValueRef byte_count_val = LLVMConstInt(usize_type_ref, type_size(g, ret_type), false);
	2325	ZigLLVMBuildMemCpy(g->builder,
	2326	dest_ptr_casted, abi_align,
	2327	src_ptr_casted, abi_align, byte_count_val, is_volatile);
	2328	LLVMBuildBr(g->builder, copy_end_block);
	2329
	2330	LLVMPositionBuilderAtEnd(g->builder, copy_end_block);
	2331	if (codegen_fn_has_err_ret_tracing_arg(g, ret_type)) {
	2332	LLVMValueRef awaiter_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
	2333	frame_index_trace_arg(g, ret_type) + 1, "");
	2334	LLVMValueRef dest_trace_ptr = LLVMBuildLoad(g->builder, awaiter_trace_ptr_ptr, "");
	2335	LLVMValueRef my_err_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope);
	2336	LLVMValueRef args[] = { dest_trace_ptr, my_err_trace_val };
	2337	ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2,
	2338	get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
	2339	}
	2340	}
	2341
	2342	// Resume the caller by tail calling them.
	2343	ZigType *any_frame_type = get_any_frame_type(g, ret_type);
	2344	LLVMValueRef their_frame_ptr = LLVMBuildIntToPtr(g->builder, prev_val, get_llvm_type(g, any_frame_type), "");
	2345	LLVMValueRef call_inst = gen_resume(g, nullptr, their_frame_ptr, ResumeIdReturn);
	2346	set_tail_call_if_appropriate(g, call_inst);
	2347	LLVMBuildRetVoid(g->builder);
	2348	}
	2349
	2350	static LLVMValueRef ir_render_return(CodeGen g, IrExecutable executable, IrInstructionReturn *instruction) {
	2351	if (fn_is_async(g->cur_fn)) {
	2352	gen_async_return(g, instruction);
	2353	return nullptr;
	2354	}
	2355
2289	2356	if (want_first_arg_sret(g, &g->cur_fn->type_entry->data.fn.fn_type_id)) {
2290		if (return_instruction->value == nullptr) {
	2357	if (instruction->operand == nullptr) {
2291	2358	LLVMBuildRetVoid(g->builder);
2292	2359	return nullptr;
2293	2360	}
2294	2361	assert(g->cur_ret_ptr);
2295		src_assert(return_instruction->value->value.special != ConstValSpecialRuntime,
2296		return_instruction->base.source_node);
2297		LLVMValueRef value = ir_llvm_value(g, return_instruction->value);
2298		ZigType *return_type = return_instruction->value->value.type;
	2362	src_assert(instruction->operand->value.special != ConstValSpecialRuntime,
	2363	instruction->base.source_node);
	2364	LLVMValueRef value = ir_llvm_value(g, instruction->operand);
	2365	ZigType *return_type = instruction->operand->value.type;
2299	2366	gen_assign_raw(g, g->cur_ret_ptr, get_pointer_to_type(g, return_type, false), value);
2300	2367	LLVMBuildRetVoid(g->builder);
2301	2368	} else if (g->cur_fn->type_entry->data.fn.fn_type_id.cc != CallingConventionAsync &&
2302	2369	handle_is_ptr(g->cur_fn->type_entry->data.fn.fn_type_id.return_type))
2303	2370	{
2304		if (return_instruction->value == nullptr) {
	2371	if (instruction->operand == nullptr) {
2305	2372	LLVMValueRef by_val_value = gen_load_untyped(g, g->cur_ret_ptr, 0, false, "");
2306	2373	LLVMBuildRet(g->builder, by_val_value);
2307	2374	} else {
2308		LLVMValueRef value = ir_llvm_value(g, return_instruction->value);
	2375	LLVMValueRef value = ir_llvm_value(g, instruction->operand);
2309	2376	LLVMValueRef by_val_value = gen_load_untyped(g, value, 0, false, "");
2310	2377	LLVMBuildRet(g->builder, by_val_value);
2311	2378	}
2312		} else if (return_instruction->value == nullptr) {
	2379	} else if (instruction->operand == nullptr) {
2313	2380	LLVMBuildRetVoid(g->builder);
2314	2381	} else {
2315		LLVMValueRef value = ir_llvm_value(g, return_instruction->value);
	2382	LLVMValueRef value = ir_llvm_value(g, instruction->operand);
2316	2383	LLVMBuildRet(g->builder, value);
2317	2384	}
2318	2385	return nullptr;
...	...	@@ -3242,14 +3309,17 @@ static LLVMValueRef ir_render_bool_not(CodeGen g, IrExecutable executable, IrI
3242	3309	return LLVMBuildICmp(g->builder, LLVMIntEQ, value, zero, "");
3243	3310	}
3244	3311
3245		static LLVMValueRef ir_render_decl_var(CodeGen g, IrExecutable executable, IrInstructionDeclVarGen *instruction) {
3246		ZigVar *var = instruction->var;
3247
	3312	static void render_decl_var(CodeGen g, ZigVar var) {
3248	3313	if (!type_has_bits(var->var_type))
3249		return nullptr;
	3314	return;
3250	3315
3251		var->value_ref = ir_llvm_value(g, instruction->var_ptr);
	3316	var->value_ref = ir_llvm_value(g, var->ptr_instruction);
3252	3317	gen_var_debug_decl(g, var);
	3318	}
	3319
	3320	static LLVMValueRef ir_render_decl_var(CodeGen g, IrExecutable executable, IrInstructionDeclVarGen *instruction) {
	3321	instruction->var->ptr_instruction = instruction->var_ptr;
	3322	render_decl_var(g, instruction->var);
3253	3323	return nullptr;
3254	3324	}
3255	3325
...	...	@@ -3467,8 +3537,9 @@ static LLVMValueRef ir_render_var_ptr(CodeGen g, IrExecutable executable, IrIn
3467	3537	static LLVMValueRef ir_render_return_ptr(CodeGen g, IrExecutable executable,
3468	3538	IrInstructionReturnPtr *instruction)
3469	3539	{
3470		src_assert(g->cur_ret_ptr != nullptr \|\| !type_has_bits(instruction->base.value.type),
3471		instruction->base.source_node);
	3540	if (!type_has_bits(instruction->base.value.type))
	3541	return nullptr;
	3542	src_assert(g->cur_ret_ptr != nullptr, instruction->base.source_node);
3472	3543	return g->cur_ret_ptr;
3473	3544	}
3474	3545
...	...	@@ -3566,26 +3637,6 @@ static LLVMValueRef ir_render_elem_ptr(CodeGen g, IrExecutable executable, IrI
3566	3637	}
3567	3638	}
3568	3639
3569		static bool get_prefix_arg_err_ret_stack(CodeGen g, FnTypeId fn_type_id) {
3570		return g->have_err_ret_tracing &&
3571		(fn_type_id->return_type->id == ZigTypeIdErrorUnion \|\|
3572		fn_type_id->return_type->id == ZigTypeIdErrorSet \|\|
3573		fn_type_id->cc == CallingConventionAsync);
3574		}
3575
3576		static size_t get_async_allocator_arg_index(CodeGen g, FnTypeId fn_type_id) {
3577		// 0 1 2 3
3578		// err_ret_stack allocator_ptr err_code other_args...
3579		return get_prefix_arg_err_ret_stack(g, fn_type_id) ? 1 : 0;
3580		}
3581
3582		static size_t get_async_err_code_arg_index(CodeGen g, FnTypeId fn_type_id) {
3583		// 0 1 2 3
3584		// err_ret_stack allocator_ptr err_code other_args...
3585		return 1 + get_async_allocator_arg_index(g, fn_type_id);
3586		}
3587
3588
3589	3640	static LLVMValueRef get_new_stack_addr(CodeGen *g, LLVMValueRef new_stack) {
3590	3641	LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, new_stack, (unsigned)slice_ptr_index, "");
3591	3642	LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, new_stack, (unsigned)slice_len_index, "");
...	...	@@ -3623,16 +3674,124 @@ static void set_call_instr_sret(CodeGen *g, LLVMValueRef call_instr) {
3623	3674	LLVMAddCallSiteAttribute(call_instr, 1, sret_attr);
3624	3675	}
3625	3676
	3677	static void render_async_spills(CodeGen *g) {
	3678	ZigType *fn_type = g->cur_fn->type_entry;
	3679	ZigType *import = get_scope_import(&g->cur_fn->fndef_scope->base);
	3680	uint32_t async_var_index = frame_index_arg(g, fn_type->data.fn.fn_type_id.return_type);
	3681	for (size_t var_i = 0; var_i < g->cur_fn->variable_list.length; var_i += 1) {
	3682	ZigVar *var = g->cur_fn->variable_list.at(var_i);
	3683
	3684	if (!type_has_bits(var->var_type)) {
	3685	continue;
	3686	}
	3687	if (ir_get_var_is_comptime(var))
	3688	continue;
	3689	switch (type_requires_comptime(g, var->var_type)) {
	3690	case ReqCompTimeInvalid:
	3691	zig_unreachable();
	3692	case ReqCompTimeYes:
	3693	continue;
	3694	case ReqCompTimeNo:
	3695	break;
	3696	}
	3697	if (var->src_arg_index == SIZE_MAX) {
	3698	continue;
	3699	}
	3700
	3701	var->value_ref = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, async_var_index,
	3702	buf_ptr(&var->name));
	3703	async_var_index += 1;
	3704	if (var->decl_node) {
	3705	var->di_loc_var = ZigLLVMCreateAutoVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
	3706	buf_ptr(&var->name), import->data.structure.root_struct->di_file,
	3707	(unsigned)(var->decl_node->line + 1),
	3708	get_llvm_di_type(g, var->var_type), !g->strip_debug_symbols, 0);
	3709	gen_var_debug_decl(g, var);
	3710	}
	3711	}
	3712
	3713	ZigType *frame_type = g->cur_fn->frame_type->data.frame.locals_struct;
	3714
	3715	for (size_t alloca_i = 0; alloca_i < g->cur_fn->alloca_gen_list.length; alloca_i += 1) {
	3716	IrInstructionAllocaGen *instruction = g->cur_fn->alloca_gen_list.at(alloca_i);
	3717	if (instruction->field_index == SIZE_MAX)
	3718	continue;
	3719
	3720	size_t gen_index = frame_type->data.structure.fields[instruction->field_index].gen_index;
	3721	instruction->base.llvm_value = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, gen_index,
	3722	instruction->name_hint);
	3723	}
	3724	}
	3725
	3726	static void render_async_var_decls(CodeGen g, Scope scope) {
	3727	for (;;) {
	3728	switch (scope->id) {
	3729	case ScopeIdCImport:
	3730	zig_unreachable();
	3731	case ScopeIdFnDef:
	3732	return;
	3733	case ScopeIdVarDecl: {
	3734	ZigVar var = reinterpret_cast<ScopeVarDecl >(scope)->var;
	3735	if (var->ptr_instruction != nullptr) {
	3736	render_decl_var(g, var);
	3737	}
	3738	// fallthrough
	3739	}
	3740	case ScopeIdDecls:
	3741	case ScopeIdBlock:
	3742	case ScopeIdDefer:
	3743	case ScopeIdDeferExpr:
	3744	case ScopeIdLoop:
	3745	case ScopeIdSuspend:
	3746	case ScopeIdCompTime:
	3747	case ScopeIdRuntime:
	3748	scope = scope->parent;
	3749	continue;
	3750	}
	3751	}
	3752	}
	3753
	3754	static LLVMValueRef gen_frame_size(CodeGen *g, LLVMValueRef fn_val) {
	3755	LLVMTypeRef usize_llvm_type = g->builtin_types.entry_usize->llvm_type;
	3756	LLVMTypeRef ptr_usize_llvm_type = LLVMPointerType(usize_llvm_type, 0);
	3757	LLVMValueRef casted_fn_val = LLVMBuildBitCast(g->builder, fn_val, ptr_usize_llvm_type, "");
	3758	LLVMValueRef negative_one = LLVMConstInt(LLVMInt32Type(), -1, true);
	3759	LLVMValueRef prefix_ptr = LLVMBuildInBoundsGEP(g->builder, casted_fn_val, &negative_one, 1, "");
	3760	return LLVMBuildLoad(g->builder, prefix_ptr, "");
	3761	}
	3762
	3763	static void gen_init_stack_trace(CodeGen *g, LLVMValueRef trace_field_ptr, LLVMValueRef addrs_field_ptr) {
	3764	LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
	3765	LLVMValueRef zero = LLVMConstNull(usize_type_ref);
	3766
	3767	LLVMValueRef index_ptr = LLVMBuildStructGEP(g->builder, trace_field_ptr, 0, "");
	3768	LLVMBuildStore(g->builder, zero, index_ptr);
	3769
	3770	LLVMValueRef addrs_slice_ptr = LLVMBuildStructGEP(g->builder, trace_field_ptr, 1, "");
	3771	LLVMValueRef addrs_ptr_ptr = LLVMBuildStructGEP(g->builder, addrs_slice_ptr, slice_ptr_index, "");
	3772	LLVMValueRef indices[] = { LLVMConstNull(usize_type_ref), LLVMConstNull(usize_type_ref) };
	3773	LLVMValueRef trace_field_addrs_as_ptr = LLVMBuildInBoundsGEP(g->builder, addrs_field_ptr, indices, 2, "");
	3774	LLVMBuildStore(g->builder, trace_field_addrs_as_ptr, addrs_ptr_ptr);
	3775
	3776	LLVMValueRef addrs_len_ptr = LLVMBuildStructGEP(g->builder, addrs_slice_ptr, slice_len_index, "");
	3777	LLVMBuildStore(g->builder, LLVMConstInt(usize_type_ref, stack_trace_ptr_count, false), addrs_len_ptr);
	3778	}
	3779
3626	3780	static LLVMValueRef ir_render_call(CodeGen g, IrExecutable executable, IrInstructionCallGen *instruction) {
	3781	LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
	3782
3627	3783	LLVMValueRef fn_val;
3628	3784	ZigType *fn_type;
	3785	bool callee_is_async;
3629	3786	if (instruction->fn_entry) {
3630	3787	fn_val = fn_llvm_value(g, instruction->fn_entry);
3631	3788	fn_type = instruction->fn_entry->type_entry;
	3789	callee_is_async = fn_is_async(instruction->fn_entry);
3632	3790	} else {
3633	3791	assert(instruction->fn_ref);
3634	3792	fn_val = ir_llvm_value(g, instruction->fn_ref);
3635	3793	fn_type = instruction->fn_ref->value.type;
	3794	callee_is_async = fn_type->data.fn.fn_type_id.cc == CallingConventionAsync;
3636	3795	}
3637	3796
3638	3797	FnTypeId *fn_type_id = &fn_type->data.fn.fn_type_id;
...	...	@@ -3643,27 +3802,154 @@ static LLVMValueRef ir_render_call(CodeGen g, IrExecutable executable, IrInstr
3643	3802	CallingConvention cc = fn_type->data.fn.fn_type_id.cc;
3644	3803
3645	3804	bool first_arg_ret = ret_has_bits && want_first_arg_sret(g, fn_type_id);
3646		bool prefix_arg_err_ret_stack = get_prefix_arg_err_ret_stack(g, fn_type_id);
	3805	bool prefix_arg_err_ret_stack = codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type);
3647	3806	bool is_var_args = fn_type_id->is_var_args;
3648	3807	ZigList<LLVMValueRef> gen_param_values = {};
	3808	ZigList<ZigType *> gen_param_types = {};
3649	3809	LLVMValueRef result_loc = instruction->result_loc ? ir_llvm_value(g, instruction->result_loc) : nullptr;
3650		if (first_arg_ret) {
3651		gen_param_values.append(result_loc);
3652		}
3653		if (prefix_arg_err_ret_stack) {
3654		gen_param_values.append(get_cur_err_ret_trace_val(g, instruction->base.scope));
3655		}
3656		if (instruction->is_async) {
3657		gen_param_values.append(ir_llvm_value(g, instruction->async_allocator));
	3810	LLVMValueRef zero = LLVMConstNull(usize_type_ref);
	3811	LLVMValueRef frame_result_loc;
	3812	LLVMValueRef awaiter_init_val;
	3813	LLVMValueRef ret_ptr;
	3814	if (callee_is_async) {
	3815	if (instruction->is_async) {
	3816	if (instruction->new_stack == nullptr) {
	3817	awaiter_init_val = zero;
	3818	frame_result_loc = result_loc;
	3819
	3820	if (ret_has_bits) {
	3821	// Use the result location which is inside the frame if this is an async call.
	3822	ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
	3823	}
	3824	} else if (cc == CallingConventionAsync) {
	3825	awaiter_init_val = zero;
	3826	LLVMValueRef frame_slice_ptr = ir_llvm_value(g, instruction->new_stack);
	3827	if (ir_want_runtime_safety(g, &instruction->base)) {
	3828	LLVMValueRef given_len_ptr = LLVMBuildStructGEP(g->builder, frame_slice_ptr, slice_len_index, "");
	3829	LLVMValueRef given_frame_len = LLVMBuildLoad(g->builder, given_len_ptr, "");
	3830	LLVMValueRef actual_frame_len = gen_frame_size(g, fn_val);
	3831
	3832	LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "FrameSizeCheckFail");
	3833	LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "FrameSizeCheckOk");
	3834
	3835	LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntUGE, given_frame_len, actual_frame_len, "");
	3836	LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
	3837
	3838	LLVMPositionBuilderAtEnd(g->builder, fail_block);
	3839	gen_safety_crash(g, PanicMsgIdFrameTooSmall);
3658	3840
3659		LLVMValueRef err_val_ptr = LLVMBuildStructGEP(g->builder, result_loc, err_union_err_index, "");
3660		gen_param_values.append(err_val_ptr);
	3841	LLVMPositionBuilderAtEnd(g->builder, ok_block);
	3842	}
	3843	LLVMValueRef frame_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_slice_ptr, slice_ptr_index, "");
	3844	LLVMValueRef frame_ptr = LLVMBuildLoad(g->builder, frame_ptr_ptr, "");
	3845	frame_result_loc = LLVMBuildBitCast(g->builder, frame_ptr,
	3846	get_llvm_type(g, instruction->base.value.type), "");
	3847
	3848	if (ret_has_bits) {
	3849	// Use the result location provided to the @asyncCall builtin
	3850	ret_ptr = result_loc;
	3851	}
	3852	}
	3853
	3854	// even if prefix_arg_err_ret_stack is true, let the async function do its own
	3855	// initialization.
	3856	} else {
	3857	// async function called as a normal function
	3858
	3859	frame_result_loc = ir_llvm_value(g, instruction->frame_result_loc);
	3860	awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_frame_ptr, usize_type_ref, ""); // caller's own frame pointer
	3861	if (ret_has_bits) {
	3862	if (result_loc == nullptr) {
	3863	// return type is a scalar, but we still need a pointer to it. Use the async fn frame.
	3864	ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
	3865	} else {
	3866	// Use the call instruction's result location.
	3867	ret_ptr = result_loc;
	3868	}
	3869
	3870	// Store a zero in the awaiter's result ptr to indicate we do not need a copy made.
	3871	LLVMValueRef awaiter_ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 1, "");
	3872	LLVMValueRef zero_ptr = LLVMConstNull(LLVMGetElementType(LLVMTypeOf(awaiter_ret_ptr)));
	3873	LLVMBuildStore(g->builder, zero_ptr, awaiter_ret_ptr);
	3874	}
	3875
	3876	if (prefix_arg_err_ret_stack) {
	3877	LLVMValueRef err_ret_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
	3878	frame_index_trace_arg(g, src_return_type) + 1, "");
	3879	LLVMValueRef my_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope);
	3880	LLVMBuildStore(g->builder, my_err_ret_trace_val, err_ret_trace_ptr_ptr);
	3881	}
	3882	}
	3883
	3884	assert(frame_result_loc != nullptr);
	3885
	3886	LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_fn_ptr_index, "");
	3887	LLVMValueRef bitcasted_fn_val = LLVMBuildBitCast(g->builder, fn_val,
	3888	LLVMGetElementType(LLVMTypeOf(fn_ptr_ptr)), "");
	3889	LLVMBuildStore(g->builder, bitcasted_fn_val, fn_ptr_ptr);
	3890
	3891	LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_resume_index, "");
	3892	LLVMBuildStore(g->builder, zero, resume_index_ptr);
	3893
	3894	LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_awaiter_index, "");
	3895	LLVMBuildStore(g->builder, awaiter_init_val, awaiter_ptr);
	3896
	3897	if (ret_has_bits) {
	3898	LLVMValueRef ret_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start, "");
	3899	LLVMBuildStore(g->builder, ret_ptr, ret_ptr_ptr);
	3900	}
	3901	} else if (instruction->is_async) {
	3902	// Async call of blocking function
	3903	if (instruction->new_stack != nullptr) {
	3904	zig_panic("TODO @asyncCall of non-async function");
	3905	}
	3906	frame_result_loc = result_loc;
	3907	awaiter_init_val = LLVMConstAllOnes(usize_type_ref);
	3908
	3909	LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_awaiter_index, "");
	3910	LLVMBuildStore(g->builder, awaiter_init_val, awaiter_ptr);
	3911
	3912	if (ret_has_bits) {
	3913	LLVMValueRef ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
	3914	LLVMValueRef ret_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start, "");
	3915	LLVMBuildStore(g->builder, ret_ptr, ret_ptr_ptr);
	3916
	3917	if (first_arg_ret) {
	3918	gen_param_values.append(ret_ptr);
	3919	}
	3920	if (prefix_arg_err_ret_stack) {
	3921	// Set up the callee stack trace pointer pointing into the frame.
	3922	// Then we have to wire up the StackTrace pointers.
	3923	// Await is responsible for merging error return traces.
	3924	uint32_t trace_field_index_start = frame_index_trace_arg(g, src_return_type);
	3925	LLVMValueRef callee_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
	3926	trace_field_index_start, "");
	3927	LLVMValueRef trace_field_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
	3928	trace_field_index_start + 2, "");
	3929	LLVMValueRef addrs_field_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
	3930	trace_field_index_start + 3, "");
	3931
	3932	LLVMBuildStore(g->builder, trace_field_ptr, callee_trace_ptr_ptr);
	3933
	3934	gen_init_stack_trace(g, trace_field_ptr, addrs_field_ptr);
	3935
	3936	gen_param_values.append(get_cur_err_ret_trace_val(g, instruction->base.scope));
	3937	}
	3938	}
	3939	} else {
	3940	if (first_arg_ret) {
	3941	gen_param_values.append(result_loc);
	3942	}
	3943	if (prefix_arg_err_ret_stack) {
	3944	gen_param_values.append(get_cur_err_ret_trace_val(g, instruction->base.scope));
	3945	}
3661	3946	}
3662	3947	FnWalk fn_walk = {};
3663	3948	fn_walk.id = FnWalkIdCall;
3664	3949	fn_walk.data.call.inst = instruction;
3665	3950	fn_walk.data.call.is_var_args = is_var_args;
3666	3951	fn_walk.data.call.gen_param_values = &gen_param_values;
	3952	fn_walk.data.call.gen_param_types = &gen_param_types;
3667	3953	walk_function_params(g, fn_type, &fn_walk);
3668	3954
3669	3955	ZigLLVM_FnInline fn_inline;
...	...	@@ -3679,12 +3965,71 @@ static LLVMValueRef ir_render_call(CodeGen g, IrExecutable executable, IrInstr
3679	3965	break;
3680	3966	}
3681	3967
3682		LLVMCallConv llvm_cc = get_llvm_cc(g, cc);
3683		LLVMValueRef result;
	3968	LLVMCallConv llvm_cc = get_llvm_cc(g, cc);
	3969	LLVMValueRef result;
	3970
	3971	if (callee_is_async) {
	3972	uint32_t arg_start_i = frame_index_arg(g, fn_type->data.fn.fn_type_id.return_type);
	3973
	3974	LLVMValueRef casted_frame;
	3975	if (instruction->new_stack != nullptr) {
	3976	// We need the frame type to be a pointer to a struct that includes the args
	3977	size_t field_count = arg_start_i + gen_param_values.length;
	3978	LLVMTypeRef *field_types = allocate_nonzero<LLVMTypeRef>(field_count);
	3979	LLVMGetStructElementTypes(LLVMGetElementType(LLVMTypeOf(frame_result_loc)), field_types);
	3980	assert(LLVMCountStructElementTypes(LLVMGetElementType(LLVMTypeOf(frame_result_loc))) == arg_start_i);
	3981	for (size_t arg_i = 0; arg_i < gen_param_values.length; arg_i += 1) {
	3982	field_types[arg_start_i + arg_i] = LLVMTypeOf(gen_param_values.at(arg_i));
	3983	}
	3984	LLVMTypeRef frame_with_args_type = LLVMStructType(field_types, field_count, false);
	3985	LLVMTypeRef ptr_frame_with_args_type = LLVMPointerType(frame_with_args_type, 0);
	3986
	3987	casted_frame = LLVMBuildBitCast(g->builder, frame_result_loc, ptr_frame_with_args_type, "");
	3988	} else {
	3989	casted_frame = frame_result_loc;
	3990	}
	3991
	3992	for (size_t arg_i = 0; arg_i < gen_param_values.length; arg_i += 1) {
	3993	LLVMValueRef arg_ptr = LLVMBuildStructGEP(g->builder, casted_frame, arg_start_i + arg_i, "");
	3994	gen_assign_raw(g, arg_ptr, get_pointer_to_type(g, gen_param_types.at(arg_i), true),
	3995	gen_param_values.at(arg_i));
	3996	}
	3997
	3998	if (instruction->is_async) {
	3999	gen_resume(g, fn_val, frame_result_loc, ResumeIdCall);
	4000	if (instruction->new_stack != nullptr) {
	4001	return frame_result_loc;
	4002	}
	4003	return nullptr;
	4004	} else {
	4005	ZigType *ptr_result_type = get_pointer_to_type(g, src_return_type, true);
	4006
	4007	LLVMBasicBlockRef call_bb = gen_suspend_begin(g, "CallResume");
	4008
	4009	LLVMValueRef call_inst = gen_resume(g, fn_val, frame_result_loc, ResumeIdCall);
	4010	set_tail_call_if_appropriate(g, call_inst);
	4011	LLVMBuildRetVoid(g->builder);
	4012
	4013	LLVMPositionBuilderAtEnd(g->builder, call_bb);
	4014	gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedAnAwaitingFn, nullptr);
	4015	render_async_var_decls(g, instruction->base.scope);
	4016
	4017	if (!type_has_bits(src_return_type))
	4018	return nullptr;
	4019
	4020	if (result_loc != nullptr)
	4021	return get_handle_value(g, result_loc, src_return_type, ptr_result_type);
	4022
	4023	LLVMValueRef result_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
	4024	return LLVMBuildLoad(g->builder, result_ptr, "");
	4025	}
	4026	}
3684	4027
3685	4028	if (instruction->new_stack == nullptr) {
3686	4029	result = ZigLLVMBuildCall(g->builder, fn_val,
3687	4030	gen_param_values.items, (unsigned)gen_param_values.length, llvm_cc, fn_inline, "");
	4031	} else if (instruction->is_async) {
	4032	zig_panic("TODO @asyncCall of non-async function");
3688	4033	} else {
3689	4034	LLVMValueRef stacksave_fn_val = get_stacksave_fn_val(g);
3690	4035	LLVMValueRef stackrestore_fn_val = get_stackrestore_fn_val(g);
...	...	@@ -3697,13 +4042,6 @@ static LLVMValueRef ir_render_call(CodeGen g, IrExecutable executable, IrInstr
3697	4042	LLVMBuildCall(g->builder, stackrestore_fn_val, &old_stack_ref, 1, "");
3698	4043	}
3699	4044
3700
3701		if (instruction->is_async) {
3702		LLVMValueRef payload_ptr = LLVMBuildStructGEP(g->builder, result_loc, err_union_payload_index, "");
3703		LLVMBuildStore(g->builder, result, payload_ptr);
3704		return result_loc;
3705		}
3706
3707	4045	if (src_return_type->id == ZigTypeIdUnreachable) {
3708	4046	return LLVMBuildUnreachable(g->builder);
3709	4047	} else if (!ret_has_bits) {
...	...	@@ -4200,7 +4538,7 @@ static LLVMValueRef get_enum_tag_name_function(CodeGen g, ZigType enum_type) {
4200	4538	LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
4201	4539	addLLVMFnAttr(fn_val, "nounwind");
4202	4540	add_uwtable_attr(g, fn_val);
4203		if (g->build_mode == BuildModeDebug) {
	4541	if (codegen_have_frame_pointer(g)) {
4204	4542	ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
4205	4543	ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
4206	4544	}
...	...	@@ -4347,10 +4685,6 @@ static LLVMValueRef ir_render_align_cast(CodeGen g, IrExecutable executable, I
4347	4685	{
4348	4686	align_bytes = target_type->data.maybe.child_type->data.fn.fn_type_id.alignment;
4349	4687	ptr_val = target_val;
4350		} else if (target_type->id == ZigTypeIdOptional &&
4351		target_type->data.maybe.child_type->id == ZigTypeIdPromise)
4352		{
4353		zig_panic("TODO audit this function");
4354	4688	} else if (target_type->id == ZigTypeIdStruct && target_type->data.structure.is_slice) {
4355	4689	ZigType *slice_ptr_type = target_type->data.structure.fields[slice_ptr_index].type_entry;
4356	4690	align_bytes = get_ptr_align(g, slice_ptr_type);
...	...	@@ -4388,26 +4722,11 @@ static LLVMValueRef ir_render_error_return_trace(CodeGen g, IrExecutable execu
4388	4722	{
4389	4723	LLVMValueRef cur_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope);
4390	4724	if (cur_err_ret_trace_val == nullptr) {
4391		ZigType *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g);
4392		return LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type));
	4725	return LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g)));
4393	4726	}
4394	4727	return cur_err_ret_trace_val;
4395	4728	}
4396	4729
4397		static LLVMValueRef ir_render_cancel(CodeGen g, IrExecutable executable, IrInstructionCancel *instruction) {
4398		LLVMValueRef target_handle = ir_llvm_value(g, instruction->target);
4399		LLVMBuildCall(g->builder, get_coro_destroy_fn_val(g), &target_handle, 1, "");
4400		return nullptr;
4401		}
4402
4403		static LLVMValueRef ir_render_get_implicit_allocator(CodeGen g, IrExecutable executable,
4404		IrInstructionGetImplicitAllocator *instruction)
4405		{
4406		assert(instruction->id == ImplicitAllocatorIdArg);
4407		size_t allocator_arg_index = get_async_allocator_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id);
4408		return LLVMGetParam(g->cur_fn_val, allocator_arg_index);
4409		}
4410
4411	4730	static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) {
4412	4731	switch (atomic_order) {
4413	4732	case AtomicOrderUnordered: return LLVMAtomicOrderingUnordered;
...	...	@@ -4722,24 +5041,8 @@ static LLVMValueRef ir_render_frame_address(CodeGen g, IrExecutable executable
4722	5041	return LLVMBuildPtrToInt(g->builder, ptr_val, g->builtin_types.entry_usize->llvm_type, "");
4723	5042	}
4724	5043
4725		static LLVMValueRef get_handle_fn_val(CodeGen *g) {
4726		if (g->coro_frame_fn_val)
4727		return g->coro_frame_fn_val;
4728
4729		LLVMTypeRef fn_type = LLVMFunctionType( LLVMPointerType(LLVMInt8Type(), 0)
4730		, nullptr, 0, false);
4731		Buf *name = buf_sprintf("llvm.coro.frame");
4732		g->coro_frame_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
4733		assert(LLVMGetIntrinsicID(g->coro_frame_fn_val));
4734
4735		return g->coro_frame_fn_val;
4736		}
4737
4738		static LLVMValueRef ir_render_handle(CodeGen g, IrExecutable executable,
4739		IrInstructionHandle *instruction)
4740		{
4741		LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, g->builtin_types.entry_promise));
4742		return LLVMBuildCall(g->builder, get_handle_fn_val(g), &zero, 0, "");
	5044	static LLVMValueRef ir_render_handle(CodeGen g, IrExecutable executable, IrInstructionFrameHandle *instruction) {
	5045	return g->cur_frame_ptr;
4743	5046	}
4744	5047
4745	5048	static LLVMValueRef render_shl_with_overflow(CodeGen g, IrInstructionOverflowOp instruction) {
...	...	@@ -5005,248 +5308,6 @@ static LLVMValueRef ir_render_panic(CodeGen g, IrExecutable executable, IrInst
5005	5308	return nullptr;
5006	5309	}
5007	5310
5008		static LLVMValueRef ir_render_coro_id(CodeGen g, IrExecutable executable, IrInstructionCoroId *instruction) {
5009		LLVMValueRef promise_ptr = ir_llvm_value(g, instruction->promise_ptr);
5010		LLVMValueRef align_val = LLVMConstInt(LLVMInt32Type(), get_coro_frame_align_bytes(g), false);
5011		LLVMValueRef null = LLVMConstIntToPtr(LLVMConstNull(g->builtin_types.entry_usize->llvm_type),
5012		LLVMPointerType(LLVMInt8Type(), 0));
5013		LLVMValueRef params[] = {
5014		align_val,
5015		promise_ptr,
5016		null,
5017		null,
5018		};
5019		return LLVMBuildCall(g->builder, get_coro_id_fn_val(g), params, 4, "");
5020		}
5021
5022		static LLVMValueRef ir_render_coro_alloc(CodeGen g, IrExecutable executable, IrInstructionCoroAlloc *instruction) {
5023		LLVMValueRef token = ir_llvm_value(g, instruction->coro_id);
5024		return LLVMBuildCall(g->builder, get_coro_alloc_fn_val(g), &token, 1, "");
5025		}
5026
5027		static LLVMValueRef ir_render_coro_size(CodeGen g, IrExecutable executable, IrInstructionCoroSize *instruction) {
5028		return LLVMBuildCall(g->builder, get_coro_size_fn_val(g), nullptr, 0, "");
5029		}
5030
5031		static LLVMValueRef ir_render_coro_begin(CodeGen g, IrExecutable executable, IrInstructionCoroBegin *instruction) {
5032		LLVMValueRef coro_id = ir_llvm_value(g, instruction->coro_id);
5033		LLVMValueRef coro_mem_ptr = ir_llvm_value(g, instruction->coro_mem_ptr);
5034		LLVMValueRef params[] = {
5035		coro_id,
5036		coro_mem_ptr,
5037		};
5038		return LLVMBuildCall(g->builder, get_coro_begin_fn_val(g), params, 2, "");
5039		}
5040
5041		static LLVMValueRef ir_render_coro_alloc_fail(CodeGen g, IrExecutable executable,
5042		IrInstructionCoroAllocFail *instruction)
5043		{
5044		size_t err_code_ptr_arg_index = get_async_err_code_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id);
5045		LLVMValueRef err_code_ptr_val = LLVMGetParam(g->cur_fn_val, err_code_ptr_arg_index);
5046		LLVMValueRef err_code = ir_llvm_value(g, instruction->err_val);
5047		LLVMBuildStore(g->builder, err_code, err_code_ptr_val);
5048
5049		LLVMValueRef return_value;
5050		if (ir_want_runtime_safety(g, &instruction->base)) {
5051		return_value = LLVMConstNull(LLVMPointerType(LLVMInt8Type(), 0));
5052		} else {
5053		return_value = LLVMGetUndef(LLVMPointerType(LLVMInt8Type(), 0));
5054		}
5055		LLVMBuildRet(g->builder, return_value);
5056		return nullptr;
5057		}
5058
5059		static LLVMValueRef ir_render_coro_suspend(CodeGen g, IrExecutable executable, IrInstructionCoroSuspend *instruction) {
5060		LLVMValueRef save_point;
5061		if (instruction->save_point == nullptr) {
5062		save_point = LLVMConstNull(ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()));
5063		} else {
5064		save_point = ir_llvm_value(g, instruction->save_point);
5065		}
5066		LLVMValueRef is_final = ir_llvm_value(g, instruction->is_final);
5067		LLVMValueRef params[] = {
5068		save_point,
5069		is_final,
5070		};
5071		return LLVMBuildCall(g->builder, get_coro_suspend_fn_val(g), params, 2, "");
5072		}
5073
5074		static LLVMValueRef ir_render_coro_end(CodeGen g, IrExecutable executable, IrInstructionCoroEnd *instruction) {
5075		LLVMValueRef params[] = {
5076		LLVMConstNull(LLVMPointerType(LLVMInt8Type(), 0)),
5077		LLVMConstNull(LLVMInt1Type()),
5078		};
5079		return LLVMBuildCall(g->builder, get_coro_end_fn_val(g), params, 2, "");
5080		}
5081
5082		static LLVMValueRef ir_render_coro_free(CodeGen g, IrExecutable executable, IrInstructionCoroFree *instruction) {
5083		LLVMValueRef coro_id = ir_llvm_value(g, instruction->coro_id);
5084		LLVMValueRef coro_handle = ir_llvm_value(g, instruction->coro_handle);
5085		LLVMValueRef params[] = {
5086		coro_id,
5087		coro_handle,
5088		};
5089		return LLVMBuildCall(g->builder, get_coro_free_fn_val(g), params, 2, "");
5090		}
5091
5092		static LLVMValueRef ir_render_coro_resume(CodeGen g, IrExecutable executable, IrInstructionCoroResume *instruction) {
5093		LLVMValueRef awaiter_handle = ir_llvm_value(g, instruction->awaiter_handle);
5094		return LLVMBuildCall(g->builder, get_coro_resume_fn_val(g), &awaiter_handle, 1, "");
5095		}
5096
5097		static LLVMValueRef ir_render_coro_save(CodeGen g, IrExecutable executable, IrInstructionCoroSave *instruction) {
5098		LLVMValueRef coro_handle = ir_llvm_value(g, instruction->coro_handle);
5099		return LLVMBuildCall(g->builder, get_coro_save_fn_val(g), &coro_handle, 1, "");
5100		}
5101
5102		static LLVMValueRef ir_render_coro_promise(CodeGen g, IrExecutable executable, IrInstructionCoroPromise *instruction) {
5103		LLVMValueRef coro_handle = ir_llvm_value(g, instruction->coro_handle);
5104		LLVMValueRef params[] = {
5105		coro_handle,
5106		LLVMConstInt(LLVMInt32Type(), get_coro_frame_align_bytes(g), false),
5107		LLVMConstNull(LLVMInt1Type()),
5108		};
5109		LLVMValueRef uncasted_result = LLVMBuildCall(g->builder, get_coro_promise_fn_val(g), params, 3, "");
5110		return LLVMBuildBitCast(g->builder, uncasted_result, get_llvm_type(g, instruction->base.value.type), "");
5111		}
5112
5113		static LLVMValueRef get_coro_alloc_helper_fn_val(CodeGen g, LLVMTypeRef alloc_fn_type_ref, ZigType fn_type) {
5114		if (g->coro_alloc_helper_fn_val != nullptr)
5115		return g->coro_alloc_helper_fn_val;
5116
5117		assert(fn_type->id == ZigTypeIdFn);
5118
5119		ZigType *ptr_to_err_code_type = get_pointer_to_type(g, g->builtin_types.entry_global_error_set, false);
5120
5121		LLVMTypeRef alloc_raw_fn_type_ref = LLVMGetElementType(alloc_fn_type_ref);
5122		LLVMTypeRef *alloc_fn_arg_types = allocate<LLVMTypeRef>(LLVMCountParamTypes(alloc_raw_fn_type_ref));
5123		LLVMGetParamTypes(alloc_raw_fn_type_ref, alloc_fn_arg_types);
5124
5125		ZigList<LLVMTypeRef> arg_types = {};
5126		arg_types.append(alloc_fn_type_ref);
5127		if (g->have_err_ret_tracing) {
5128		arg_types.append(alloc_fn_arg_types[1]);
5129		}
5130		arg_types.append(alloc_fn_arg_types[g->have_err_ret_tracing ? 2 : 1]);
5131		arg_types.append(get_llvm_type(g, ptr_to_err_code_type));
5132		arg_types.append(g->builtin_types.entry_usize->llvm_type);
5133
5134		LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0),
5135		arg_types.items, arg_types.length, false);
5136
5137		Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_coro_alloc_helper"), false);
5138		LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref);
5139		LLVMSetLinkage(fn_val, LLVMInternalLinkage);
5140		LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
5141		addLLVMFnAttr(fn_val, "nounwind");
5142		addLLVMArgAttr(fn_val, (unsigned)0, "nonnull");
5143		addLLVMArgAttr(fn_val, (unsigned)1, "nonnull");
5144
5145		LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
5146		LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
5147		ZigFn *prev_cur_fn = g->cur_fn;
5148		LLVMValueRef prev_cur_fn_val = g->cur_fn_val;
5149
5150		LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
5151		LLVMPositionBuilderAtEnd(g->builder, entry_block);
5152		ZigLLVMClearCurrentDebugLocation(g->builder);
5153		g->cur_fn = nullptr;
5154		g->cur_fn_val = fn_val;
5155
5156		LLVMValueRef sret_ptr = LLVMBuildAlloca(g->builder, LLVMGetElementType(alloc_fn_arg_types[0]), "");
5157
5158		size_t next_arg = 0;
5159		LLVMValueRef realloc_fn_val = LLVMGetParam(fn_val, next_arg);
5160		next_arg += 1;
5161
5162		LLVMValueRef stack_trace_val;
5163		if (g->have_err_ret_tracing) {
5164		stack_trace_val = LLVMGetParam(fn_val, next_arg);
5165		next_arg += 1;
5166		}
5167
5168		LLVMValueRef allocator_val = LLVMGetParam(fn_val, next_arg);
5169		next_arg += 1;
5170		LLVMValueRef err_code_ptr = LLVMGetParam(fn_val, next_arg);
5171		next_arg += 1;
5172		LLVMValueRef coro_size = LLVMGetParam(fn_val, next_arg);
5173		next_arg += 1;
5174		LLVMValueRef alignment_val = LLVMConstInt(g->builtin_types.entry_u29->llvm_type,
5175		get_coro_frame_align_bytes(g), false);
5176
5177		ConstExprValue *zero_array = create_const_str_lit(g, buf_create_from_str(""));
5178		ConstExprValue *undef_slice_zero = create_const_slice(g, zero_array, 0, 0, false);
5179		render_const_val(g, undef_slice_zero, "");
5180		render_const_val_global(g, undef_slice_zero, "");
5181
5182		ZigList<LLVMValueRef> args = {};
5183		args.append(sret_ptr);
5184		if (g->have_err_ret_tracing) {
5185		args.append(stack_trace_val);
5186		}
5187		args.append(allocator_val);
5188		args.append(undef_slice_zero->global_refs->llvm_global);
5189		args.append(LLVMGetUndef(g->builtin_types.entry_u29->llvm_type));
5190		args.append(coro_size);
5191		args.append(alignment_val);
5192		LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, realloc_fn_val, args.items, args.length,
5193		get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
5194		set_call_instr_sret(g, call_instruction);
5195		LLVMValueRef err_val_ptr = LLVMBuildStructGEP(g->builder, sret_ptr, err_union_err_index, "");
5196		LLVMValueRef err_val = LLVMBuildLoad(g->builder, err_val_ptr, "");
5197		LLVMBuildStore(g->builder, err_val, err_code_ptr);
5198		LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, err_val, LLVMConstNull(LLVMTypeOf(err_val)), "");
5199		LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(fn_val, "AllocOk");
5200		LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(fn_val, "AllocFail");
5201		LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
5202
5203		LLVMPositionBuilderAtEnd(g->builder, ok_block);
5204		LLVMValueRef payload_ptr = LLVMBuildStructGEP(g->builder, sret_ptr, err_union_payload_index, "");
5205		ZigType *u8_ptr_type = get_pointer_to_type_extra(g, g->builtin_types.entry_u8, false, false,
5206		PtrLenUnknown, get_abi_alignment(g, g->builtin_types.entry_u8), 0, 0, false);
5207		ZigType *slice_type = get_slice_type(g, u8_ptr_type);
5208		size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index;
5209		LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, payload_ptr, ptr_field_index, "");
5210		LLVMValueRef ptr_val = LLVMBuildLoad(g->builder, ptr_field_ptr, "");
5211		LLVMBuildRet(g->builder, ptr_val);
5212
5213		LLVMPositionBuilderAtEnd(g->builder, fail_block);
5214		LLVMBuildRet(g->builder, LLVMConstNull(LLVMPointerType(LLVMInt8Type(), 0)));
5215
5216		g->cur_fn = prev_cur_fn;
5217		g->cur_fn_val = prev_cur_fn_val;
5218		LLVMPositionBuilderAtEnd(g->builder, prev_block);
5219		if (!g->strip_debug_symbols) {
5220		LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
5221		}
5222
5223		g->coro_alloc_helper_fn_val = fn_val;
5224		return fn_val;
5225		}
5226
5227		static LLVMValueRef ir_render_coro_alloc_helper(CodeGen g, IrExecutable executable,
5228		IrInstructionCoroAllocHelper *instruction)
5229		{
5230		LLVMValueRef realloc_fn = ir_llvm_value(g, instruction->realloc_fn);
5231		LLVMValueRef coro_size = ir_llvm_value(g, instruction->coro_size);
5232		LLVMValueRef fn_val = get_coro_alloc_helper_fn_val(g, LLVMTypeOf(realloc_fn), instruction->realloc_fn->value.type);
5233		size_t err_code_ptr_arg_index = get_async_err_code_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id);
5234		size_t allocator_arg_index = get_async_allocator_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id);
5235
5236		ZigList<LLVMValueRef> params = {};
5237		params.append(realloc_fn);
5238		uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, g->cur_fn);
5239		if (err_ret_trace_arg_index != UINT32_MAX) {
5240		params.append(LLVMGetParam(g->cur_fn_val, err_ret_trace_arg_index));
5241		}
5242		params.append(LLVMGetParam(g->cur_fn_val, allocator_arg_index));
5243		params.append(LLVMGetParam(g->cur_fn_val, err_code_ptr_arg_index));
5244		params.append(coro_size);
5245
5246		return ZigLLVMBuildCall(g->builder, fn_val, params.items, params.length,
5247		get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
5248		}
5249
5250	5311	static LLVMValueRef ir_render_atomic_rmw(CodeGen g, IrExecutable executable,
5251	5312	IrInstructionAtomicRmw *instruction)
5252	5313	{
...	...	@@ -5263,14 +5324,15 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen g, IrExecutable executable,
5263	5324	LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
5264	5325
5265	5326	if (get_codegen_ptr_type(operand_type) == nullptr) {
5266		return LLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, false);
	5327	return LLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, g->is_single_threaded);
5267	5328	}
5268	5329
5269	5330	// it's a pointer but we need to treat it as an int
5270	5331	LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr,
5271	5332	LLVMPointerType(g->builtin_types.entry_usize->llvm_type, 0), "");
5272	5333	LLVMValueRef casted_operand = LLVMBuildPtrToInt(g->builder, operand, g->builtin_types.entry_usize->llvm_type, "");
5273		LLVMValueRef uncasted_result = LLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering, false);
	5334	LLVMValueRef uncasted_result = LLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering,
	5335	g->is_single_threaded);
5274	5336	return LLVMBuildIntToPtr(g->builder, uncasted_result, get_llvm_type(g, operand_type), "");
5275	5337	}
5276	5338
...	...	@@ -5284,27 +5346,6 @@ static LLVMValueRef ir_render_atomic_load(CodeGen g, IrExecutable executable,
5284	5346	return load_inst;
5285	5347	}
5286	5348
5287		static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen g, IrExecutable executable,
5288		IrInstructionMergeErrRetTraces *instruction)
5289		{
5290		assert(g->have_err_ret_tracing);
5291
5292		LLVMValueRef src_trace_ptr = ir_llvm_value(g, instruction->src_err_ret_trace_ptr);
5293		LLVMValueRef dest_trace_ptr = ir_llvm_value(g, instruction->dest_err_ret_trace_ptr);
5294
5295		LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr };
5296		ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
5297		return nullptr;
5298		}
5299
5300		static LLVMValueRef ir_render_mark_err_ret_trace_ptr(CodeGen g, IrExecutable executable,
5301		IrInstructionMarkErrRetTracePtr *instruction)
5302		{
5303		assert(g->have_err_ret_tracing);
5304		g->cur_err_ret_trace_val_stack = ir_llvm_value(g, instruction->err_ret_trace_ptr);
5305		return nullptr;
5306		}
5307
5308	5349	static LLVMValueRef ir_render_float_op(CodeGen g, IrExecutable executable, IrInstructionFloatOp *instruction) {
5309	5350	LLVMValueRef op = ir_llvm_value(g, instruction->op1);
5310	5351	assert(instruction->base.value.type->id == ZigTypeIdFloat);
...	...	@@ -5424,6 +5465,174 @@ static LLVMValueRef ir_render_assert_non_null(CodeGen g, IrExecutable executab
5424	5465	return nullptr;
5425	5466	}
5426	5467
	5468	static LLVMValueRef ir_render_suspend_begin(CodeGen g, IrExecutable executable,
	5469	IrInstructionSuspendBegin *instruction)
	5470	{
	5471	instruction->resume_bb = gen_suspend_begin(g, "SuspendResume");
	5472	return nullptr;
	5473	}
	5474
	5475	static LLVMValueRef ir_render_suspend_finish(CodeGen g, IrExecutable executable,
	5476	IrInstructionSuspendFinish *instruction)
	5477	{
	5478	LLVMBuildRetVoid(g->builder);
	5479
	5480	LLVMPositionBuilderAtEnd(g->builder, instruction->begin->resume_bb);
	5481	render_async_var_decls(g, instruction->base.scope);
	5482	return nullptr;
	5483	}
	5484
	5485	static LLVMValueRef ir_render_await(CodeGen g, IrExecutable executable, IrInstructionAwaitGen *instruction) {
	5486	LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
	5487	LLVMValueRef zero = LLVMConstNull(usize_type_ref);
	5488	LLVMValueRef target_frame_ptr = ir_llvm_value(g, instruction->frame);
	5489	ZigType *result_type = instruction->base.value.type;
	5490	ZigType *ptr_result_type = get_pointer_to_type(g, result_type, true);
	5491
	5492	// Prepare to be suspended
	5493	LLVMBasicBlockRef resume_bb = gen_suspend_begin(g, "AwaitResume");
	5494	LLVMBasicBlockRef end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "AwaitEnd");
	5495
	5496	// At this point resuming the function will continue from resume_bb.
	5497	// This code is as if it is running inside the suspend block.
	5498
	5499	// supply the awaiter return pointer
	5500	LLVMValueRef result_loc = (instruction->result_loc == nullptr) ?
	5501	nullptr : ir_llvm_value(g, instruction->result_loc);
	5502	if (type_has_bits(result_type)) {
	5503	LLVMValueRef awaiter_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_ret_start + 1, "");
	5504	if (result_loc == nullptr) {
	5505	// no copy needed
	5506	LLVMBuildStore(g->builder, LLVMConstNull(LLVMGetElementType(LLVMTypeOf(awaiter_ret_ptr_ptr))),
	5507	awaiter_ret_ptr_ptr);
	5508	} else {
	5509	LLVMBuildStore(g->builder, result_loc, awaiter_ret_ptr_ptr);
	5510	}
	5511	}
	5512
	5513	// supply the error return trace pointer
	5514	if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) {
	5515	LLVMValueRef my_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope);
	5516	assert(my_err_ret_trace_val != nullptr);
	5517	LLVMValueRef err_ret_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr,
	5518	frame_index_trace_arg(g, result_type) + 1, "");
	5519	LLVMBuildStore(g->builder, my_err_ret_trace_val, err_ret_trace_ptr_ptr);
	5520	}
	5521
	5522	// caller's own frame pointer
	5523	LLVMValueRef awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_frame_ptr, usize_type_ref, "");
	5524	LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_awaiter_index, "");
	5525	LLVMValueRef prev_val = gen_maybe_atomic_op(g, LLVMAtomicRMWBinOpXchg, awaiter_ptr, awaiter_init_val,
	5526	LLVMAtomicOrderingRelease);
	5527
	5528	LLVMBasicBlockRef bad_await_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadAwait");
	5529	LLVMBasicBlockRef complete_suspend_block = LLVMAppendBasicBlock(g->cur_fn_val, "CompleteSuspend");
	5530	LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");
	5531
	5532	LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
	5533	LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, bad_await_block, 2);
	5534
	5535	LLVMAddCase(switch_instr, zero, complete_suspend_block);
	5536	LLVMAddCase(switch_instr, all_ones, early_return_block);
	5537
	5538	// We discovered that another awaiter was already here.
	5539	LLVMPositionBuilderAtEnd(g->builder, bad_await_block);
	5540	gen_assertion(g, PanicMsgIdBadAwait, &instruction->base);
	5541
	5542	// Rely on the target to resume us from suspension.
	5543	LLVMPositionBuilderAtEnd(g->builder, complete_suspend_block);
	5544	LLVMBuildRetVoid(g->builder);
	5545
	5546	// Early return: The async function has already completed. We must copy the result and
	5547	// the error return trace if applicable.
	5548	LLVMPositionBuilderAtEnd(g->builder, early_return_block);
	5549	if (type_has_bits(result_type) && result_loc != nullptr) {
	5550	LLVMValueRef their_result_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_ret_start, "");
	5551	LLVMValueRef their_result_ptr = LLVMBuildLoad(g->builder, their_result_ptr_ptr, "");
	5552	LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
	5553	LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, result_loc, ptr_u8, "");
	5554	LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, their_result_ptr, ptr_u8, "");
	5555	bool is_volatile = false;
	5556	uint32_t abi_align = get_abi_alignment(g, result_type);
	5557	LLVMValueRef byte_count_val = LLVMConstInt(usize_type_ref, type_size(g, result_type), false);
	5558	ZigLLVMBuildMemCpy(g->builder,
	5559	dest_ptr_casted, abi_align,
	5560	src_ptr_casted, abi_align, byte_count_val, is_volatile);
	5561	}
	5562	if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) {
	5563	LLVMValueRef their_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr,
	5564	frame_index_trace_arg(g, result_type), "");
	5565	LLVMValueRef src_trace_ptr = LLVMBuildLoad(g->builder, their_trace_ptr_ptr, "");
	5566	LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, instruction->base.scope);
	5567	LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr };
	5568	ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2,
	5569	get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
	5570	}
	5571	LLVMBuildBr(g->builder, end_bb);
	5572
	5573	LLVMPositionBuilderAtEnd(g->builder, resume_bb);
	5574	gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedAnAwaitingFn, nullptr);
	5575	LLVMBuildBr(g->builder, end_bb);
	5576
	5577	LLVMPositionBuilderAtEnd(g->builder, end_bb);
	5578	if (type_has_bits(result_type) && result_loc != nullptr) {
	5579	return get_handle_value(g, result_loc, result_type, ptr_result_type);
	5580	}
	5581	return nullptr;
	5582	}
	5583
	5584	static LLVMValueRef ir_render_resume(CodeGen g, IrExecutable executable, IrInstructionResume *instruction) {
	5585	LLVMValueRef frame = ir_llvm_value(g, instruction->frame);
	5586	ZigType *frame_type = instruction->frame->value.type;
	5587	assert(frame_type->id == ZigTypeIdAnyFrame);
	5588
	5589	gen_resume(g, nullptr, frame, ResumeIdManual);
	5590	return nullptr;
	5591	}
	5592
	5593	static LLVMValueRef ir_render_frame_size(CodeGen g, IrExecutable executable,
	5594	IrInstructionFrameSizeGen *instruction)
	5595	{
	5596	LLVMValueRef fn_val = ir_llvm_value(g, instruction->fn);
	5597	return gen_frame_size(g, fn_val);
	5598	}
	5599
	5600	static LLVMValueRef ir_render_spill_begin(CodeGen g, IrExecutable executable,
	5601	IrInstructionSpillBegin *instruction)
	5602	{
	5603	if (!fn_is_async(g->cur_fn))
	5604	return nullptr;
	5605
	5606	switch (instruction->spill_id) {
	5607	case SpillIdInvalid:
	5608	zig_unreachable();
	5609	case SpillIdRetErrCode: {
	5610	LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
	5611	LLVMValueRef ptr = ir_llvm_value(g, g->cur_fn->err_code_spill);
	5612	LLVMBuildStore(g->builder, operand, ptr);
	5613	return nullptr;
	5614	}
	5615
	5616	}
	5617	zig_unreachable();
	5618	}
	5619
	5620	static LLVMValueRef ir_render_spill_end(CodeGen g, IrExecutable executable, IrInstructionSpillEnd *instruction) {
	5621	if (!fn_is_async(g->cur_fn))
	5622	return ir_llvm_value(g, instruction->begin->operand);
	5623
	5624	switch (instruction->begin->spill_id) {
	5625	case SpillIdInvalid:
	5626	zig_unreachable();
	5627	case SpillIdRetErrCode: {
	5628	LLVMValueRef ptr = ir_llvm_value(g, g->cur_fn->err_code_spill);
	5629	return LLVMBuildLoad(g->builder, ptr, "");
	5630	}
	5631
	5632	}
	5633	zig_unreachable();
	5634	}
	5635
5427	5636	static void set_debug_location(CodeGen g, IrInstruction instruction) {
5428	5637	AstNode *source_node = instruction->source_node;
5429	5638	Scope *scope = instruction->scope;
...	...	@@ -5445,7 +5654,7 @@ static LLVMValueRef ir_render_instruction(CodeGen g, IrExecutable executable,
5445	5654	case IrInstructionIdSetRuntimeSafety:
5446	5655	case IrInstructionIdSetFloatMode:
5447	5656	case IrInstructionIdArrayType:
5448		case IrInstructionIdPromiseType:
	5657	case IrInstructionIdAnyFrameType:
5449	5658	case IrInstructionIdSliceType:
5450	5659	case IrInstructionIdSizeOf:
5451	5660	case IrInstructionIdSwitchTarget:
...	...	@@ -5485,8 +5694,6 @@ static LLVMValueRef ir_render_instruction(CodeGen g, IrExecutable executable,
5485	5694	case IrInstructionIdTagType:
5486	5695	case IrInstructionIdExport:
5487	5696	case IrInstructionIdErrorUnion:
5488		case IrInstructionIdPromiseResultType:
5489		case IrInstructionIdAwaitBookkeeping:
5490	5697	case IrInstructionIdAddImplicitReturnType:
5491	5698	case IrInstructionIdIntCast:
5492	5699	case IrInstructionIdFloatCast:
...	...	@@ -5508,17 +5715,19 @@ static LLVMValueRef ir_render_instruction(CodeGen g, IrExecutable executable,
5508	5715	case IrInstructionIdCallSrc:
5509	5716	case IrInstructionIdAllocaSrc:
5510	5717	case IrInstructionIdEndExpr:
5511		case IrInstructionIdAllocaGen:
5512	5718	case IrInstructionIdImplicitCast:
5513	5719	case IrInstructionIdResolveResult:
5514	5720	case IrInstructionIdResetResult:
5515		case IrInstructionIdResultPtr:
5516	5721	case IrInstructionIdContainerInitList:
5517	5722	case IrInstructionIdSliceSrc:
5518	5723	case IrInstructionIdRef:
5519	5724	case IrInstructionIdBitCastSrc:
5520	5725	case IrInstructionIdTestErrSrc:
5521	5726	case IrInstructionIdUnionInitNamedField:
	5727	case IrInstructionIdFrameType:
	5728	case IrInstructionIdFrameSizeSrc:
	5729	case IrInstructionIdAllocaGen:
	5730	case IrInstructionIdAwaitSrc:
5522	5731	zig_unreachable();
5523	5732
5524	5733	case IrInstructionIdDeclVarGen:
...	...	@@ -5597,8 +5806,8 @@ static LLVMValueRef ir_render_instruction(CodeGen g, IrExecutable executable,
5597	5806	return ir_render_return_address(g, executable, (IrInstructionReturnAddress *)instruction);
5598	5807	case IrInstructionIdFrameAddress:
5599	5808	return ir_render_frame_address(g, executable, (IrInstructionFrameAddress *)instruction);
5600		case IrInstructionIdHandle:
5601		return ir_render_handle(g, executable, (IrInstructionHandle *)instruction);
	5809	case IrInstructionIdFrameHandle:
	5810	return ir_render_handle(g, executable, (IrInstructionFrameHandle *)instruction);
5602	5811	case IrInstructionIdOverflowOp:
5603	5812	return ir_render_overflow_op(g, executable, (IrInstructionOverflowOp *)instruction);
5604	5813	case IrInstructionIdTestErrGen:
...	...	@@ -5641,44 +5850,12 @@ static LLVMValueRef ir_render_instruction(CodeGen g, IrExecutable executable,
5641	5850	return ir_render_align_cast(g, executable, (IrInstructionAlignCast *)instruction);
5642	5851	case IrInstructionIdErrorReturnTrace:
5643	5852	return ir_render_error_return_trace(g, executable, (IrInstructionErrorReturnTrace *)instruction);
5644		case IrInstructionIdCancel:
5645		return ir_render_cancel(g, executable, (IrInstructionCancel *)instruction);
5646		case IrInstructionIdGetImplicitAllocator:
5647		return ir_render_get_implicit_allocator(g, executable, (IrInstructionGetImplicitAllocator *)instruction);
5648		case IrInstructionIdCoroId:
5649		return ir_render_coro_id(g, executable, (IrInstructionCoroId *)instruction);
5650		case IrInstructionIdCoroAlloc:
5651		return ir_render_coro_alloc(g, executable, (IrInstructionCoroAlloc *)instruction);
5652		case IrInstructionIdCoroSize:
5653		return ir_render_coro_size(g, executable, (IrInstructionCoroSize *)instruction);
5654		case IrInstructionIdCoroBegin:
5655		return ir_render_coro_begin(g, executable, (IrInstructionCoroBegin *)instruction);
5656		case IrInstructionIdCoroAllocFail:
5657		return ir_render_coro_alloc_fail(g, executable, (IrInstructionCoroAllocFail *)instruction);
5658		case IrInstructionIdCoroSuspend:
5659		return ir_render_coro_suspend(g, executable, (IrInstructionCoroSuspend *)instruction);
5660		case IrInstructionIdCoroEnd:
5661		return ir_render_coro_end(g, executable, (IrInstructionCoroEnd *)instruction);
5662		case IrInstructionIdCoroFree:
5663		return ir_render_coro_free(g, executable, (IrInstructionCoroFree *)instruction);
5664		case IrInstructionIdCoroResume:
5665		return ir_render_coro_resume(g, executable, (IrInstructionCoroResume *)instruction);
5666		case IrInstructionIdCoroSave:
5667		return ir_render_coro_save(g, executable, (IrInstructionCoroSave *)instruction);
5668		case IrInstructionIdCoroPromise:
5669		return ir_render_coro_promise(g, executable, (IrInstructionCoroPromise *)instruction);
5670		case IrInstructionIdCoroAllocHelper:
5671		return ir_render_coro_alloc_helper(g, executable, (IrInstructionCoroAllocHelper *)instruction);
5672	5853	case IrInstructionIdAtomicRmw:
5673	5854	return ir_render_atomic_rmw(g, executable, (IrInstructionAtomicRmw *)instruction);
5674	5855	case IrInstructionIdAtomicLoad:
5675	5856	return ir_render_atomic_load(g, executable, (IrInstructionAtomicLoad *)instruction);
5676	5857	case IrInstructionIdSaveErrRetAddr:
5677	5858	return ir_render_save_err_ret_addr(g, executable, (IrInstructionSaveErrRetAddr *)instruction);
5678		case IrInstructionIdMergeErrRetTraces:
5679		return ir_render_merge_err_ret_traces(g, executable, (IrInstructionMergeErrRetTraces *)instruction);
5680		case IrInstructionIdMarkErrRetTracePtr:
5681		return ir_render_mark_err_ret_trace_ptr(g, executable, (IrInstructionMarkErrRetTracePtr *)instruction);
5682	5859	case IrInstructionIdFloatOp:
5683	5860	return ir_render_float_op(g, executable, (IrInstructionFloatOp *)instruction);
5684	5861	case IrInstructionIdMulAdd:
...	...	@@ -5695,6 +5872,20 @@ static LLVMValueRef ir_render_instruction(CodeGen g, IrExecutable executable,
5695	5872	return ir_render_resize_slice(g, executable, (IrInstructionResizeSlice *)instruction);
5696	5873	case IrInstructionIdPtrOfArrayToSlice:
5697	5874	return ir_render_ptr_of_array_to_slice(g, executable, (IrInstructionPtrOfArrayToSlice *)instruction);
	5875	case IrInstructionIdSuspendBegin:
	5876	return ir_render_suspend_begin(g, executable, (IrInstructionSuspendBegin *)instruction);
	5877	case IrInstructionIdSuspendFinish:
	5878	return ir_render_suspend_finish(g, executable, (IrInstructionSuspendFinish *)instruction);
	5879	case IrInstructionIdResume:
	5880	return ir_render_resume(g, executable, (IrInstructionResume *)instruction);
	5881	case IrInstructionIdFrameSizeGen:
	5882	return ir_render_frame_size(g, executable, (IrInstructionFrameSizeGen *)instruction);
	5883	case IrInstructionIdAwaitGen:
	5884	return ir_render_await(g, executable, (IrInstructionAwaitGen *)instruction);
	5885	case IrInstructionIdSpillBegin:
	5886	return ir_render_spill_begin(g, executable, (IrInstructionSpillBegin *)instruction);
	5887	case IrInstructionIdSpillEnd:
	5888	return ir_render_spill_end(g, executable, (IrInstructionSpillEnd *)instruction);
5698	5889	}
5699	5890	zig_unreachable();
5700	5891	}
...	...	@@ -5704,6 +5895,7 @@ static void ir_render(CodeGen g, ZigFn fn_entry) {
5704	5895
5705	5896	IrExecutable *executable = &fn_entry->analyzed_executable;
5706	5897	assert(executable->basic_block_list.length > 0);
	5898
5707	5899	for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) {
5708	5900	IrBasicBlock *current_block = executable->basic_block_list.at(block_i);
5709	5901	assert(current_block->llvm_block);
...	...	@@ -5894,7 +6086,6 @@ static LLVMValueRef pack_const_int(CodeGen *g, LLVMTypeRef big_int_type_ref, Con
5894	6086	case ZigTypeIdPointer:
5895	6087	case ZigTypeIdFn:
5896	6088	case ZigTypeIdOptional:
5897		case ZigTypeIdPromise:
5898	6089	{
5899	6090	LLVMValueRef ptr_val = gen_const_val(g, const_val, "");
5900	6091	LLVMValueRef ptr_size_int_val = LLVMConstPtrToInt(ptr_val, g->builtin_types.entry_usize->llvm_type);
...	...	@@ -5957,7 +6148,10 @@ static LLVMValueRef pack_const_int(CodeGen *g, LLVMTypeRef big_int_type_ref, Con
5957	6148	}
5958	6149	return val;
5959	6150	}
5960
	6151	case ZigTypeIdFnFrame:
	6152	zig_panic("TODO bit pack an async function frame");
	6153	case ZigTypeIdAnyFrame:
	6154	zig_panic("TODO bit pack an anyframe");
5961	6155	}
5962	6156	zig_unreachable();
5963	6157	}
...	...	@@ -6110,6 +6304,9 @@ static LLVMValueRef gen_const_val(CodeGen g, ConstExprValue const_val, const c
6110	6304	break;
6111	6305	}
6112	6306
	6307	if ((err = type_resolve(g, type_entry, ResolveStatusLLVMFull)))
	6308	zig_unreachable();
	6309
6113	6310	switch (type_entry->id) {
6114	6311	case ZigTypeIdInt:
6115	6312	return bigint_to_llvm_const(get_llvm_type(g, type_entry), &const_val->data.x_bigint);
...	...	@@ -6181,6 +6378,7 @@ static LLVMValueRef gen_const_val(CodeGen g, ConstExprValue const_val, const c
6181	6378	LLVMValueRef *fields = allocate<LLVMValueRef>(type_entry->data.structure.gen_field_count);
6182	6379	size_t src_field_count = type_entry->data.structure.src_field_count;
6183	6380	bool make_unnamed_struct = false;
	6381	assert(type_entry->data.structure.resolve_status == ResolveStatusLLVMFull);
6184	6382	if (type_entry->data.structure.layout == ContainerLayoutPacked) {
6185	6383	size_t src_field_index = 0;
6186	6384	while (src_field_index < src_field_count) {
...	...	@@ -6250,6 +6448,22 @@ static LLVMValueRef gen_const_val(CodeGen g, ConstExprValue const_val, const c
6250	6448	LLVMValueRef val = gen_const_val(g, field_val, "");
6251	6449	fields[type_struct_field->gen_index] = val;
6252	6450	make_unnamed_struct = make_unnamed_struct \|\| is_llvm_value_unnamed_type(g, field_val->type, val);
	6451
	6452	size_t end_pad_gen_index = (i + 1 < src_field_count) ?
	6453	type_entry->data.structure.fields[i + 1].gen_index :
	6454	type_entry->data.structure.gen_field_count;
	6455	size_t next_offset = (i + 1 < src_field_count) ?
	6456	type_entry->data.structure.fields[i + 1].offset : type_entry->abi_size;
	6457	if (end_pad_gen_index != SIZE_MAX) {
	6458	for (size_t gen_i = type_struct_field->gen_index + 1; gen_i < end_pad_gen_index;
	6459	gen_i += 1)
	6460	{
	6461	size_t pad_bytes = next_offset -
	6462	(type_struct_field->offset + type_struct_field->type_entry->abi_size);
	6463	LLVMTypeRef llvm_array_type = LLVMArrayType(LLVMInt8Type(), pad_bytes);
	6464	fields[gen_i] = LLVMGetUndef(llvm_array_type);
	6465	}
	6466	}
6253	6467	}
6254	6468	}
6255	6469	if (make_unnamed_struct) {
...	...	@@ -6437,13 +6651,18 @@ static LLVMValueRef gen_const_val(CodeGen g, ConstExprValue const_val, const c
6437	6651	err_payload_value = gen_const_val(g, payload_val, "");
6438	6652	make_unnamed_struct = is_llvm_value_unnamed_type(g, payload_val->type, err_payload_value);
6439	6653	}
6440		LLVMValueRef fields[2];
	6654	LLVMValueRef fields[3];
6441	6655	fields[err_union_err_index] = err_tag_value;
6442	6656	fields[err_union_payload_index] = err_payload_value;
	6657	size_t field_count = 2;
	6658	if (type_entry->data.error_union.pad_llvm_type != nullptr) {
	6659	fields[2] = LLVMGetUndef(type_entry->data.error_union.pad_llvm_type);
	6660	field_count = 3;
	6661	}
6443	6662	if (make_unnamed_struct) {
6444		return LLVMConstStruct(fields, 2, false);
	6663	return LLVMConstStruct(fields, field_count, false);
6445	6664	} else {
6446		return LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, 2);
	6665	return LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, field_count);
6447	6666	}
6448	6667	}
6449	6668	}
...	...	@@ -6460,9 +6679,11 @@ static LLVMValueRef gen_const_val(CodeGen g, ConstExprValue const_val, const c
6460	6679	case ZigTypeIdBoundFn:
6461	6680	case ZigTypeIdArgTuple:
6462	6681	case ZigTypeIdOpaque:
6463		case ZigTypeIdPromise:
6464	6682	zig_unreachable();
6465
	6683	case ZigTypeIdFnFrame:
	6684	zig_panic("TODO");
	6685	case ZigTypeIdAnyFrame:
	6686	zig_panic("TODO");
6466	6687	}
6467	6688	zig_unreachable();
6468	6689	}
...	...	@@ -6546,12 +6767,20 @@ static void generate_error_name_table(CodeGen *g) {
6546	6767	static void build_all_basic_blocks(CodeGen g, ZigFn fn) {
6547	6768	IrExecutable *executable = &fn->analyzed_executable;
6548	6769	assert(executable->basic_block_list.length > 0);
	6770	LLVMValueRef fn_val = fn_llvm_value(g, fn);
	6771	LLVMBasicBlockRef first_bb = nullptr;
	6772	if (fn_is_async(fn)) {
	6773	first_bb = LLVMAppendBasicBlock(fn_val, "AsyncSwitch");
	6774	g->cur_preamble_llvm_block = first_bb;
	6775	}
6549	6776	for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) {
6550	6777	IrBasicBlock *bb = executable->basic_block_list.at(block_i);
6551		bb->llvm_block = LLVMAppendBasicBlock(fn_llvm_value(g, fn), bb->name_hint);
	6778	bb->llvm_block = LLVMAppendBasicBlock(fn_val, bb->name_hint);
	6779	}
	6780	if (first_bb == nullptr) {
	6781	first_bb = executable->basic_block_list.at(0)->llvm_block;
6552	6782	}
6553		IrBasicBlock *entry_bb = executable->basic_block_list.at(0);
6554		LLVMPositionBuilderAtEnd(g->builder, entry_bb->llvm_block);
	6783	LLVMPositionBuilderAtEnd(g->builder, first_bb);
6555	6784	}
6556	6785
6557	6786	static void gen_global_var(CodeGen g, ZigVar var, LLVMValueRef init_val,
...	...	@@ -6728,13 +6957,19 @@ static void do_code_gen(CodeGen *g) {
6728	6957	build_all_basic_blocks(g, fn_table_entry);
6729	6958	clear_debug_source_node(g);
6730	6959
6731		if (want_sret) {
6732		g->cur_ret_ptr = LLVMGetParam(fn, 0);
6733		} else if (handle_is_ptr(fn_type_id->return_type)) {
6734		g->cur_ret_ptr = build_alloca(g, fn_type_id->return_type, "result", 0);
6735		// TODO add debug info variable for this
	6960	bool is_async = fn_is_async(fn_table_entry);
	6961
	6962	if (is_async) {
	6963	g->cur_frame_ptr = LLVMGetParam(fn, 0);
6736	6964	} else {
6737		g->cur_ret_ptr = nullptr;
	6965	if (want_sret) {
	6966	g->cur_ret_ptr = LLVMGetParam(fn, 0);
	6967	} else if (handle_is_ptr(fn_type_id->return_type)) {
	6968	g->cur_ret_ptr = build_alloca(g, fn_type_id->return_type, "result", 0);
	6969	// TODO add debug info variable for this
	6970	} else {
	6971	g->cur_ret_ptr = nullptr;
	6972	}
6738	6973	}
6739	6974
6740	6975	uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn_table_entry);
...	...	@@ -6746,39 +6981,41 @@ static void do_code_gen(CodeGen *g) {
6746	6981	}
6747	6982
6748	6983	// error return tracing setup
6749		bool is_async = cc == CallingConventionAsync;
6750		bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && !is_async && !have_err_ret_trace_arg;
	6984	bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn &&
	6985	!is_async && !have_err_ret_trace_arg;
6751	6986	LLVMValueRef err_ret_array_val = nullptr;
6752	6987	if (have_err_ret_trace_stack) {
6753	6988	ZigType *array_type = get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count);
6754	6989	err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type));
6755	6990
6756		// populate g->stack_trace_type
6757		(void)get_ptr_to_stack_trace_type(g);
6758		g->cur_err_ret_trace_val_stack = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type));
	6991	(void)get_llvm_type(g, get_stack_trace_type(g));
	6992	g->cur_err_ret_trace_val_stack = build_alloca(g, get_stack_trace_type(g), "error_return_trace",
	6993	get_abi_alignment(g, g->stack_trace_type));
6759	6994	} else {
6760	6995	g->cur_err_ret_trace_val_stack = nullptr;
6761	6996	}
6762	6997
6763		// allocate temporary stack data
6764		for (size_t alloca_i = 0; alloca_i < fn_table_entry->alloca_gen_list.length; alloca_i += 1) {
6765		IrInstructionAllocaGen *instruction = fn_table_entry->alloca_gen_list.at(alloca_i);
6766		ZigType *ptr_type = instruction->base.value.type;
6767		assert(ptr_type->id == ZigTypeIdPointer);
6768		ZigType *child_type = ptr_type->data.pointer.child_type;
6769		if (!type_has_bits(child_type))
6770		continue;
6771		if (instruction->base.ref_count == 0)
6772		continue;
6773		if (instruction->base.value.special != ConstValSpecialRuntime) {
6774		if (const_ptr_pointee(nullptr, g, &instruction->base.value, nullptr)->special !=
6775		ConstValSpecialRuntime)
6776		{
	6998	if (!is_async) {
	6999	// allocate temporary stack data
	7000	for (size_t alloca_i = 0; alloca_i < fn_table_entry->alloca_gen_list.length; alloca_i += 1) {
	7001	IrInstructionAllocaGen *instruction = fn_table_entry->alloca_gen_list.at(alloca_i);
	7002	ZigType *ptr_type = instruction->base.value.type;
	7003	assert(ptr_type->id == ZigTypeIdPointer);
	7004	ZigType *child_type = ptr_type->data.pointer.child_type;
	7005	if (!type_has_bits(child_type))
6777	7006	continue;
	7007	if (instruction->base.ref_count == 0)
	7008	continue;
	7009	if (instruction->base.value.special != ConstValSpecialRuntime) {
	7010	if (const_ptr_pointee(nullptr, g, &instruction->base.value, nullptr)->special !=
	7011	ConstValSpecialRuntime)
	7012	{
	7013	continue;
	7014	}
6778	7015	}
	7016	instruction->base.llvm_value = build_alloca(g, child_type, instruction->name_hint,
	7017	get_ptr_align(g, ptr_type));
6779	7018	}
6780		instruction->base.llvm_value = build_alloca(g, child_type, instruction->name_hint,
6781		get_ptr_align(g, ptr_type));
6782	7019	}
6783	7020
6784	7021	ZigType *import = get_scope_import(&fn_table_entry->fndef_scope->base);
...	...	@@ -6816,7 +7053,7 @@ static void do_code_gen(CodeGen *g) {
6816	7053	} else if (is_c_abi) {
6817	7054	fn_walk_var.data.vars.var = var;
6818	7055	iter_function_params_c_abi(g, fn_table_entry->type_entry, &fn_walk_var, var->src_arg_index);
6819		} else {
	7056	} else if (!is_async) {
6820	7057	ZigType *gen_type;
6821	7058	FnGenParamInfo *gen_info = &fn_table_entry->type_entry->data.fn.gen_param_info[var->src_arg_index];
6822	7059	assert(gen_info->gen_index != SIZE_MAX);
...	...	@@ -6867,14 +7104,76 @@ static void do_code_gen(CodeGen *g) {
6867	7104	gen_store(g, LLVMConstInt(usize->llvm_type, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false));
6868	7105	}
6869	7106
6870		// create debug variable declarations for parameters
6871		// rely on the first variables in the variable_list being parameters.
6872		FnWalk fn_walk_init = {};
6873		fn_walk_init.id = FnWalkIdInits;
6874		fn_walk_init.data.inits.fn = fn_table_entry;
6875		fn_walk_init.data.inits.llvm_fn = fn;
6876		fn_walk_init.data.inits.gen_i = gen_i_init;
6877		walk_function_params(g, fn_table_entry->type_entry, &fn_walk_init);
	7107	if (is_async) {
	7108	(void)get_llvm_type(g, fn_table_entry->frame_type);
	7109	g->cur_resume_block_count = 0;
	7110
	7111	LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
	7112	LLVMValueRef size_val = LLVMConstInt(usize_type_ref, fn_table_entry->frame_type->abi_size, false);
	7113	ZigLLVMFunctionSetPrefixData(fn_table_entry->llvm_value, size_val);
	7114
	7115	if (!g->strip_debug_symbols) {
	7116	AstNode *source_node = fn_table_entry->proto_node;
	7117	ZigLLVMSetCurrentDebugLocation(g->builder, (int)source_node->line + 1,
	7118	(int)source_node->column + 1, get_di_scope(g, fn_table_entry->child_scope));
	7119	}
	7120	IrExecutable *executable = &fn_table_entry->analyzed_executable;
	7121	LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
	7122	LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);
	7123	gen_assertion_scope(g, PanicMsgIdBadResume, fn_table_entry->child_scope);
	7124
	7125	LLVMPositionBuilderAtEnd(g->builder, g->cur_preamble_llvm_block);
	7126	render_async_spills(g);
	7127	g->cur_async_awaiter_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_awaiter_index, "");
	7128	LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_resume_index, "");
	7129	g->cur_async_resume_index_ptr = resume_index_ptr;
	7130
	7131	if (type_has_bits(fn_type_id->return_type)) {
	7132	LLVMValueRef cur_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_ret_start, "");
	7133	g->cur_ret_ptr = LLVMBuildLoad(g->builder, cur_ret_ptr_ptr, "");
	7134	}
	7135	uint32_t trace_field_index_stack = UINT32_MAX;
	7136	if (codegen_fn_has_err_ret_tracing_stack(g, fn_table_entry, true)) {
	7137	trace_field_index_stack = frame_index_trace_stack(g, fn_type_id);
	7138	g->cur_err_ret_trace_val_stack = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
	7139	trace_field_index_stack, "");
	7140	}
	7141
	7142	LLVMValueRef resume_index = LLVMBuildLoad(g->builder, resume_index_ptr, "");
	7143	LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, resume_index, bad_resume_block, 4);
	7144	g->cur_async_switch_instr = switch_instr;
	7145
	7146	LLVMValueRef zero = LLVMConstNull(usize_type_ref);
	7147	IrBasicBlock *entry_block = executable->basic_block_list.at(0);
	7148	LLVMAddCase(switch_instr, zero, entry_block->llvm_block);
	7149	g->cur_resume_block_count += 1;
	7150	LLVMPositionBuilderAtEnd(g->builder, entry_block->llvm_block);
	7151	if (trace_field_index_stack != UINT32_MAX) {
	7152	if (codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type)) {
	7153	LLVMValueRef trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
	7154	frame_index_trace_arg(g, fn_type_id->return_type), "");
	7155	LLVMValueRef zero_ptr = LLVMConstNull(LLVMGetElementType(LLVMTypeOf(trace_ptr_ptr)));
	7156	LLVMBuildStore(g->builder, zero_ptr, trace_ptr_ptr);
	7157	}
	7158
	7159	LLVMValueRef trace_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
	7160	trace_field_index_stack, "");
	7161	LLVMValueRef addrs_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
	7162	trace_field_index_stack + 1, "");
	7163
	7164	gen_init_stack_trace(g, trace_field_ptr, addrs_field_ptr);
	7165	}
	7166	render_async_var_decls(g, entry_block->instruction_list.at(0)->scope);
	7167	} else {
	7168	// create debug variable declarations for parameters
	7169	// rely on the first variables in the variable_list being parameters.
	7170	FnWalk fn_walk_init = {};
	7171	fn_walk_init.id = FnWalkIdInits;
	7172	fn_walk_init.data.inits.fn = fn_table_entry;
	7173	fn_walk_init.data.inits.llvm_fn = fn;
	7174	fn_walk_init.data.inits.gen_i = gen_i_init;
	7175	walk_function_params(g, fn_table_entry->type_entry, &fn_walk_init);
	7176	}
6878	7177
6879	7178	ir_render(g, fn_table_entry);
6880	7179
...	...	@@ -6893,8 +7192,6 @@ static void do_code_gen(CodeGen *g) {
6893	7192	LLVMDumpModule(g->module);
6894	7193	}
6895	7194
6896		// in release mode, we're sooooo confident that we've generated correct ir,
6897		// that we skip the verify module step in order to get better performance.
6898	7195	#ifndef NDEBUG
6899	7196	char *error = nullptr;
6900	7197	LLVMVerifyModule(g->module, LLVMAbortProcessAction, &error);
...	...	@@ -7163,16 +7460,8 @@ static void define_builtin_types(CodeGen *g) {
7163	7460
7164	7461	g->primitive_type_table.put(&entry->name, entry);
7165	7462	}
7166		{
7167		ZigType *entry = get_promise_type(g, nullptr);
7168		g->primitive_type_table.put(&entry->name, entry);
7169		entry->size_in_bits = g->builtin_types.entry_usize->size_in_bits;
7170		entry->abi_align = g->builtin_types.entry_usize->abi_align;
7171		entry->abi_size = g->builtin_types.entry_usize->abi_size;
7172		}
7173	7463	}
7174	7464
7175
7176	7465	static BuiltinFnEntry create_builtin_fn(CodeGen g, BuiltinFnId id, const char *name, size_t count) {
7177	7466	BuiltinFnEntry *builtin_fn = allocate<BuiltinFnEntry>(1);
7178	7467	buf_init_from_str(&builtin_fn->name, name);
...	...	@@ -7185,8 +7474,6 @@ static BuiltinFnEntry create_builtin_fn(CodeGen g, BuiltinFnId id, const char
7185	7474	static void define_builtin_fns(CodeGen *g) {
7186	7475	create_builtin_fn(g, BuiltinFnIdBreakpoint, "breakpoint", 0);
7187	7476	create_builtin_fn(g, BuiltinFnIdReturnAddress, "returnAddress", 0);
7188		create_builtin_fn(g, BuiltinFnIdFrameAddress, "frameAddress", 0);
7189		create_builtin_fn(g, BuiltinFnIdHandle, "handle", 0);
7190	7477	create_builtin_fn(g, BuiltinFnIdMemcpy, "memcpy", 3);
7191	7478	create_builtin_fn(g, BuiltinFnIdMemset, "memset", 3);
7192	7479	create_builtin_fn(g, BuiltinFnIdSizeof, "sizeOf", 1);
...	...	@@ -7262,13 +7549,13 @@ static void define_builtin_fns(CodeGen *g) {
7262	7549	create_builtin_fn(g, BuiltinFnIdFloor, "floor", 2);
7263	7550	create_builtin_fn(g, BuiltinFnIdCeil, "ceil", 2);
7264	7551	create_builtin_fn(g, BuiltinFnIdTrunc, "trunc", 2);
7265		//Needs library support on Windows
7266		//create_builtin_fn(g, BuiltinFnIdNearbyInt, "nearbyInt", 2);
	7552	create_builtin_fn(g, BuiltinFnIdNearbyInt, "nearbyInt", 2);
7267	7553	create_builtin_fn(g, BuiltinFnIdRound, "round", 2);
7268	7554	create_builtin_fn(g, BuiltinFnIdMulAdd, "mulAdd", 4);
7269	7555	create_builtin_fn(g, BuiltinFnIdInlineCall, "inlineCall", SIZE_MAX);
7270	7556	create_builtin_fn(g, BuiltinFnIdNoInlineCall, "noInlineCall", SIZE_MAX);
7271	7557	create_builtin_fn(g, BuiltinFnIdNewStackCall, "newStackCall", SIZE_MAX);
	7558	create_builtin_fn(g, BuiltinFnIdAsyncCall, "asyncCall", SIZE_MAX);
7272	7559	create_builtin_fn(g, BuiltinFnIdTypeId, "typeId", 1);
7273	7560	create_builtin_fn(g, BuiltinFnIdShlExact, "shlExact", 2);
7274	7561	create_builtin_fn(g, BuiltinFnIdShrExact, "shrExact", 2);
...	...	@@ -7287,6 +7574,10 @@ static void define_builtin_fns(CodeGen *g) {
7287	7574	create_builtin_fn(g, BuiltinFnIdThis, "This", 0);
7288	7575	create_builtin_fn(g, BuiltinFnIdHasDecl, "hasDecl", 2);
7289	7576	create_builtin_fn(g, BuiltinFnIdUnionInit, "unionInit", 3);
	7577	create_builtin_fn(g, BuiltinFnIdFrameHandle, "frame", 0);
	7578	create_builtin_fn(g, BuiltinFnIdFrameType, "Frame", 1);
	7579	create_builtin_fn(g, BuiltinFnIdFrameAddress, "frameAddress", 0);
	7580	create_builtin_fn(g, BuiltinFnIdFrameSize, "frameSize", 1);
7290	7581	}
7291	7582
7292	7583	static const char *bool_to_str(bool b) {
...	...	@@ -7598,7 +7889,8 @@ Buf codegen_generate_builtin_source(CodeGen g) {
7598	7889	" BoundFn: Fn,\n"
7599	7890	" ArgTuple: void,\n"
7600	7891	" Opaque: void,\n"
7601		" Promise: Promise,\n"
	7892	" Frame: void,\n"
	7893	" AnyFrame: AnyFrame,\n"
7602	7894	" Vector: Vector,\n"
7603	7895	" EnumLiteral: void,\n"
7604	7896	"\n\n"
...	...	@@ -7711,11 +8003,10 @@ Buf codegen_generate_builtin_source(CodeGen g) {
7711	8003	" is_generic: bool,\n"
7712	8004	" is_var_args: bool,\n"
7713	8005	" return_type: ?type,\n"
7714		" async_allocator_type: ?type,\n"
7715	8006	" args: []FnArg,\n"
7716	8007	" };\n"
7717	8008	"\n"
7718		" pub const Promise = struct {\n"
	8009	" pub const AnyFrame = struct {\n"
7719	8010	" child: ?type,\n"
7720	8011	" };\n"
7721	8012	"\n"
...	...	@@ -8308,6 +8599,12 @@ void add_cc_args(CodeGen g, ZigList<const char > &args, const char *out_dep_pa
8308	8599	args.append("-g");
8309	8600	}
8310	8601
	8602	if (codegen_have_frame_pointer(g)) {
	8603	args.append("-fno-omit-frame-pointer");
	8604	} else {
	8605	args.append("-fomit-frame-pointer");
	8606	}
	8607
8311	8608	switch (g->build_mode) {
8312	8609	case BuildModeDebug:
8313	8610	// windows c runtime requires -D_DEBUG if using debug libraries
...	...	@@ -8320,7 +8617,6 @@ void add_cc_args(CodeGen g, ZigList<const char > &args, const char *out_dep_pa
8320	8617	} else {
8321	8618	args.append("-fno-stack-protector");
8322	8619	}
8323		args.append("-fno-omit-frame-pointer");
8324	8620	break;
8325	8621	case BuildModeSafeRelease:
8326	8622	// See the comment in the BuildModeFastRelease case for why we pass -O2 rather
...	...	@@ -8334,7 +8630,6 @@ void add_cc_args(CodeGen g, ZigList<const char > &args, const char *out_dep_pa
8334	8630	} else {
8335	8631	args.append("-fno-stack-protector");
8336	8632	}
8337		args.append("-fomit-frame-pointer");
8338	8633	break;
8339	8634	case BuildModeFastRelease:
8340	8635	args.append("-DNDEBUG");
...	...	@@ -8345,13 +8640,11 @@ void add_cc_args(CodeGen g, ZigList<const char > &args, const char *out_dep_pa
8345	8640	// running in -O2 and thus the -O3 path has been tested less.
8346	8641	args.append("-O2");
8347	8642	args.append("-fno-stack-protector");
8348		args.append("-fomit-frame-pointer");
8349	8643	break;
8350	8644	case BuildModeSmallRelease:
8351	8645	args.append("-DNDEBUG");
8352	8646	args.append("-Os");
8353	8647	args.append("-fno-stack-protector");
8354		args.append("-fomit-frame-pointer");
8355	8648	break;
8356	8649	}
8357	8650
...	...	@@ -8878,7 +9171,8 @@ static void prepend_c_type_to_decl_list(CodeGen g, GenH gen_h, ZigType *type_e
8878	9171	case ZigTypeIdArgTuple:
8879	9172	case ZigTypeIdErrorUnion:
8880	9173	case ZigTypeIdErrorSet:
8881		case ZigTypeIdPromise:
	9174	case ZigTypeIdFnFrame:
	9175	case ZigTypeIdAnyFrame:
8882	9176	zig_unreachable();
8883	9177	case ZigTypeIdVoid:
8884	9178	case ZigTypeIdUnreachable:
...	...	@@ -9062,7 +9356,8 @@ static void get_c_type(CodeGen g, GenH gen_h, ZigType type_entry, Buf out_bu
9062	9356	case ZigTypeIdUndefined:
9063	9357	case ZigTypeIdNull:
9064	9358	case ZigTypeIdArgTuple:
9065		case ZigTypeIdPromise:
	9359	case ZigTypeIdFnFrame:
	9360	case ZigTypeIdAnyFrame:
9066	9361	zig_unreachable();
9067	9362	}
9068	9363	}
...	...	@@ -9229,9 +9524,11 @@ static void gen_h_file(CodeGen *g) {
9229	9524	case ZigTypeIdArgTuple:
9230	9525	case ZigTypeIdOptional:
9231	9526	case ZigTypeIdFn:
9232		case ZigTypeIdPromise:
9233	9527	case ZigTypeIdVector:
	9528	case ZigTypeIdFnFrame:
	9529	case ZigTypeIdAnyFrame:
9234	9530	zig_unreachable();
	9531
9235	9532	case ZigTypeIdEnum:
9236	9533	if (type_entry->data.enumeration.layout == ContainerLayoutExtern) {
9237	9534	fprintf(out_h, "enum %s {\n", buf_ptr(type_h_name(type_entry)));
...	...	@@ -9770,3 +10067,18 @@ CodeGen codegen_create(Buf main_pkg_path, Buf *root_src_path, const ZigTarget
9770	10067	return g;
9771	10068	}
9772	10069
	10070	bool codegen_fn_has_err_ret_tracing_arg(CodeGen g, ZigType return_type) {
	10071	return g->have_err_ret_tracing &&
	10072	(return_type->id == ZigTypeIdErrorUnion \|\|
	10073	return_type->id == ZigTypeIdErrorSet);
	10074	}
	10075
	10076	bool codegen_fn_has_err_ret_tracing_stack(CodeGen g, ZigFn fn, bool is_async) {
	10077	if (is_async) {
	10078	return g->have_err_ret_tracing && (fn->calls_or_awaits_errorable_fn \|\|
	10079	codegen_fn_has_err_ret_tracing_arg(g, fn->type_entry->data.fn.fn_type_id.return_type));
	10080	} else {
	10081	return g->have_err_ret_tracing && fn->calls_or_awaits_errorable_fn &&
	10082	!codegen_fn_has_err_ret_tracing_arg(g, fn->type_entry->data.fn.fn_type_id.return_type);
	10083	}
	10084	}

src/codegen.hpp+2

...	...	@@ -61,5 +61,7 @@ Buf codegen_generate_builtin_source(CodeGen g);
61	61	TargetSubsystem detect_subsystem(CodeGen *g);
62	62
63	63	void codegen_release_caches(CodeGen *codegen);
	64	bool codegen_fn_has_err_ret_tracing_arg(CodeGen g, ZigType return_type);
	65	bool codegen_fn_has_err_ret_tracing_stack(CodeGen g, ZigFn fn, bool is_async);
64	66
65	67	#endif

src/ir.cpp+853-1711

...	...	@@ -26,6 +26,7 @@ struct IrBuilder {
26	26	CodeGen *codegen;
27	27	IrExecutable *exec;
28	28	IrBasicBlock *current_basic_block;
	29	AstNode *main_block_node;
29	30	};
30	31
31	32	struct IrAnalyze {
...	...	@@ -99,7 +100,6 @@ struct ConstCastOnly {
99	100	ConstCastErrUnionErrSetMismatch *error_union_error_set;
100	101	ConstCastTypeMismatch *type_mismatch;
101	102	ConstCastOnly *return_type;
102		ConstCastOnly *async_allocator_type;
103	103	ConstCastOnly *null_wrap_ptr_child;
104	104	ConstCastArg fn_arg;
105	105	ConstCastArgNoAlias arg_no_alias;
...	...	@@ -305,6 +305,7 @@ static bool types_have_same_zig_comptime_repr(ZigType a, ZigType b) {
305	305	case ZigTypeIdBoundFn:
306	306	case ZigTypeIdErrorSet:
307	307	case ZigTypeIdOpaque:
	308	case ZigTypeIdAnyFrame:
308	309	return true;
309	310	case ZigTypeIdFloat:
310	311	return a->data.floating.bit_count == b->data.floating.bit_count;
...	...	@@ -319,8 +320,8 @@ static bool types_have_same_zig_comptime_repr(ZigType a, ZigType b) {
319	320	case ZigTypeIdUnion:
320	321	case ZigTypeIdFn:
321	322	case ZigTypeIdArgTuple:
322		case ZigTypeIdPromise:
323	323	case ZigTypeIdVector:
	324	case ZigTypeIdFnFrame:
324	325	return false;
325	326	}
326	327	zig_unreachable();
...	...	@@ -565,8 +566,8 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionArrayType *) {
565	566	return IrInstructionIdArrayType;
566	567	}
567	568
568		static constexpr IrInstructionId ir_instruction_id(IrInstructionPromiseType *) {
569		return IrInstructionIdPromiseType;
	569	static constexpr IrInstructionId ir_instruction_id(IrInstructionAnyFrameType *) {
	570	return IrInstructionIdAnyFrameType;
570	571	}
571	572
572	573	static constexpr IrInstructionId ir_instruction_id(IrInstructionSliceType *) {
...	...	@@ -761,8 +762,20 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionFrameAddress *)
761	762	return IrInstructionIdFrameAddress;
762	763	}
763	764
764		static constexpr IrInstructionId ir_instruction_id(IrInstructionHandle *) {
765		return IrInstructionIdHandle;
	765	static constexpr IrInstructionId ir_instruction_id(IrInstructionFrameHandle *) {
	766	return IrInstructionIdFrameHandle;
	767	}
	768
	769	static constexpr IrInstructionId ir_instruction_id(IrInstructionFrameType *) {
	770	return IrInstructionIdFrameType;
	771	}
	772
	773	static constexpr IrInstructionId ir_instruction_id(IrInstructionFrameSizeSrc *) {
	774	return IrInstructionIdFrameSizeSrc;
	775	}
	776
	777	static constexpr IrInstructionId ir_instruction_id(IrInstructionFrameSizeGen *) {
	778	return IrInstructionIdFrameSizeGen;
766	779	}
767	780
768	781	static constexpr IrInstructionId ir_instruction_id(IrInstructionAlignOf *) {
...	...	@@ -933,10 +946,6 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionResetResult *) {
933	946	return IrInstructionIdResetResult;
934	947	}
935	948
936		static constexpr IrInstructionId ir_instruction_id(IrInstructionResultPtr *) {
937		return IrInstructionIdResultPtr;
938		}
939
940	949	static constexpr IrInstructionId ir_instruction_id(IrInstructionPtrOfArrayToSlice *) {
941	950	return IrInstructionIdPtrOfArrayToSlice;
942	951	}
...	...	@@ -961,62 +970,6 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionErrorUnion *) {
961	970	return IrInstructionIdErrorUnion;
962	971	}
963	972
964		static constexpr IrInstructionId ir_instruction_id(IrInstructionCancel *) {
965		return IrInstructionIdCancel;
966		}
967
968		static constexpr IrInstructionId ir_instruction_id(IrInstructionGetImplicitAllocator *) {
969		return IrInstructionIdGetImplicitAllocator;
970		}
971
972		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroId *) {
973		return IrInstructionIdCoroId;
974		}
975
976		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroAlloc *) {
977		return IrInstructionIdCoroAlloc;
978		}
979
980		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroSize *) {
981		return IrInstructionIdCoroSize;
982		}
983
984		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroBegin *) {
985		return IrInstructionIdCoroBegin;
986		}
987
988		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroAllocFail *) {
989		return IrInstructionIdCoroAllocFail;
990		}
991
992		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroSuspend *) {
993		return IrInstructionIdCoroSuspend;
994		}
995
996		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroEnd *) {
997		return IrInstructionIdCoroEnd;
998		}
999
1000		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroFree *) {
1001		return IrInstructionIdCoroFree;
1002		}
1003
1004		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroResume *) {
1005		return IrInstructionIdCoroResume;
1006		}
1007
1008		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroSave *) {
1009		return IrInstructionIdCoroSave;
1010		}
1011
1012		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroPromise *) {
1013		return IrInstructionIdCoroPromise;
1014		}
1015
1016		static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroAllocHelper *) {
1017		return IrInstructionIdCoroAllocHelper;
1018		}
1019
1020	973	static constexpr IrInstructionId ir_instruction_id(IrInstructionAtomicRmw *) {
1021	974	return IrInstructionIdAtomicRmw;
1022	975	}
...	...	@@ -1025,14 +978,6 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionAtomicLoad *) {
1025	978	return IrInstructionIdAtomicLoad;
1026	979	}
1027	980
1028		static constexpr IrInstructionId ir_instruction_id(IrInstructionPromiseResultType *) {
1029		return IrInstructionIdPromiseResultType;
1030		}
1031
1032		static constexpr IrInstructionId ir_instruction_id(IrInstructionAwaitBookkeeping *) {
1033		return IrInstructionIdAwaitBookkeeping;
1034		}
1035
1036	981	static constexpr IrInstructionId ir_instruction_id(IrInstructionSaveErrRetAddr *) {
1037	982	return IrInstructionIdSaveErrRetAddr;
1038	983	}
...	...	@@ -1041,14 +986,6 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionAddImplicitRetur
1041	986	return IrInstructionIdAddImplicitReturnType;
1042	987	}
1043	988
1044		static constexpr IrInstructionId ir_instruction_id(IrInstructionMergeErrRetTraces *) {
1045		return IrInstructionIdMergeErrRetTraces;
1046		}
1047
1048		static constexpr IrInstructionId ir_instruction_id(IrInstructionMarkErrRetTracePtr *) {
1049		return IrInstructionIdMarkErrRetTracePtr;
1050		}
1051
1052	989	static constexpr IrInstructionId ir_instruction_id(IrInstructionFloatOp *) {
1053	990	return IrInstructionIdFloatOp;
1054	991	}
...	...	@@ -1097,6 +1034,34 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionUnionInitNamedFi
1097	1034	return IrInstructionIdUnionInitNamedField;
1098	1035	}
1099	1036
	1037	static constexpr IrInstructionId ir_instruction_id(IrInstructionSuspendBegin *) {
	1038	return IrInstructionIdSuspendBegin;
	1039	}
	1040
	1041	static constexpr IrInstructionId ir_instruction_id(IrInstructionSuspendFinish *) {
	1042	return IrInstructionIdSuspendFinish;
	1043	}
	1044
	1045	static constexpr IrInstructionId ir_instruction_id(IrInstructionAwaitSrc *) {
	1046	return IrInstructionIdAwaitSrc;
	1047	}
	1048
	1049	static constexpr IrInstructionId ir_instruction_id(IrInstructionAwaitGen *) {
	1050	return IrInstructionIdAwaitGen;
	1051	}
	1052
	1053	static constexpr IrInstructionId ir_instruction_id(IrInstructionResume *) {
	1054	return IrInstructionIdResume;
	1055	}
	1056
	1057	static constexpr IrInstructionId ir_instruction_id(IrInstructionSpillBegin *) {
	1058	return IrInstructionIdSpillBegin;
	1059	}
	1060
	1061	static constexpr IrInstructionId ir_instruction_id(IrInstructionSpillEnd *) {
	1062	return IrInstructionIdSpillEnd;
	1063	}
	1064
1100	1065	template<typename T>
1101	1066	static T ir_create_instruction(IrBuilder irb, Scope scope, AstNode source_node) {
1102	1067	T *special_instruction = allocate<T>(1);
...	...	@@ -1149,14 +1114,14 @@ static IrInstruction ir_build_cond_br(IrBuilder irb, Scope scope, AstNode so
1149	1114	}
1150	1115
1151	1116	static IrInstruction ir_build_return(IrBuilder irb, Scope scope, AstNode source_node,
1152		IrInstruction *return_value)
	1117	IrInstruction *operand)
1153	1118	{
1154	1119	IrInstructionReturn *return_instruction = ir_build_instruction<IrInstructionReturn>(irb, scope, source_node);
1155	1120	return_instruction->base.value.type = irb->codegen->builtin_types.entry_unreachable;
1156	1121	return_instruction->base.value.special = ConstValSpecialStatic;
1157		return_instruction->value = return_value;
	1122	return_instruction->operand = operand;
1158	1123
1159		if (return_value != nullptr) ir_ref_instruction(return_value, irb->current_basic_block);
	1124	if (operand != nullptr) ir_ref_instruction(operand, irb->current_basic_block);
1160	1125
1161	1126	return &return_instruction->base;
1162	1127	}
...	...	@@ -1214,14 +1179,6 @@ static IrInstruction ir_build_const_usize(IrBuilder irb, Scope *scope, AstNode
1214	1179	return &const_instruction->base;
1215	1180	}
1216	1181
1217		static IrInstruction ir_build_const_u8(IrBuilder irb, Scope scope, AstNode source_node, uint8_t value) {
1218		IrInstructionConst *const_instruction = ir_build_instruction<IrInstructionConst>(irb, scope, source_node);
1219		const_instruction->base.value.type = irb->codegen->builtin_types.entry_u8;
1220		const_instruction->base.value.special = ConstValSpecialStatic;
1221		bigint_init_unsigned(&const_instruction->base.value.data.x_bigint, value);
1222		return &const_instruction->base;
1223		}
1224
1225	1182	static IrInstruction ir_create_const_type(IrBuilder irb, Scope scope, AstNode source_node,
1226	1183	ZigType *type_entry)
1227	1184	{
...	...	@@ -1429,7 +1386,7 @@ static IrInstruction ir_build_union_field_ptr(IrBuilder irb, Scope *scope, Ast
1429	1386
1430	1387	static IrInstruction ir_build_call_src(IrBuilder irb, Scope scope, AstNode source_node,
1431	1388	ZigFn fn_entry, IrInstruction fn_ref, size_t arg_count, IrInstruction **args,
1432		bool is_comptime, FnInline fn_inline, bool is_async, IrInstruction *async_allocator,
	1389	bool is_comptime, FnInline fn_inline, bool is_async,
1433	1390	IrInstruction new_stack, ResultLoc result_loc)
1434	1391	{
1435	1392	IrInstructionCallSrc *call_instruction = ir_build_instruction<IrInstructionCallSrc>(irb, scope, source_node);
...	...	@@ -1440,22 +1397,24 @@ static IrInstruction ir_build_call_src(IrBuilder irb, Scope scope, AstNode s
1440	1397	call_instruction->args = args;
1441	1398	call_instruction->arg_count = arg_count;
1442	1399	call_instruction->is_async = is_async;
1443		call_instruction->async_allocator = async_allocator;
1444	1400	call_instruction->new_stack = new_stack;
1445	1401	call_instruction->result_loc = result_loc;
1446	1402
1447	1403	if (fn_ref != nullptr) ir_ref_instruction(fn_ref, irb->current_basic_block);
1448	1404	for (size_t i = 0; i < arg_count; i += 1)
1449	1405	ir_ref_instruction(args[i], irb->current_basic_block);
1450		if (async_allocator != nullptr) ir_ref_instruction(async_allocator, irb->current_basic_block);
	1406	if (is_async && new_stack != nullptr) {
	1407	// in this case the arg at the end is the return pointer
	1408	ir_ref_instruction(args[arg_count], irb->current_basic_block);
	1409	}
1451	1410	if (new_stack != nullptr) ir_ref_instruction(new_stack, irb->current_basic_block);
1452	1411
1453	1412	return &call_instruction->base;
1454	1413	}
1455	1414
1456		static IrInstruction ir_build_call_gen(IrAnalyze ira, IrInstruction *source_instruction,
	1415	static IrInstructionCallGen ir_build_call_gen(IrAnalyze ira, IrInstruction *source_instruction,
1457	1416	ZigFn fn_entry, IrInstruction fn_ref, size_t arg_count, IrInstruction **args,
1458		FnInline fn_inline, bool is_async, IrInstruction async_allocator, IrInstruction new_stack,
	1417	FnInline fn_inline, bool is_async, IrInstruction *new_stack,
1459	1418	IrInstruction result_loc, ZigType return_type)
1460	1419	{
1461	1420	IrInstructionCallGen *call_instruction = ir_build_instruction<IrInstructionCallGen>(&ira->new_irb,
...	...	@@ -1467,18 +1426,16 @@ static IrInstruction ir_build_call_gen(IrAnalyze ira, IrInstruction *source_in
1467	1426	call_instruction->args = args;
1468	1427	call_instruction->arg_count = arg_count;
1469	1428	call_instruction->is_async = is_async;
1470		call_instruction->async_allocator = async_allocator;
1471	1429	call_instruction->new_stack = new_stack;
1472	1430	call_instruction->result_loc = result_loc;
1473	1431
1474	1432	if (fn_ref != nullptr) ir_ref_instruction(fn_ref, ira->new_irb.current_basic_block);
1475	1433	for (size_t i = 0; i < arg_count; i += 1)
1476	1434	ir_ref_instruction(args[i], ira->new_irb.current_basic_block);
1477		if (async_allocator != nullptr) ir_ref_instruction(async_allocator, ira->new_irb.current_basic_block);
1478	1435	if (new_stack != nullptr) ir_ref_instruction(new_stack, ira->new_irb.current_basic_block);
1479	1436	if (result_loc != nullptr) ir_ref_instruction(result_loc, ira->new_irb.current_basic_block);
1480	1437
1481		return &call_instruction->base;
	1438	return call_instruction;
1482	1439	}
1483	1440
1484	1441	static IrInstruction ir_build_phi(IrBuilder irb, Scope scope, AstNode source_node,
...	...	@@ -1754,17 +1711,16 @@ static IrInstruction ir_build_array_type(IrBuilder irb, Scope *scope, AstNode
1754	1711	return &instruction->base;
1755	1712	}
1756	1713
1757		static IrInstruction ir_build_promise_type(IrBuilder irb, Scope scope, AstNode source_node,
	1714	static IrInstruction ir_build_anyframe_type(IrBuilder irb, Scope scope, AstNode source_node,
1758	1715	IrInstruction *payload_type)
1759	1716	{
1760		IrInstructionPromiseType *instruction = ir_build_instruction<IrInstructionPromiseType>(irb, scope, source_node);
	1717	IrInstructionAnyFrameType *instruction = ir_build_instruction<IrInstructionAnyFrameType>(irb, scope, source_node);
1761	1718	instruction->payload_type = payload_type;
1762	1719
1763	1720	if (payload_type != nullptr) ir_ref_instruction(payload_type, irb->current_basic_block);
1764	1721
1765	1722	return &instruction->base;
1766	1723	}
1767
1768	1724	static IrInstruction ir_build_slice_type(IrBuilder irb, Scope scope, AstNode source_node,
1769	1725	IrInstruction child_type, bool is_const, bool is_volatile, IrInstruction align_value, bool is_allow_zero)
1770	1726	{
...	...	@@ -2443,7 +2399,35 @@ static IrInstruction ir_build_frame_address(IrBuilder irb, Scope *scope, AstNo
2443	2399	}
2444	2400
2445	2401	static IrInstruction ir_build_handle(IrBuilder irb, Scope scope, AstNode source_node) {
2446		IrInstructionHandle *instruction = ir_build_instruction<IrInstructionHandle>(irb, scope, source_node);
	2402	IrInstructionFrameHandle *instruction = ir_build_instruction<IrInstructionFrameHandle>(irb, scope, source_node);
	2403	return &instruction->base;
	2404	}
	2405
	2406	static IrInstruction ir_build_frame_type(IrBuilder irb, Scope scope, AstNode source_node, IrInstruction *fn) {
	2407	IrInstructionFrameType *instruction = ir_build_instruction<IrInstructionFrameType>(irb, scope, source_node);
	2408	instruction->fn = fn;
	2409
	2410	ir_ref_instruction(fn, irb->current_basic_block);
	2411
	2412	return &instruction->base;
	2413	}
	2414
	2415	static IrInstruction ir_build_frame_size_src(IrBuilder irb, Scope scope, AstNode source_node, IrInstruction *fn) {
	2416	IrInstructionFrameSizeSrc *instruction = ir_build_instruction<IrInstructionFrameSizeSrc>(irb, scope, source_node);
	2417	instruction->fn = fn;
	2418
	2419	ir_ref_instruction(fn, irb->current_basic_block);
	2420
	2421	return &instruction->base;
	2422	}
	2423
	2424	static IrInstruction ir_build_frame_size_gen(IrBuilder irb, Scope scope, AstNode source_node, IrInstruction *fn)
	2425	{
	2426	IrInstructionFrameSizeGen *instruction = ir_build_instruction<IrInstructionFrameSizeGen>(irb, scope, source_node);
	2427	instruction->fn = fn;
	2428
	2429	ir_ref_instruction(fn, irb->current_basic_block);
	2430
2447	2431	return &instruction->base;
2448	2432	}
2449	2433
...	...	@@ -2546,11 +2530,12 @@ static IrInstruction ir_build_align_of(IrBuilder irb, Scope scope, AstNode s
2546	2530	}
2547	2531
2548	2532	static IrInstruction ir_build_test_err_src(IrBuilder irb, Scope scope, AstNode source_node,
2549		IrInstruction *base_ptr, bool resolve_err_set)
	2533	IrInstruction *base_ptr, bool resolve_err_set, bool base_ptr_is_payload)
2550	2534	{
2551	2535	IrInstructionTestErrSrc *instruction = ir_build_instruction<IrInstructionTestErrSrc>(irb, scope, source_node);
2552	2536	instruction->base_ptr = base_ptr;
2553	2537	instruction->resolve_err_set = resolve_err_set;
	2538	instruction->base_ptr_is_payload = base_ptr_is_payload;
2554	2539
2555	2540	ir_ref_instruction(base_ptr, irb->current_basic_block);
2556	2541
...	...	@@ -2596,13 +2581,12 @@ static IrInstruction ir_build_unwrap_err_payload(IrBuilder irb, Scope *scope,
2596	2581
2597	2582	static IrInstruction ir_build_fn_proto(IrBuilder irb, Scope scope, AstNode source_node,
2598	2583	IrInstruction *param_types, IrInstruction align_value, IrInstruction *return_type,
2599		IrInstruction *async_allocator_type_value, bool is_var_args)
	2584	bool is_var_args)
2600	2585	{
2601	2586	IrInstructionFnProto *instruction = ir_build_instruction<IrInstructionFnProto>(irb, scope, source_node);
2602	2587	instruction->param_types = param_types;
2603	2588	instruction->align_value = align_value;
2604	2589	instruction->return_type = return_type;
2605		instruction->async_allocator_type_value = async_allocator_type_value;
2606	2590	instruction->is_var_args = is_var_args;
2607	2591
2608	2592	assert(source_node->type == NodeTypeFnProto);
...	...	@@ -2612,7 +2596,6 @@ static IrInstruction ir_build_fn_proto(IrBuilder irb, Scope scope, AstNode s
2612	2596	if (param_types[i] != nullptr) ir_ref_instruction(param_types[i], irb->current_basic_block);
2613	2597	}
2614	2598	if (align_value != nullptr) ir_ref_instruction(align_value, irb->current_basic_block);
2615		if (async_allocator_type_value != nullptr) ir_ref_instruction(async_allocator_type_value, irb->current_basic_block);
2616	2599	ir_ref_instruction(return_type, irb->current_basic_block);
2617	2600
2618	2601	return &instruction->base;
...	...	@@ -2994,18 +2977,6 @@ static IrInstruction ir_build_reset_result(IrBuilder irb, Scope *scope, AstNod
2994	2977	return &instruction->base;
2995	2978	}
2996	2979
2997		static IrInstruction ir_build_result_ptr(IrBuilder irb, Scope scope, AstNode source_node,
2998		ResultLoc result_loc, IrInstruction result)
2999		{
3000		IrInstructionResultPtr *instruction = ir_build_instruction<IrInstructionResultPtr>(irb, scope, source_node);
3001		instruction->result_loc = result_loc;
3002		instruction->result = result;
3003
3004		ir_ref_instruction(result, irb->current_basic_block);
3005
3006		return &instruction->base;
3007		}
3008
3009	2980	static IrInstruction ir_build_opaque_type(IrBuilder irb, Scope scope, AstNode source_node) {
3010	2981	IrInstructionOpaqueType *instruction = ir_build_instruction<IrInstructionOpaqueType>(irb, scope, source_node);
3011	2982
...	...	@@ -3056,149 +3027,6 @@ static IrInstruction ir_build_error_union(IrBuilder irb, Scope *scope, AstNode
3056	3027	return &instruction->base;
3057	3028	}
3058	3029
3059		static IrInstruction ir_build_cancel(IrBuilder irb, Scope scope, AstNode source_node,
3060		IrInstruction *target)
3061		{
3062		IrInstructionCancel *instruction = ir_build_instruction<IrInstructionCancel>(irb, scope, source_node);
3063		instruction->target = target;
3064
3065		ir_ref_instruction(target, irb->current_basic_block);
3066
3067		return &instruction->base;
3068		}
3069
3070		static IrInstruction ir_build_get_implicit_allocator(IrBuilder irb, Scope scope, AstNode source_node,
3071		ImplicitAllocatorId id)
3072		{
3073		IrInstructionGetImplicitAllocator *instruction = ir_build_instruction<IrInstructionGetImplicitAllocator>(irb, scope, source_node);
3074		instruction->id = id;
3075
3076		return &instruction->base;
3077		}
3078
3079		static IrInstruction ir_build_coro_id(IrBuilder irb, Scope scope, AstNode source_node, IrInstruction *promise_ptr) {
3080		IrInstructionCoroId *instruction = ir_build_instruction<IrInstructionCoroId>(irb, scope, source_node);
3081		instruction->promise_ptr = promise_ptr;
3082
3083		ir_ref_instruction(promise_ptr, irb->current_basic_block);
3084
3085		return &instruction->base;
3086		}
3087
3088		static IrInstruction ir_build_coro_alloc(IrBuilder irb, Scope scope, AstNode source_node, IrInstruction *coro_id) {
3089		IrInstructionCoroAlloc *instruction = ir_build_instruction<IrInstructionCoroAlloc>(irb, scope, source_node);
3090		instruction->coro_id = coro_id;
3091
3092		ir_ref_instruction(coro_id, irb->current_basic_block);
3093
3094		return &instruction->base;
3095		}
3096
3097		static IrInstruction ir_build_coro_size(IrBuilder irb, Scope scope, AstNode source_node) {
3098		IrInstructionCoroSize *instruction = ir_build_instruction<IrInstructionCoroSize>(irb, scope, source_node);
3099
3100		return &instruction->base;
3101		}
3102
3103		static IrInstruction ir_build_coro_begin(IrBuilder irb, Scope scope, AstNode source_node, IrInstruction coro_id, IrInstruction coro_mem_ptr) {
3104		IrInstructionCoroBegin *instruction = ir_build_instruction<IrInstructionCoroBegin>(irb, scope, source_node);
3105		instruction->coro_id = coro_id;
3106		instruction->coro_mem_ptr = coro_mem_ptr;
3107
3108		ir_ref_instruction(coro_id, irb->current_basic_block);
3109		ir_ref_instruction(coro_mem_ptr, irb->current_basic_block);
3110
3111		return &instruction->base;
3112		}
3113
3114		static IrInstruction ir_build_coro_alloc_fail(IrBuilder irb, Scope scope, AstNode source_node, IrInstruction *err_val) {
3115		IrInstructionCoroAllocFail *instruction = ir_build_instruction<IrInstructionCoroAllocFail>(irb, scope, source_node);
3116		instruction->base.value.type = irb->codegen->builtin_types.entry_unreachable;
3117		instruction->base.value.special = ConstValSpecialStatic;
3118		instruction->err_val = err_val;
3119
3120		ir_ref_instruction(err_val, irb->current_basic_block);
3121
3122		return &instruction->base;
3123		}
3124
3125		static IrInstruction ir_build_coro_suspend(IrBuilder irb, Scope scope, AstNode source_node,
3126		IrInstruction save_point, IrInstruction is_final)
3127		{
3128		IrInstructionCoroSuspend *instruction = ir_build_instruction<IrInstructionCoroSuspend>(irb, scope, source_node);
3129		instruction->save_point = save_point;
3130		instruction->is_final = is_final;
3131
3132		if (save_point != nullptr) ir_ref_instruction(save_point, irb->current_basic_block);
3133		ir_ref_instruction(is_final, irb->current_basic_block);
3134
3135		return &instruction->base;
3136		}
3137
3138		static IrInstruction ir_build_coro_end(IrBuilder irb, Scope scope, AstNode source_node) {
3139		IrInstructionCoroEnd *instruction = ir_build_instruction<IrInstructionCoroEnd>(irb, scope, source_node);
3140		return &instruction->base;
3141		}
3142
3143		static IrInstruction ir_build_coro_free(IrBuilder irb, Scope scope, AstNode source_node,
3144		IrInstruction coro_id, IrInstruction coro_handle)
3145		{
3146		IrInstructionCoroFree *instruction = ir_build_instruction<IrInstructionCoroFree>(irb, scope, source_node);
3147		instruction->coro_id = coro_id;
3148		instruction->coro_handle = coro_handle;
3149
3150		ir_ref_instruction(coro_id, irb->current_basic_block);
3151		ir_ref_instruction(coro_handle, irb->current_basic_block);
3152
3153		return &instruction->base;
3154		}
3155
3156		static IrInstruction ir_build_coro_resume(IrBuilder irb, Scope scope, AstNode source_node,
3157		IrInstruction *awaiter_handle)
3158		{
3159		IrInstructionCoroResume *instruction = ir_build_instruction<IrInstructionCoroResume>(irb, scope, source_node);
3160		instruction->awaiter_handle = awaiter_handle;
3161
3162		ir_ref_instruction(awaiter_handle, irb->current_basic_block);
3163
3164		return &instruction->base;
3165		}
3166
3167		static IrInstruction ir_build_coro_save(IrBuilder irb, Scope scope, AstNode source_node,
3168		IrInstruction *coro_handle)
3169		{
3170		IrInstructionCoroSave *instruction = ir_build_instruction<IrInstructionCoroSave>(irb, scope, source_node);
3171		instruction->coro_handle = coro_handle;
3172
3173		ir_ref_instruction(coro_handle, irb->current_basic_block);
3174
3175		return &instruction->base;
3176		}
3177
3178		static IrInstruction ir_build_coro_promise(IrBuilder irb, Scope scope, AstNode source_node,
3179		IrInstruction *coro_handle)
3180		{
3181		IrInstructionCoroPromise *instruction = ir_build_instruction<IrInstructionCoroPromise>(irb, scope, source_node);
3182		instruction->coro_handle = coro_handle;
3183
3184		ir_ref_instruction(coro_handle, irb->current_basic_block);
3185
3186		return &instruction->base;
3187		}
3188
3189		static IrInstruction ir_build_coro_alloc_helper(IrBuilder irb, Scope scope, AstNode source_node,
3190		IrInstruction realloc_fn, IrInstruction coro_size)
3191		{
3192		IrInstructionCoroAllocHelper *instruction = ir_build_instruction<IrInstructionCoroAllocHelper>(irb, scope, source_node);
3193		instruction->realloc_fn = realloc_fn;
3194		instruction->coro_size = coro_size;
3195
3196		ir_ref_instruction(realloc_fn, irb->current_basic_block);
3197		ir_ref_instruction(coro_size, irb->current_basic_block);
3198
3199		return &instruction->base;
3200		}
3201
3202	3030	static IrInstruction ir_build_atomic_rmw(IrBuilder irb, Scope scope, AstNode source_node,
3203	3031	IrInstruction operand_type, IrInstruction ptr, IrInstruction op, IrInstruction operand,
3204	3032	IrInstruction *ordering, AtomicRmwOp resolved_op, AtomicOrder resolved_ordering)
...	...	@@ -3238,28 +3066,6 @@ static IrInstruction ir_build_atomic_load(IrBuilder irb, Scope *scope, AstNode
3238	3066	return &instruction->base;
3239	3067	}
3240	3068
3241		static IrInstruction ir_build_promise_result_type(IrBuilder irb, Scope scope, AstNode source_node,
3242		IrInstruction *promise_type)
3243		{
3244		IrInstructionPromiseResultType *instruction = ir_build_instruction<IrInstructionPromiseResultType>(irb, scope, source_node);
3245		instruction->promise_type = promise_type;
3246
3247		ir_ref_instruction(promise_type, irb->current_basic_block);
3248
3249		return &instruction->base;
3250		}
3251
3252		static IrInstruction ir_build_await_bookkeeping(IrBuilder irb, Scope scope, AstNode source_node,
3253		IrInstruction *promise_result_type)
3254		{
3255		IrInstructionAwaitBookkeeping *instruction = ir_build_instruction<IrInstructionAwaitBookkeeping>(irb, scope, source_node);
3256		instruction->promise_result_type = promise_result_type;
3257
3258		ir_ref_instruction(promise_result_type, irb->current_basic_block);
3259
3260		return &instruction->base;
3261		}
3262
3263	3069	static IrInstruction ir_build_save_err_ret_addr(IrBuilder irb, Scope scope, AstNode source_node) {
3264	3070	IrInstructionSaveErrRetAddr *instruction = ir_build_instruction<IrInstructionSaveErrRetAddr>(irb, scope, source_node);
3265	3071	return &instruction->base;
...	...	@@ -3276,30 +3082,6 @@ static IrInstruction ir_build_add_implicit_return_type(IrBuilder irb, Scope *s
3276	3082	return &instruction->base;
3277	3083	}
3278	3084
3279		static IrInstruction ir_build_merge_err_ret_traces(IrBuilder irb, Scope scope, AstNode source_node,
3280		IrInstruction coro_promise_ptr, IrInstruction src_err_ret_trace_ptr, IrInstruction *dest_err_ret_trace_ptr)
3281		{
3282		IrInstructionMergeErrRetTraces *instruction = ir_build_instruction<IrInstructionMergeErrRetTraces>(irb, scope, source_node);
3283		instruction->coro_promise_ptr = coro_promise_ptr;
3284		instruction->src_err_ret_trace_ptr = src_err_ret_trace_ptr;
3285		instruction->dest_err_ret_trace_ptr = dest_err_ret_trace_ptr;
3286
3287		ir_ref_instruction(coro_promise_ptr, irb->current_basic_block);
3288		ir_ref_instruction(src_err_ret_trace_ptr, irb->current_basic_block);
3289		ir_ref_instruction(dest_err_ret_trace_ptr, irb->current_basic_block);
3290
3291		return &instruction->base;
3292		}
3293
3294		static IrInstruction ir_build_mark_err_ret_trace_ptr(IrBuilder irb, Scope scope, AstNode source_node, IrInstruction *err_ret_trace_ptr) {
3295		IrInstructionMarkErrRetTracePtr *instruction = ir_build_instruction<IrInstructionMarkErrRetTracePtr>(irb, scope, source_node);
3296		instruction->err_ret_trace_ptr = err_ret_trace_ptr;
3297
3298		ir_ref_instruction(err_ret_trace_ptr, irb->current_basic_block);
3299
3300		return &instruction->base;
3301		}
3302
3303	3085	static IrInstruction ir_build_has_decl(IrBuilder irb, Scope scope, AstNode source_node,
3304	3086	IrInstruction container, IrInstruction name)
3305	3087	{
...	...	@@ -3435,7 +3217,7 @@ static IrInstruction ir_build_alloca_src(IrBuilder irb, Scope *scope, AstNode
3435	3217	return &instruction->base;
3436	3218	}
3437	3219
3438		static IrInstructionAllocaGen ir_create_alloca_gen(IrAnalyze ira, IrInstruction *source_instruction,
	3220	static IrInstructionAllocaGen ir_build_alloca_gen(IrAnalyze ira, IrInstruction *source_instruction,
3439	3221	uint32_t align, const char *name_hint)
3440	3222	{
3441	3223	IrInstructionAllocaGen *instruction = ir_create_instruction<IrInstructionAllocaGen>(&ira->new_irb,
...	...	@@ -3459,6 +3241,87 @@ static IrInstruction ir_build_end_expr(IrBuilder irb, Scope scope, AstNode s
3459	3241	return &instruction->base;
3460	3242	}
3461	3243
	3244	static IrInstructionSuspendBegin ir_build_suspend_begin(IrBuilder irb, Scope scope, AstNode source_node) {
	3245	IrInstructionSuspendBegin *instruction = ir_build_instruction<IrInstructionSuspendBegin>(irb, scope, source_node);
	3246	instruction->base.value.type = irb->codegen->builtin_types.entry_void;
	3247
	3248	return instruction;
	3249	}
	3250
	3251	static IrInstruction ir_build_suspend_finish(IrBuilder irb, Scope scope, AstNode source_node,
	3252	IrInstructionSuspendBegin *begin)
	3253	{
	3254	IrInstructionSuspendFinish *instruction = ir_build_instruction<IrInstructionSuspendFinish>(irb, scope, source_node);
	3255	instruction->base.value.type = irb->codegen->builtin_types.entry_void;
	3256	instruction->begin = begin;
	3257
	3258	ir_ref_instruction(&begin->base, irb->current_basic_block);
	3259
	3260	return &instruction->base;
	3261	}
	3262
	3263	static IrInstruction ir_build_await_src(IrBuilder irb, Scope scope, AstNode source_node,
	3264	IrInstruction frame, ResultLoc result_loc)
	3265	{
	3266	IrInstructionAwaitSrc *instruction = ir_build_instruction<IrInstructionAwaitSrc>(irb, scope, source_node);
	3267	instruction->frame = frame;
	3268	instruction->result_loc = result_loc;
	3269
	3270	ir_ref_instruction(frame, irb->current_basic_block);
	3271
	3272	return &instruction->base;
	3273	}
	3274
	3275	static IrInstruction ir_build_await_gen(IrAnalyze ira, IrInstruction *source_instruction,
	3276	IrInstruction frame, ZigType result_type, IrInstruction *result_loc)
	3277	{
	3278	IrInstructionAwaitGen *instruction = ir_build_instruction<IrInstructionAwaitGen>(&ira->new_irb,
	3279	source_instruction->scope, source_instruction->source_node);
	3280	instruction->base.value.type = result_type;
	3281	instruction->frame = frame;
	3282	instruction->result_loc = result_loc;
	3283
	3284	ir_ref_instruction(frame, ira->new_irb.current_basic_block);
	3285	if (result_loc != nullptr) ir_ref_instruction(result_loc, ira->new_irb.current_basic_block);
	3286
	3287	return &instruction->base;
	3288	}
	3289
	3290	static IrInstruction ir_build_resume(IrBuilder irb, Scope scope, AstNode source_node, IrInstruction *frame) {
	3291	IrInstructionResume *instruction = ir_build_instruction<IrInstructionResume>(irb, scope, source_node);
	3292	instruction->base.value.type = irb->codegen->builtin_types.entry_void;
	3293	instruction->frame = frame;
	3294
	3295	ir_ref_instruction(frame, irb->current_basic_block);
	3296
	3297	return &instruction->base;
	3298	}
	3299
	3300	static IrInstructionSpillBegin ir_build_spill_begin(IrBuilder irb, Scope scope, AstNode source_node,
	3301	IrInstruction *operand, SpillId spill_id)
	3302	{
	3303	IrInstructionSpillBegin *instruction = ir_build_instruction<IrInstructionSpillBegin>(irb, scope, source_node);
	3304	instruction->base.value.special = ConstValSpecialStatic;
	3305	instruction->base.value.type = irb->codegen->builtin_types.entry_void;
	3306	instruction->operand = operand;
	3307	instruction->spill_id = spill_id;
	3308
	3309	ir_ref_instruction(operand, irb->current_basic_block);
	3310
	3311	return instruction;
	3312	}
	3313
	3314	static IrInstruction ir_build_spill_end(IrBuilder irb, Scope scope, AstNode source_node,
	3315	IrInstructionSpillBegin *begin)
	3316	{
	3317	IrInstructionSpillEnd *instruction = ir_build_instruction<IrInstructionSpillEnd>(irb, scope, source_node);
	3318	instruction->begin = begin;
	3319
	3320	ir_ref_instruction(&begin->base, irb->current_basic_block);
	3321
	3322	return &instruction->base;
	3323	}
	3324
3462	3325	static void ir_count_defers(IrBuilder irb, Scope inner_scope, Scope outer_scope, size_t results) {
3463	3326	results[ReturnKindUnconditional] = 0;
3464	3327	results[ReturnKindError] = 0;
...	...	@@ -3489,7 +3352,6 @@ static void ir_count_defers(IrBuilder irb, Scope inner_scope, Scope *outer_sco
3489	3352	continue;
3490	3353	case ScopeIdDeferExpr:
3491	3354	case ScopeIdCImport:
3492		case ScopeIdCoroPrelude:
3493	3355	zig_unreachable();
3494	3356	}
3495	3357	}
...	...	@@ -3545,7 +3407,6 @@ static bool ir_gen_defers_for_block(IrBuilder irb, Scope inner_scope, Scope *o
3545	3407	continue;
3546	3408	case ScopeIdDeferExpr:
3547	3409	case ScopeIdCImport:
3548		case ScopeIdCoroPrelude:
3549	3410	zig_unreachable();
3550	3411	}
3551	3412	}
...	...	@@ -3588,66 +3449,6 @@ static ScopeDeferExpr get_scope_defer_expr(Scope scope) {
3588	3449	return nullptr;
3589	3450	}
3590	3451
3591		static bool exec_is_async(IrExecutable *exec) {
3592		ZigFn *fn_entry = exec_fn_entry(exec);
3593		return fn_entry != nullptr && fn_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync;
3594		}
3595
3596		static IrInstruction ir_gen_async_return(IrBuilder irb, Scope scope, AstNode node, IrInstruction *return_value,
3597		bool is_generated_code)
3598		{
3599		ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, return_value));
3600
3601		bool is_async = exec_is_async(irb->exec);
3602		if (!is_async) {
3603		IrInstruction *return_inst = ir_build_return(irb, scope, node, return_value);
3604		return_inst->is_gen = is_generated_code;
3605		return return_inst;
3606		}
3607
3608		IrBasicBlock *suspended_block = ir_create_basic_block(irb, scope, "Suspended");
3609		IrBasicBlock *not_suspended_block = ir_create_basic_block(irb, scope, "NotSuspended");
3610		IrBasicBlock *store_awaiter_block = ir_create_basic_block(irb, scope, "StoreAwaiter");
3611		IrBasicBlock *check_canceled_block = ir_create_basic_block(irb, scope, "CheckCanceled");
3612
3613		IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, scope, node, 0x7); // 0b111
3614		IrInstruction *ptr_mask = ir_build_un_op(irb, scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000
3615		IrInstruction *is_canceled_mask = ir_build_const_usize(irb, scope, node, 0x1); // 0b001
3616		IrInstruction *is_suspended_mask = ir_build_const_usize(irb, scope, node, 0x2); // 0b010
3617		IrInstruction *promise_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_promise);
3618		IrInstruction *is_comptime = ir_build_const_bool(irb, scope, node, false);
3619		IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
3620
3621		ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_field_ptr, return_value);
3622		IrInstruction *usize_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize);
3623		IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node,
3624		usize_type_val, irb->exec->atomic_state_field_ptr, nullptr, ptr_mask, nullptr,
3625		AtomicRmwOp_or, AtomicOrderSeqCst);
3626
3627		IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false);
3628		IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false);
3629		ir_build_cond_br(irb, scope, node, is_suspended_bool, suspended_block, not_suspended_block, is_comptime);
3630
3631		ir_set_cursor_at_end_and_append_block(irb, suspended_block);
3632		ir_build_unreachable(irb, scope, node);
3633
3634		ir_set_cursor_at_end_and_append_block(irb, not_suspended_block);
3635		IrInstruction *await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false);
3636		// if we ever add null checking safety to the ptrtoint instruction, it needs to be disabled here
3637		IrInstruction *have_await_handle = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false);
3638		ir_build_cond_br(irb, scope, node, have_await_handle, store_awaiter_block, check_canceled_block, is_comptime);
3639
3640		ir_set_cursor_at_end_and_append_block(irb, store_awaiter_block);
3641		IrInstruction *await_handle = ir_build_int_to_ptr(irb, scope, node, promise_type_val, await_handle_addr);
3642		ir_build_store_ptr(irb, scope, node, irb->exec->await_handle_var_ptr, await_handle);
3643		ir_build_br(irb, scope, node, irb->exec->coro_normal_final, is_comptime);
3644
3645		ir_set_cursor_at_end_and_append_block(irb, check_canceled_block);
3646		IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false);
3647		IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false);
3648		return ir_build_cond_br(irb, scope, node, is_canceled_bool, irb->exec->coro_final_cleanup_block, irb->exec->coro_early_final, is_comptime);
3649		}
3650
3651	3452	static IrInstruction ir_gen_return(IrBuilder irb, Scope scope, AstNode node, LVal lval, ResultLoc *result_loc) {
3652	3453	assert(node->type == NodeTypeReturnExpr);
3653	3454
...	...	@@ -3689,57 +3490,58 @@ static IrInstruction ir_gen_return(IrBuilder irb, Scope scope, AstNode node,
3689	3490	return_value = ir_build_const_void(irb, scope, node);
3690	3491	}
3691	3492
	3493	ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, return_value));
	3494
3692	3495	size_t defer_counts[2];
3693	3496	ir_count_defers(irb, scope, outer_scope, defer_counts);
3694	3497	bool have_err_defers = defer_counts[ReturnKindError] > 0;
3695		if (have_err_defers \|\| irb->codegen->have_err_ret_tracing) {
3696		IrBasicBlock *err_block = ir_create_basic_block(irb, scope, "ErrRetErr");
3697		IrBasicBlock *ok_block = ir_create_basic_block(irb, scope, "ErrRetOk");
3698		if (!have_err_defers) {
3699		ir_gen_defers_for_block(irb, scope, outer_scope, false);
3700		}
	3498	if (!have_err_defers && !irb->codegen->have_err_ret_tracing) {
	3499	// only generate unconditional defers
	3500	ir_gen_defers_for_block(irb, scope, outer_scope, false);
	3501	IrInstruction *result = ir_build_return(irb, scope, node, return_value);
	3502	result_loc_ret->base.source_instruction = result;
	3503	return result;
	3504	}
	3505	bool should_inline = ir_should_inline(irb->exec, scope);
3701	3506
3702		IrInstruction *ret_ptr = ir_build_result_ptr(irb, scope, node, &result_loc_ret->base,
3703		return_value);
3704		IrInstruction *is_err = ir_build_test_err_src(irb, scope, node, ret_ptr, false);
	3507	IrBasicBlock *err_block = ir_create_basic_block(irb, scope, "ErrRetErr");
	3508	IrBasicBlock *ok_block = ir_create_basic_block(irb, scope, "ErrRetOk");
3705	3509
3706		bool should_inline = ir_should_inline(irb->exec, scope);
3707		IrInstruction *is_comptime;
3708		if (should_inline) {
3709		is_comptime = ir_build_const_bool(irb, scope, node, true);
3710		} else {
3711		is_comptime = ir_build_test_comptime(irb, scope, node, is_err);
3712		}
	3510	if (!have_err_defers) {
	3511	ir_gen_defers_for_block(irb, scope, outer_scope, false);
	3512	}
3713	3513
3714		ir_mark_gen(ir_build_cond_br(irb, scope, node, is_err, err_block, ok_block, is_comptime));
3715		IrBasicBlock *ret_stmt_block = ir_create_basic_block(irb, scope, "RetStmt");
	3514	IrInstruction *is_err = ir_build_test_err_src(irb, scope, node, return_value, false, true);
3716	3515
3717		ir_set_cursor_at_end_and_append_block(irb, err_block);
3718		if (have_err_defers) {
3719		ir_gen_defers_for_block(irb, scope, outer_scope, true);
3720		}
3721		if (irb->codegen->have_err_ret_tracing && !should_inline) {
3722		ir_build_save_err_ret_addr(irb, scope, node);
3723		}
3724		ir_build_br(irb, scope, node, ret_stmt_block, is_comptime);
	3516	IrInstruction *is_comptime;
	3517	if (should_inline) {
	3518	is_comptime = ir_build_const_bool(irb, scope, node, should_inline);
	3519	} else {
	3520	is_comptime = ir_build_test_comptime(irb, scope, node, is_err);
	3521	}
3725	3522
3726		ir_set_cursor_at_end_and_append_block(irb, ok_block);
3727		if (have_err_defers) {
3728		ir_gen_defers_for_block(irb, scope, outer_scope, false);
3729		}
3730		ir_build_br(irb, scope, node, ret_stmt_block, is_comptime);
	3523	ir_mark_gen(ir_build_cond_br(irb, scope, node, is_err, err_block, ok_block, is_comptime));
	3524	IrBasicBlock *ret_stmt_block = ir_create_basic_block(irb, scope, "RetStmt");
3731	3525
3732		ir_set_cursor_at_end_and_append_block(irb, ret_stmt_block);
3733		IrInstruction *result = ir_gen_async_return(irb, scope, node, return_value, false);
3734		result_loc_ret->base.source_instruction = result;
3735		return result;
3736		} else {
3737		// generate unconditional defers
	3526	ir_set_cursor_at_end_and_append_block(irb, err_block);
	3527	if (have_err_defers) {
	3528	ir_gen_defers_for_block(irb, scope, outer_scope, true);
	3529	}
	3530	if (irb->codegen->have_err_ret_tracing && !should_inline) {
	3531	ir_build_save_err_ret_addr(irb, scope, node);
	3532	}
	3533	ir_build_br(irb, scope, node, ret_stmt_block, is_comptime);
	3534
	3535	ir_set_cursor_at_end_and_append_block(irb, ok_block);
	3536	if (have_err_defers) {
3738	3537	ir_gen_defers_for_block(irb, scope, outer_scope, false);
3739		IrInstruction *result = ir_gen_async_return(irb, scope, node, return_value, false);
3740		result_loc_ret->base.source_instruction = result;
3741		return result;
3742	3538	}
	3539	ir_build_br(irb, scope, node, ret_stmt_block, is_comptime);
	3540
	3541	ir_set_cursor_at_end_and_append_block(irb, ret_stmt_block);
	3542	IrInstruction *result = ir_build_return(irb, scope, node, return_value);
	3543	result_loc_ret->base.source_instruction = result;
	3544	return result;
3743	3545	}
3744	3546	case ReturnKindError:
3745	3547	{
...	...	@@ -3747,7 +3549,7 @@ static IrInstruction ir_gen_return(IrBuilder irb, Scope scope, AstNode node,
3747	3549	IrInstruction *err_union_ptr = ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr);
3748	3550	if (err_union_ptr == irb->codegen->invalid_instruction)
3749	3551	return irb->codegen->invalid_instruction;
3750		IrInstruction *is_err_val = ir_build_test_err_src(irb, scope, node, err_union_ptr, true);
	3552	IrInstruction *is_err_val = ir_build_test_err_src(irb, scope, node, err_union_ptr, true, false);
3751	3553
3752	3554	IrBasicBlock *return_block = ir_create_basic_block(irb, scope, "ErrRetReturn");
3753	3555	IrBasicBlock *continue_block = ir_create_basic_block(irb, scope, "ErrRetContinue");
...	...	@@ -3761,19 +3563,21 @@ static IrInstruction ir_gen_return(IrBuilder irb, Scope scope, AstNode node,
3761	3563	ir_mark_gen(ir_build_cond_br(irb, scope, node, is_err_val, return_block, continue_block, is_comptime));
3762	3564
3763	3565	ir_set_cursor_at_end_and_append_block(irb, return_block);
	3566	IrInstruction *err_val_ptr = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr);
	3567	IrInstruction *err_val = ir_build_load_ptr(irb, scope, node, err_val_ptr);
	3568	ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, err_val));
	3569	IrInstructionSpillBegin *spill_begin = ir_build_spill_begin(irb, scope, node, err_val,
	3570	SpillIdRetErrCode);
	3571	ResultLocReturn *result_loc_ret = allocate<ResultLocReturn>(1);
	3572	result_loc_ret->base.id = ResultLocIdReturn;
	3573	ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
	3574	ir_build_end_expr(irb, scope, node, err_val, &result_loc_ret->base);
3764	3575	if (!ir_gen_defers_for_block(irb, scope, outer_scope, true)) {
3765		IrInstruction *err_val_ptr = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr);
3766		IrInstruction *err_val = ir_build_load_ptr(irb, scope, node, err_val_ptr);
3767
3768		ResultLocReturn *result_loc_ret = allocate<ResultLocReturn>(1);
3769		result_loc_ret->base.id = ResultLocIdReturn;
3770		ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
3771		ir_build_end_expr(irb, scope, node, err_val, &result_loc_ret->base);
3772
3773	3576	if (irb->codegen->have_err_ret_tracing && !should_inline) {
3774	3577	ir_build_save_err_ret_addr(irb, scope, node);
3775	3578	}
3776		IrInstruction *ret_inst = ir_gen_async_return(irb, scope, node, err_val, false);
	3579	err_val = ir_build_spill_end(irb, scope, node, spill_begin);
	3580	IrInstruction *ret_inst = ir_build_return(irb, scope, node, err_val);
3777	3581	result_loc_ret->base.source_instruction = ret_inst;
3778	3582	}
3779	3583
...	...	@@ -3971,18 +3775,31 @@ static IrInstruction ir_gen_block(IrBuilder irb, Scope *parent_scope, AstNode
3971	3775	incoming_values.append(else_expr_result);
3972	3776	}
3973	3777
3974		if (block_node->data.block.name != nullptr) {
	3778	bool is_return_from_fn = block_node == irb->main_block_node;
	3779	if (!is_return_from_fn) {
3975	3780	ir_gen_defers_for_block(irb, child_scope, outer_block_scope, false);
	3781	}
	3782
	3783	IrInstruction *result;
	3784	if (block_node->data.block.name != nullptr) {
3976	3785	ir_mark_gen(ir_build_br(irb, parent_scope, block_node, scope_block->end_block, scope_block->is_comptime));
3977	3786	ir_set_cursor_at_end_and_append_block(irb, scope_block->end_block);
3978	3787	IrInstruction *phi = ir_build_phi(irb, parent_scope, block_node, incoming_blocks.length,
3979	3788	incoming_blocks.items, incoming_values.items, scope_block->peer_parent);
3980		return ir_expr_wrap(irb, parent_scope, phi, result_loc);
	3789	result = ir_expr_wrap(irb, parent_scope, phi, result_loc);
3981	3790	} else {
3982		ir_gen_defers_for_block(irb, child_scope, outer_block_scope, false);
3983	3791	IrInstruction *void_inst = ir_mark_gen(ir_build_const_void(irb, child_scope, block_node));
3984		return ir_lval_wrap(irb, parent_scope, void_inst, lval, result_loc);
	3792	result = ir_lval_wrap(irb, parent_scope, void_inst, lval, result_loc);
3985	3793	}
	3794	if (!is_return_from_fn)
	3795	return result;
	3796
	3797	// no need for save_err_ret_addr because this cannot return error
	3798	// only generate unconditional defers
	3799
	3800	ir_mark_gen(ir_build_add_implicit_return_type(irb, child_scope, block_node, result));
	3801	ir_gen_defers_for_block(irb, child_scope, outer_block_scope, false);
	3802	return ir_mark_gen(ir_build_return(irb, child_scope, result->source_node, result));
3986	3803	}
3987	3804
3988	3805	static IrInstruction ir_gen_bin_op_id(IrBuilder irb, Scope scope, AstNode node, IrBinOp op_id) {
...	...	@@ -4561,8 +4378,6 @@ static IrInstruction ir_gen_builtin_fn_call(IrBuilder irb, Scope *scope, AstNo
4561	4378	return irb->codegen->invalid_instruction;
4562	4379	}
4563	4380
4564		bool is_async = exec_is_async(irb->exec);
4565
4566	4381	switch (builtin_fn->id) {
4567	4382	case BuiltinFnIdInvalid:
4568	4383	zig_unreachable();
...	...	@@ -5185,16 +5000,30 @@ static IrInstruction ir_gen_builtin_fn_call(IrBuilder irb, Scope *scope, AstNo
5185	5000	return ir_lval_wrap(irb, scope, ir_build_return_address(irb, scope, node), lval, result_loc);
5186	5001	case BuiltinFnIdFrameAddress:
5187	5002	return ir_lval_wrap(irb, scope, ir_build_frame_address(irb, scope, node), lval, result_loc);
5188		case BuiltinFnIdHandle:
	5003	case BuiltinFnIdFrameHandle:
5189	5004	if (!irb->exec->fn_entry) {
5190		add_node_error(irb->codegen, node, buf_sprintf("@handle() called outside of function definition"));
5191		return irb->codegen->invalid_instruction;
5192		}
5193		if (!is_async) {
5194		add_node_error(irb->codegen, node, buf_sprintf("@handle() in non-async function"));
	5005	add_node_error(irb->codegen, node, buf_sprintf("@frame() called outside of function definition"));
5195	5006	return irb->codegen->invalid_instruction;
5196	5007	}
5197	5008	return ir_lval_wrap(irb, scope, ir_build_handle(irb, scope, node), lval, result_loc);
	5009	case BuiltinFnIdFrameType: {
	5010	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5011	IrInstruction *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5012	if (arg0_value == irb->codegen->invalid_instruction)
	5013	return arg0_value;
	5014
	5015	IrInstruction *frame_type = ir_build_frame_type(irb, scope, node, arg0_value);
	5016	return ir_lval_wrap(irb, scope, frame_type, lval, result_loc);
	5017	}
	5018	case BuiltinFnIdFrameSize: {
	5019	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5020	IrInstruction *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5021	if (arg0_value == irb->codegen->invalid_instruction)
	5022	return arg0_value;
	5023
	5024	IrInstruction *frame_size = ir_build_frame_size_src(irb, scope, node, arg0_value);
	5025	return ir_lval_wrap(irb, scope, frame_size, lval, result_loc);
	5026	}
5198	5027	case BuiltinFnIdAlignOf:
5199	5028	{
5200	5029	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
...	...	@@ -5395,13 +5224,15 @@ static IrInstruction ir_gen_builtin_fn_call(IrBuilder irb, Scope *scope, AstNo
5395	5224	FnInline fn_inline = (builtin_fn->id == BuiltinFnIdInlineCall) ? FnInlineAlways : FnInlineNever;
5396	5225
5397	5226	IrInstruction *call = ir_build_call_src(irb, scope, node, nullptr, fn_ref, arg_count, args, false,
5398		fn_inline, false, nullptr, nullptr, result_loc);
	5227	fn_inline, false, nullptr, result_loc);
5399	5228	return ir_lval_wrap(irb, scope, call, lval, result_loc);
5400	5229	}
5401	5230	case BuiltinFnIdNewStackCall:
5402	5231	{
5403		if (node->data.fn_call_expr.params.length == 0) {
5404		add_node_error(irb->codegen, node, buf_sprintf("expected at least 1 argument, found 0"));
	5232	if (node->data.fn_call_expr.params.length < 2) {
	5233	add_node_error(irb->codegen, node,
	5234	buf_sprintf("expected at least 2 arguments, found %" ZIG_PRI_usize,
	5235	node->data.fn_call_expr.params.length));
5405	5236	return irb->codegen->invalid_instruction;
5406	5237	}
5407	5238
...	...	@@ -5426,7 +5257,51 @@ static IrInstruction ir_gen_builtin_fn_call(IrBuilder irb, Scope *scope, AstNo
5426	5257	}
5427	5258
5428	5259	IrInstruction *call = ir_build_call_src(irb, scope, node, nullptr, fn_ref, arg_count, args, false,
5429		FnInlineAuto, false, nullptr, new_stack, result_loc);
	5260	FnInlineAuto, false, new_stack, result_loc);
	5261	return ir_lval_wrap(irb, scope, call, lval, result_loc);
	5262	}
	5263	case BuiltinFnIdAsyncCall:
	5264	{
	5265	size_t arg_offset = 3;
	5266	if (node->data.fn_call_expr.params.length < arg_offset) {
	5267	add_node_error(irb->codegen, node,
	5268	buf_sprintf("expected at least %" ZIG_PRI_usize " arguments, found %" ZIG_PRI_usize,
	5269	arg_offset, node->data.fn_call_expr.params.length));
	5270	return irb->codegen->invalid_instruction;
	5271	}
	5272
	5273	AstNode *bytes_node = node->data.fn_call_expr.params.at(0);
	5274	IrInstruction *bytes = ir_gen_node(irb, bytes_node, scope);
	5275	if (bytes == irb->codegen->invalid_instruction)
	5276	return bytes;
	5277
	5278	AstNode *ret_ptr_node = node->data.fn_call_expr.params.at(1);
	5279	IrInstruction *ret_ptr = ir_gen_node(irb, ret_ptr_node, scope);
	5280	if (ret_ptr == irb->codegen->invalid_instruction)
	5281	return ret_ptr;
	5282
	5283	AstNode *fn_ref_node = node->data.fn_call_expr.params.at(2);
	5284	IrInstruction *fn_ref = ir_gen_node(irb, fn_ref_node, scope);
	5285	if (fn_ref == irb->codegen->invalid_instruction)
	5286	return fn_ref;
	5287
	5288	size_t arg_count = node->data.fn_call_expr.params.length - arg_offset;
	5289
	5290	// last "arg" is return pointer
	5291	IrInstruction *args = allocate<IrInstruction>(arg_count + 1);
	5292
	5293	for (size_t i = 0; i < arg_count; i += 1) {
	5294	AstNode *arg_node = node->data.fn_call_expr.params.at(i + arg_offset);
	5295	IrInstruction *arg = ir_gen_node(irb, arg_node, scope);
	5296	if (arg == irb->codegen->invalid_instruction)
	5297	return arg;
	5298	args[i] = arg;
	5299	}
	5300
	5301	args[arg_count] = ret_ptr;
	5302
	5303	IrInstruction *call = ir_build_call_src(irb, scope, node, nullptr, fn_ref, arg_count, args, false,
	5304	FnInlineAuto, true, bytes, result_loc);
5430	5305	return ir_lval_wrap(irb, scope, call, lval, result_loc);
5431	5306	}
5432	5307	case BuiltinFnIdTypeId:
...	...	@@ -5731,17 +5606,8 @@ static IrInstruction ir_gen_fn_call(IrBuilder irb, Scope scope, AstNode node
5731	5606	}
5732	5607
5733	5608	bool is_async = node->data.fn_call_expr.is_async;
5734		IrInstruction *async_allocator = nullptr;
5735		if (is_async) {
5736		if (node->data.fn_call_expr.async_allocator) {
5737		async_allocator = ir_gen_node(irb, node->data.fn_call_expr.async_allocator, scope);
5738		if (async_allocator == irb->codegen->invalid_instruction)
5739		return async_allocator;
5740		}
5741		}
5742
5743		IrInstruction *fn_call = ir_build_call_src(irb, scope, node, nullptr, fn_ref, arg_count, args, false, FnInlineAuto,
5744		is_async, async_allocator, nullptr, result_loc);
	5609	IrInstruction *fn_call = ir_build_call_src(irb, scope, node, nullptr, fn_ref, arg_count, args, false,
	5610	FnInlineAuto, is_async, nullptr, result_loc);
5745	5611	return ir_lval_wrap(irb, scope, fn_call, lval, result_loc);
5746	5612	}
5747	5613
...	...	@@ -6254,7 +6120,8 @@ static IrInstruction ir_gen_while_expr(IrBuilder irb, Scope scope, AstNode n
6254	6120	LValPtr, nullptr);
6255	6121	if (err_val_ptr == irb->codegen->invalid_instruction)
6256	6122	return err_val_ptr;
6257		IrInstruction *is_err = ir_build_test_err_src(irb, scope, node->data.while_expr.condition, err_val_ptr, true);
	6123	IrInstruction *is_err = ir_build_test_err_src(irb, scope, node->data.while_expr.condition, err_val_ptr,
	6124	true, false);
6258	6125	IrBasicBlock *after_cond_block = irb->current_basic_block;
6259	6126	IrInstruction *void_else_result = else_node ? nullptr : ir_mark_gen(ir_build_const_void(irb, scope, node));
6260	6127	IrInstruction *cond_br_inst;
...	...	@@ -6762,10 +6629,10 @@ static IrInstruction ir_gen_array_type(IrBuilder irb, Scope scope, AstNode n
6762	6629	}
6763	6630	}
6764	6631
6765		static IrInstruction ir_gen_promise_type(IrBuilder irb, Scope scope, AstNode node) {
6766		assert(node->type == NodeTypePromiseType);
	6632	static IrInstruction ir_gen_anyframe_type(IrBuilder irb, Scope scope, AstNode node) {
	6633	assert(node->type == NodeTypeAnyFrameType);
6767	6634
6768		AstNode *payload_type_node = node->data.promise_type.payload_type;
	6635	AstNode *payload_type_node = node->data.anyframe_type.payload_type;
6769	6636	IrInstruction *payload_type_value = nullptr;
6770	6637
6771	6638	if (payload_type_node != nullptr) {
...	...	@@ -6775,7 +6642,7 @@ static IrInstruction ir_gen_promise_type(IrBuilder irb, Scope *scope, AstNode
6775	6642
6776	6643	}
6777	6644
6778		return ir_build_promise_type(irb, scope, node, payload_type_value);
	6645	return ir_build_anyframe_type(irb, scope, node, payload_type_value);
6779	6646	}
6780	6647
6781	6648	static IrInstruction ir_gen_undefined_literal(IrBuilder irb, Scope scope, AstNode node) {
...	...	@@ -7070,7 +6937,7 @@ static IrInstruction ir_gen_if_err_expr(IrBuilder irb, Scope scope, AstNode
7070	6937	return err_val_ptr;
7071	6938
7072	6939	IrInstruction *err_val = ir_build_load_ptr(irb, scope, node, err_val_ptr);
7073		IrInstruction *is_err = ir_build_test_err_src(irb, scope, node, err_val_ptr, true);
	6940	IrInstruction *is_err = ir_build_test_err_src(irb, scope, node, err_val_ptr, true, false);
7074	6941
7075	6942	IrBasicBlock *ok_block = ir_create_basic_block(irb, scope, "TryOk");
7076	6943	IrBasicBlock *else_block = ir_create_basic_block(irb, scope, "TryElse");
...	...	@@ -7686,7 +7553,7 @@ static IrInstruction ir_gen_catch(IrBuilder irb, Scope *parent_scope, AstNode
7686	7553	if (err_union_ptr == irb->codegen->invalid_instruction)
7687	7554	return irb->codegen->invalid_instruction;
7688	7555
7689		IrInstruction *is_err = ir_build_test_err_src(irb, parent_scope, node, err_union_ptr, true);
	7556	IrInstruction *is_err = ir_build_test_err_src(irb, parent_scope, node, err_union_ptr, true, false);
7690	7557
7691	7558	IrInstruction *is_comptime;
7692	7559	if (ir_should_inline(irb->exec, parent_scope)) {
...	...	@@ -7967,352 +7834,58 @@ static IrInstruction ir_gen_fn_proto(IrBuilder irb, Scope *parent_scope, AstNo
7967	7834	IrInstruction *return_type;
7968	7835	if (node->data.fn_proto.return_var_token == nullptr) {
7969	7836	if (node->data.fn_proto.return_type == nullptr) {
7970		return_type = ir_build_const_type(irb, parent_scope, node, irb->codegen->builtin_types.entry_void);
7971		} else {
7972		return_type = ir_gen_node(irb, node->data.fn_proto.return_type, parent_scope);
7973		if (return_type == irb->codegen->invalid_instruction)
7974		return irb->codegen->invalid_instruction;
7975		}
7976		} else {
7977		add_node_error(irb->codegen, node,
7978		buf_sprintf("TODO implement inferred return types https://github.com/ziglang/zig/issues/447"));
7979		return irb->codegen->invalid_instruction;
7980		//return_type = nullptr;
7981		}
7982
7983		IrInstruction *async_allocator_type_value = nullptr;
7984		if (node->data.fn_proto.async_allocator_type != nullptr) {
7985		async_allocator_type_value = ir_gen_node(irb, node->data.fn_proto.async_allocator_type, parent_scope);
7986		if (async_allocator_type_value == irb->codegen->invalid_instruction)
7987		return irb->codegen->invalid_instruction;
7988		}
7989
7990		return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type,
7991		async_allocator_type_value, is_var_args);
7992		}
7993
7994		static IrInstruction ir_gen_cancel_target(IrBuilder irb, Scope scope, AstNode node,
7995		IrInstruction *target_inst, bool cancel_non_suspended, bool cancel_awaited)
7996		{
7997		IrBasicBlock *done_block = ir_create_basic_block(irb, scope, "CancelDone");
7998		IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, scope, "NotCanceled");
7999		IrBasicBlock *pre_return_block = ir_create_basic_block(irb, scope, "PreReturn");
8000		IrBasicBlock *post_return_block = ir_create_basic_block(irb, scope, "PostReturn");
8001		IrBasicBlock *do_cancel_block = ir_create_basic_block(irb, scope, "DoCancel");
8002
8003		IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
8004		IrInstruction *usize_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize);
8005		IrInstruction *is_comptime = ir_build_const_bool(irb, scope, node, false);
8006		IrInstruction *is_canceled_mask = ir_build_const_usize(irb, scope, node, 0x1); // 0b001
8007		IrInstruction *promise_T_type_val = ir_build_const_type(irb, scope, node,
8008		get_promise_type(irb->codegen, irb->codegen->builtin_types.entry_void));
8009		IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, scope, node, 0x7); // 0b111
8010		IrInstruction *ptr_mask = ir_build_un_op(irb, scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000
8011		IrInstruction *await_mask = ir_build_const_usize(irb, scope, node, 0x4); // 0b100
8012		IrInstruction *is_suspended_mask = ir_build_const_usize(irb, scope, node, 0x2); // 0b010
8013
8014		// TODO relies on Zig not re-ordering fields
8015		IrInstruction *casted_target_inst = ir_build_ptr_cast_src(irb, scope, node, promise_T_type_val, target_inst,
8016		false);
8017		IrInstruction *coro_promise_ptr = ir_build_coro_promise(irb, scope, node, casted_target_inst);
8018		Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME);
8019		IrInstruction *atomic_state_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr,
8020		atomic_state_field_name, false);
8021
8022		// set the is_canceled bit
8023		IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node,
8024		usize_type_val, atomic_state_ptr, nullptr, is_canceled_mask, nullptr,
8025		AtomicRmwOp_or, AtomicOrderSeqCst);
8026
8027		IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false);
8028		IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false);
8029		ir_build_cond_br(irb, scope, node, is_canceled_bool, done_block, not_canceled_block, is_comptime);
8030
8031		ir_set_cursor_at_end_and_append_block(irb, not_canceled_block);
8032		IrInstruction *awaiter_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false);
8033		IrInstruction *is_returned_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpEq, awaiter_addr, ptr_mask, false);
8034		ir_build_cond_br(irb, scope, node, is_returned_bool, post_return_block, pre_return_block, is_comptime);
8035
8036		ir_set_cursor_at_end_and_append_block(irb, post_return_block);
8037		if (cancel_awaited) {
8038		ir_build_br(irb, scope, node, do_cancel_block, is_comptime);
8039		} else {
8040		IrInstruction *is_awaited_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, await_mask, false);
8041		IrInstruction *is_awaited_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_awaited_value, zero, false);
8042		ir_build_cond_br(irb, scope, node, is_awaited_bool, done_block, do_cancel_block, is_comptime);
8043		}
8044
8045		ir_set_cursor_at_end_and_append_block(irb, pre_return_block);
8046		if (cancel_awaited) {
8047		if (cancel_non_suspended) {
8048		ir_build_br(irb, scope, node, do_cancel_block, is_comptime);
8049		} else {
8050		IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false);
8051		IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false);
8052		ir_build_cond_br(irb, scope, node, is_suspended_bool, do_cancel_block, done_block, is_comptime);
8053		}
8054		} else {
8055		ir_build_br(irb, scope, node, done_block, is_comptime);
8056		}
8057
8058		ir_set_cursor_at_end_and_append_block(irb, do_cancel_block);
8059		ir_build_cancel(irb, scope, node, target_inst);
8060		ir_build_br(irb, scope, node, done_block, is_comptime);
8061
8062		ir_set_cursor_at_end_and_append_block(irb, done_block);
8063		return ir_build_const_void(irb, scope, node);
8064		}
8065
8066		static IrInstruction ir_gen_cancel(IrBuilder irb, Scope scope, AstNode node) {
8067		assert(node->type == NodeTypeCancel);
8068
8069		IrInstruction *target_inst = ir_gen_node(irb, node->data.cancel_expr.expr, scope);
8070		if (target_inst == irb->codegen->invalid_instruction)
	7837	return_type = ir_build_const_type(irb, parent_scope, node, irb->codegen->builtin_types.entry_void);
	7838	} else {
	7839	return_type = ir_gen_node(irb, node->data.fn_proto.return_type, parent_scope);
	7840	if (return_type == irb->codegen->invalid_instruction)
	7841	return irb->codegen->invalid_instruction;
	7842	}
	7843	} else {
	7844	add_node_error(irb->codegen, node,
	7845	buf_sprintf("TODO implement inferred return types https://github.com/ziglang/zig/issues/447"));
8071	7846	return irb->codegen->invalid_instruction;
	7847	//return_type = nullptr;
	7848	}
8072	7849
8073		return ir_gen_cancel_target(irb, scope, node, target_inst, false, true);
8074		}
8075
8076		static IrInstruction ir_gen_resume_target(IrBuilder irb, Scope scope, AstNode node,
8077		IrInstruction *target_inst)
8078		{
8079		IrBasicBlock *done_block = ir_create_basic_block(irb, scope, "ResumeDone");
8080		IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, scope, "NotCanceled");
8081		IrBasicBlock *suspended_block = ir_create_basic_block(irb, scope, "IsSuspended");
8082		IrBasicBlock *not_suspended_block = ir_create_basic_block(irb, scope, "IsNotSuspended");
8083
8084		IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
8085		IrInstruction *is_canceled_mask = ir_build_const_usize(irb, scope, node, 0x1); // 0b001
8086		IrInstruction *is_suspended_mask = ir_build_const_usize(irb, scope, node, 0x2); // 0b010
8087		IrInstruction *and_mask = ir_build_un_op(irb, scope, node, IrUnOpBinNot, is_suspended_mask);
8088		IrInstruction *is_comptime = ir_build_const_bool(irb, scope, node, false);
8089		IrInstruction *usize_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize);
8090		IrInstruction *promise_T_type_val = ir_build_const_type(irb, scope, node,
8091		get_promise_type(irb->codegen, irb->codegen->builtin_types.entry_void));
8092
8093		// TODO relies on Zig not re-ordering fields
8094		IrInstruction *casted_target_inst = ir_build_ptr_cast_src(irb, scope, node, promise_T_type_val, target_inst,
8095		false);
8096		IrInstruction *coro_promise_ptr = ir_build_coro_promise(irb, scope, node, casted_target_inst);
8097		Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME);
8098		IrInstruction *atomic_state_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr,
8099		atomic_state_field_name, false);
8100
8101		// clear the is_suspended bit
8102		IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node,
8103		usize_type_val, atomic_state_ptr, nullptr, and_mask, nullptr,
8104		AtomicRmwOp_and, AtomicOrderSeqCst);
8105
8106		IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false);
8107		IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false);
8108		ir_build_cond_br(irb, scope, node, is_canceled_bool, done_block, not_canceled_block, is_comptime);
8109
8110		ir_set_cursor_at_end_and_append_block(irb, not_canceled_block);
8111		IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false);
8112		IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false);
8113		ir_build_cond_br(irb, scope, node, is_suspended_bool, suspended_block, not_suspended_block, is_comptime);
8114
8115		ir_set_cursor_at_end_and_append_block(irb, not_suspended_block);
8116		ir_build_unreachable(irb, scope, node);
8117
8118		ir_set_cursor_at_end_and_append_block(irb, suspended_block);
8119		ir_build_coro_resume(irb, scope, node, target_inst);
8120		ir_build_br(irb, scope, node, done_block, is_comptime);
8121
8122		ir_set_cursor_at_end_and_append_block(irb, done_block);
8123		return ir_build_const_void(irb, scope, node);
	7850	return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, is_var_args);
8124	7851	}
8125	7852
8126	7853	static IrInstruction ir_gen_resume(IrBuilder irb, Scope scope, AstNode node) {
8127	7854	assert(node->type == NodeTypeResume);
8128	7855
8129		IrInstruction *target_inst = ir_gen_node(irb, node->data.resume_expr.expr, scope);
	7856	IrInstruction *target_inst = ir_gen_node_extra(irb, node->data.resume_expr.expr, scope, LValPtr, nullptr);
8130	7857	if (target_inst == irb->codegen->invalid_instruction)
8131	7858	return irb->codegen->invalid_instruction;
8132	7859
8133		return ir_gen_resume_target(irb, scope, node, target_inst);
	7860	return ir_build_resume(irb, scope, node, target_inst);
8134	7861	}
8135	7862
8136		static IrInstruction ir_gen_await_expr(IrBuilder irb, Scope scope, AstNode node) {
	7863	static IrInstruction ir_gen_await_expr(IrBuilder irb, Scope scope, AstNode node, LVal lval,
	7864	ResultLoc *result_loc)
	7865	{
8137	7866	assert(node->type == NodeTypeAwaitExpr);
8138	7867
8139		IrInstruction *target_inst = ir_gen_node(irb, node->data.await_expr.expr, scope);
8140		if (target_inst == irb->codegen->invalid_instruction)
8141		return irb->codegen->invalid_instruction;
8142
8143	7868	ZigFn *fn_entry = exec_fn_entry(irb->exec);
8144	7869	if (!fn_entry) {
8145	7870	add_node_error(irb->codegen, node, buf_sprintf("await outside function definition"));
8146	7871	return irb->codegen->invalid_instruction;
8147	7872	}
8148		if (fn_entry->type_entry->data.fn.fn_type_id.cc != CallingConventionAsync) {
8149		add_node_error(irb->codegen, node, buf_sprintf("await in non-async function"));
8150		return irb->codegen->invalid_instruction;
8151		}
8152
8153		ScopeDeferExpr *scope_defer_expr = get_scope_defer_expr(scope);
8154		if (scope_defer_expr) {
8155		if (!scope_defer_expr->reported_err) {
8156		add_node_error(irb->codegen, node, buf_sprintf("cannot await inside defer expression"));
8157		scope_defer_expr->reported_err = true;
	7873	ScopeSuspend *existing_suspend_scope = get_scope_suspend(scope);
	7874	if (existing_suspend_scope) {
	7875	if (!existing_suspend_scope->reported_err) {
	7876	ErrorMsg *msg = add_node_error(irb->codegen, node, buf_sprintf("cannot await inside suspend block"));
	7877	add_error_note(irb->codegen, msg, existing_suspend_scope->base.source_node, buf_sprintf("suspend block here"));
	7878	existing_suspend_scope->reported_err = true;
8158	7879	}
8159	7880	return irb->codegen->invalid_instruction;
8160	7881	}
8161	7882
8162		Scope *outer_scope = irb->exec->begin_scope;
	7883	IrInstruction *target_inst = ir_gen_node_extra(irb, node->data.await_expr.expr, scope, LValPtr, nullptr);
	7884	if (target_inst == irb->codegen->invalid_instruction)
	7885	return irb->codegen->invalid_instruction;
8163	7886
8164		IrInstruction *coro_promise_ptr = ir_build_coro_promise(irb, scope, node, target_inst);
8165		Buf *result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME);
8166		IrInstruction *result_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_ptr_field_name, false);
8167
8168		if (irb->codegen->have_err_ret_tracing) {
8169		IrInstruction *err_ret_trace_ptr = ir_build_error_return_trace(irb, scope, node, IrInstructionErrorReturnTrace::NonNull);
8170		Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME);
8171		IrInstruction *err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name, false);
8172		ir_build_store_ptr(irb, scope, node, err_ret_trace_ptr_field_ptr, err_ret_trace_ptr);
8173		}
8174
8175		IrBasicBlock *already_awaited_block = ir_create_basic_block(irb, scope, "AlreadyAwaited");
8176		IrBasicBlock *not_awaited_block = ir_create_basic_block(irb, scope, "NotAwaited");
8177		IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, scope, "NotCanceled");
8178		IrBasicBlock *yes_suspend_block = ir_create_basic_block(irb, scope, "YesSuspend");
8179		IrBasicBlock *no_suspend_block = ir_create_basic_block(irb, scope, "NoSuspend");
8180		IrBasicBlock *merge_block = ir_create_basic_block(irb, scope, "MergeSuspend");
8181		IrBasicBlock *cleanup_block = ir_create_basic_block(irb, scope, "SuspendCleanup");
8182		IrBasicBlock *resume_block = ir_create_basic_block(irb, scope, "SuspendResume");
8183		IrBasicBlock *cancel_target_block = ir_create_basic_block(irb, scope, "CancelTarget");
8184		IrBasicBlock *do_cancel_block = ir_create_basic_block(irb, scope, "DoCancel");
8185		IrBasicBlock *do_defers_block = ir_create_basic_block(irb, scope, "DoDefers");
8186		IrBasicBlock *destroy_block = ir_create_basic_block(irb, scope, "DestroyBlock");
8187		IrBasicBlock *my_suspended_block = ir_create_basic_block(irb, scope, "AlreadySuspended");
8188		IrBasicBlock *my_not_suspended_block = ir_create_basic_block(irb, scope, "NotAlreadySuspended");
8189		IrBasicBlock *do_suspend_block = ir_create_basic_block(irb, scope, "DoSuspend");
8190
8191		Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME);
8192		IrInstruction *atomic_state_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr,
8193		atomic_state_field_name, false);
8194
8195		IrInstruction *promise_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_promise);
8196		IrInstruction *const_bool_false = ir_build_const_bool(irb, scope, node, false);
8197		IrInstruction *undef = ir_build_const_undefined(irb, scope, node);
8198		IrInstruction *usize_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize);
8199		IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
8200		IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, scope, node, 0x7); // 0b111
8201		IrInstruction *ptr_mask = ir_build_un_op(irb, scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000
8202		IrInstruction *await_mask = ir_build_const_usize(irb, scope, node, 0x4); // 0b100
8203		IrInstruction *is_canceled_mask = ir_build_const_usize(irb, scope, node, 0x1); // 0b001
8204		IrInstruction *is_suspended_mask = ir_build_const_usize(irb, scope, node, 0x2); // 0b010
8205
8206		ZigVar *result_var = ir_create_var(irb, node, scope, nullptr,
8207		false, false, true, const_bool_false);
8208		IrInstruction *target_promise_type = ir_build_typeof(irb, scope, node, target_inst);
8209		IrInstruction *promise_result_type = ir_build_promise_result_type(irb, scope, node, target_promise_type);
8210		ir_build_await_bookkeeping(irb, scope, node, promise_result_type);
8211		IrInstruction *undef_promise_result = ir_build_implicit_cast(irb, scope, node, promise_result_type, undef, nullptr);
8212		build_decl_var_and_init(irb, scope, node, result_var, undef_promise_result, "result", const_bool_false);
8213		IrInstruction *my_result_var_ptr = ir_build_var_ptr(irb, scope, node, result_var);
8214		ir_build_store_ptr(irb, scope, node, result_ptr_field_ptr, my_result_var_ptr);
8215		IrInstruction *save_token = ir_build_coro_save(irb, scope, node, irb->exec->coro_handle);
8216
8217		IrInstruction *coro_handle_addr = ir_build_ptr_to_int(irb, scope, node, irb->exec->coro_handle);
8218		IrInstruction *mask_bits = ir_build_bin_op(irb, scope, node, IrBinOpBinOr, coro_handle_addr, await_mask, false);
8219		IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node,
8220		usize_type_val, atomic_state_ptr, nullptr, mask_bits, nullptr,
8221		AtomicRmwOp_or, AtomicOrderSeqCst);
8222
8223		IrInstruction *is_awaited_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, await_mask, false);
8224		IrInstruction *is_awaited_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_awaited_value, zero, false);
8225		ir_build_cond_br(irb, scope, node, is_awaited_bool, already_awaited_block, not_awaited_block, const_bool_false);
8226
8227		ir_set_cursor_at_end_and_append_block(irb, already_awaited_block);
8228		ir_build_unreachable(irb, scope, node);
8229
8230		ir_set_cursor_at_end_and_append_block(irb, not_awaited_block);
8231		IrInstruction *await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false);
8232		IrInstruction *is_non_null = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false);
8233		IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false);
8234		IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false);
8235		ir_build_cond_br(irb, scope, node, is_canceled_bool, cancel_target_block, not_canceled_block, const_bool_false);
8236
8237		ir_set_cursor_at_end_and_append_block(irb, not_canceled_block);
8238		ir_build_cond_br(irb, scope, node, is_non_null, no_suspend_block, yes_suspend_block, const_bool_false);
8239
8240		ir_set_cursor_at_end_and_append_block(irb, cancel_target_block);
8241		ir_build_cancel(irb, scope, node, target_inst);
8242		ir_mark_gen(ir_build_br(irb, scope, node, cleanup_block, const_bool_false));
8243
8244		ir_set_cursor_at_end_and_append_block(irb, no_suspend_block);
8245		if (irb->codegen->have_err_ret_tracing) {
8246		Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME);
8247		IrInstruction *src_err_ret_trace_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_field_name, false);
8248		IrInstruction *dest_err_ret_trace_ptr = ir_build_error_return_trace(irb, scope, node, IrInstructionErrorReturnTrace::NonNull);
8249		ir_build_merge_err_ret_traces(irb, scope, node, coro_promise_ptr, src_err_ret_trace_ptr, dest_err_ret_trace_ptr);
8250		}
8251		Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME);
8252		IrInstruction *promise_result_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name, false);
8253		// If the type of the result handle_is_ptr then this does not actually perform a load. But we need it to,
8254		// because we're about to destroy the memory. So we store it into our result variable.
8255		IrInstruction *no_suspend_result = ir_build_load_ptr(irb, scope, node, promise_result_ptr);
8256		ir_build_store_ptr(irb, scope, node, my_result_var_ptr, no_suspend_result);
8257		ir_build_cancel(irb, scope, node, target_inst);
8258		ir_build_br(irb, scope, node, merge_block, const_bool_false);
8259
8260
8261		ir_set_cursor_at_end_and_append_block(irb, yes_suspend_block);
8262		IrInstruction *my_prev_atomic_value = ir_build_atomic_rmw(irb, scope, node,
8263		usize_type_val, irb->exec->atomic_state_field_ptr, nullptr, is_suspended_mask, nullptr,
8264		AtomicRmwOp_or, AtomicOrderSeqCst);
8265		IrInstruction *my_is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, my_prev_atomic_value, is_suspended_mask, false);
8266		IrInstruction *my_is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, my_is_suspended_value, zero, false);
8267		ir_build_cond_br(irb, scope, node, my_is_suspended_bool, my_suspended_block, my_not_suspended_block, const_bool_false);
8268
8269		ir_set_cursor_at_end_and_append_block(irb, my_suspended_block);
8270		ir_build_unreachable(irb, scope, node);
8271
8272		ir_set_cursor_at_end_and_append_block(irb, my_not_suspended_block);
8273		IrInstruction *my_is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, my_prev_atomic_value, is_canceled_mask, false);
8274		IrInstruction *my_is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, my_is_canceled_value, zero, false);
8275		ir_build_cond_br(irb, scope, node, my_is_canceled_bool, cleanup_block, do_suspend_block, const_bool_false);
8276
8277		ir_set_cursor_at_end_and_append_block(irb, do_suspend_block);
8278		IrInstruction *suspend_code = ir_build_coro_suspend(irb, scope, node, save_token, const_bool_false);
8279
8280		IrInstructionSwitchBrCase *cases = allocate<IrInstructionSwitchBrCase>(2);
8281		cases[0].value = ir_build_const_u8(irb, scope, node, 0);
8282		cases[0].block = resume_block;
8283		cases[1].value = ir_build_const_u8(irb, scope, node, 1);
8284		cases[1].block = destroy_block;
8285		ir_build_switch_br(irb, scope, node, suspend_code, irb->exec->coro_suspend_block,
8286		2, cases, const_bool_false, nullptr);
8287
8288		ir_set_cursor_at_end_and_append_block(irb, destroy_block);
8289		ir_gen_cancel_target(irb, scope, node, target_inst, false, true);
8290		ir_mark_gen(ir_build_br(irb, scope, node, cleanup_block, const_bool_false));
8291
8292		ir_set_cursor_at_end_and_append_block(irb, cleanup_block);
8293		IrInstruction *my_mask_bits = ir_build_bin_op(irb, scope, node, IrBinOpBinOr, ptr_mask, is_canceled_mask, false);
8294		IrInstruction *b_my_prev_atomic_value = ir_build_atomic_rmw(irb, scope, node,
8295		usize_type_val, irb->exec->atomic_state_field_ptr, nullptr, my_mask_bits, nullptr,
8296		AtomicRmwOp_or, AtomicOrderSeqCst);
8297		IrInstruction *my_await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, b_my_prev_atomic_value, ptr_mask, false);
8298		IrInstruction *dont_have_my_await_handle = ir_build_bin_op(irb, scope, node, IrBinOpCmpEq, my_await_handle_addr, zero, false);
8299		IrInstruction *dont_destroy_ourselves = ir_build_bin_op(irb, scope, node, IrBinOpBoolAnd, dont_have_my_await_handle, is_canceled_bool, false);
8300		ir_build_cond_br(irb, scope, node, dont_have_my_await_handle, do_defers_block, do_cancel_block, const_bool_false);
8301
8302		ir_set_cursor_at_end_and_append_block(irb, do_cancel_block);
8303		IrInstruction *my_await_handle = ir_build_int_to_ptr(irb, scope, node, promise_type_val, my_await_handle_addr);
8304		ir_gen_cancel_target(irb, scope, node, my_await_handle, true, false);
8305		ir_mark_gen(ir_build_br(irb, scope, node, do_defers_block, const_bool_false));
8306
8307		ir_set_cursor_at_end_and_append_block(irb, do_defers_block);
8308		ir_gen_defers_for_block(irb, scope, outer_scope, true);
8309		ir_mark_gen(ir_build_cond_br(irb, scope, node, dont_destroy_ourselves, irb->exec->coro_early_final, irb->exec->coro_final_cleanup_block, const_bool_false));
8310
8311		ir_set_cursor_at_end_and_append_block(irb, resume_block);
8312		ir_build_br(irb, scope, node, merge_block, const_bool_false);
8313
8314		ir_set_cursor_at_end_and_append_block(irb, merge_block);
8315		return ir_build_load_ptr(irb, scope, node, my_result_var_ptr);
	7887	IrInstruction *await_inst = ir_build_await_src(irb, scope, node, target_inst, result_loc);
	7888	return ir_lval_wrap(irb, scope, await_inst, lval, result_loc);
8316	7889	}
8317	7890
8318	7891	static IrInstruction ir_gen_suspend(IrBuilder irb, Scope parent_scope, AstNode node) {
...	...	@@ -8323,20 +7896,6 @@ static IrInstruction ir_gen_suspend(IrBuilder irb, Scope *parent_scope, AstNod
8323	7896	add_node_error(irb->codegen, node, buf_sprintf("suspend outside function definition"));
8324	7897	return irb->codegen->invalid_instruction;
8325	7898	}
8326		if (fn_entry->type_entry->data.fn.fn_type_id.cc != CallingConventionAsync) {
8327		add_node_error(irb->codegen, node, buf_sprintf("suspend in non-async function"));
8328		return irb->codegen->invalid_instruction;
8329		}
8330
8331		ScopeDeferExpr *scope_defer_expr = get_scope_defer_expr(parent_scope);
8332		if (scope_defer_expr) {
8333		if (!scope_defer_expr->reported_err) {
8334		ErrorMsg *msg = add_node_error(irb->codegen, node, buf_sprintf("cannot suspend inside defer expression"));
8335		add_error_note(irb->codegen, msg, scope_defer_expr->base.source_node, buf_sprintf("defer here"));
8336		scope_defer_expr->reported_err = true;
8337		}
8338		return irb->codegen->invalid_instruction;
8339		}
8340	7899	ScopeSuspend *existing_suspend_scope = get_scope_suspend(parent_scope);
8341	7900	if (existing_suspend_scope) {
8342	7901	if (!existing_suspend_scope->reported_err) {
...	...	@@ -8347,91 +7906,15 @@ static IrInstruction ir_gen_suspend(IrBuilder irb, Scope *parent_scope, AstNod
8347	7906	return irb->codegen->invalid_instruction;
8348	7907	}
8349	7908
8350		Scope *outer_scope = irb->exec->begin_scope;
8351
8352		IrBasicBlock *cleanup_block = ir_create_basic_block(irb, parent_scope, "SuspendCleanup");
8353		IrBasicBlock *resume_block = ir_create_basic_block(irb, parent_scope, "SuspendResume");
8354		IrBasicBlock *suspended_block = ir_create_basic_block(irb, parent_scope, "AlreadySuspended");
8355		IrBasicBlock *canceled_block = ir_create_basic_block(irb, parent_scope, "IsCanceled");
8356		IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, parent_scope, "NotCanceled");
8357		IrBasicBlock *not_suspended_block = ir_create_basic_block(irb, parent_scope, "NotAlreadySuspended");
8358		IrBasicBlock *cancel_awaiter_block = ir_create_basic_block(irb, parent_scope, "CancelAwaiter");
8359
8360		IrInstruction *promise_type_val = ir_build_const_type(irb, parent_scope, node, irb->codegen->builtin_types.entry_promise);
8361		IrInstruction *const_bool_true = ir_build_const_bool(irb, parent_scope, node, true);
8362		IrInstruction *const_bool_false = ir_build_const_bool(irb, parent_scope, node, false);
8363		IrInstruction *usize_type_val = ir_build_const_type(irb, parent_scope, node, irb->codegen->builtin_types.entry_usize);
8364		IrInstruction *is_canceled_mask = ir_build_const_usize(irb, parent_scope, node, 0x1); // 0b001
8365		IrInstruction *is_suspended_mask = ir_build_const_usize(irb, parent_scope, node, 0x2); // 0b010
8366		IrInstruction *zero = ir_build_const_usize(irb, parent_scope, node, 0);
8367		IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, parent_scope, node, 0x7); // 0b111
8368		IrInstruction *ptr_mask = ir_build_un_op(irb, parent_scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000
8369
8370		IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, parent_scope, node,
8371		usize_type_val, irb->exec->atomic_state_field_ptr, nullptr, is_suspended_mask, nullptr,
8372		AtomicRmwOp_or, AtomicOrderSeqCst);
8373
8374		IrInstruction *is_canceled_value = ir_build_bin_op(irb, parent_scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false);
8375		IrInstruction *is_canceled_bool = ir_build_bin_op(irb, parent_scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false);
8376		ir_build_cond_br(irb, parent_scope, node, is_canceled_bool, canceled_block, not_canceled_block, const_bool_false);
8377
8378		ir_set_cursor_at_end_and_append_block(irb, canceled_block);
8379		IrInstruction *await_handle_addr = ir_build_bin_op(irb, parent_scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false);
8380		IrInstruction *have_await_handle = ir_build_bin_op(irb, parent_scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false);
8381		IrBasicBlock *post_canceled_block = irb->current_basic_block;
8382		ir_build_cond_br(irb, parent_scope, node, have_await_handle, cancel_awaiter_block, cleanup_block, const_bool_false);
8383
8384		ir_set_cursor_at_end_and_append_block(irb, cancel_awaiter_block);
8385		IrInstruction *await_handle = ir_build_int_to_ptr(irb, parent_scope, node, promise_type_val, await_handle_addr);
8386		ir_gen_cancel_target(irb, parent_scope, node, await_handle, true, false);
8387		IrBasicBlock *post_cancel_awaiter_block = irb->current_basic_block;
8388		ir_build_br(irb, parent_scope, node, cleanup_block, const_bool_false);
8389
8390		ir_set_cursor_at_end_and_append_block(irb, not_canceled_block);
8391		IrInstruction *is_suspended_value = ir_build_bin_op(irb, parent_scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false);
8392		IrInstruction *is_suspended_bool = ir_build_bin_op(irb, parent_scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false);
8393		ir_build_cond_br(irb, parent_scope, node, is_suspended_bool, suspended_block, not_suspended_block, const_bool_false);
8394
8395		ir_set_cursor_at_end_and_append_block(irb, suspended_block);
8396		ir_build_unreachable(irb, parent_scope, node);
8397
8398		ir_set_cursor_at_end_and_append_block(irb, not_suspended_block);
8399		IrInstruction *suspend_code;
8400		if (node->data.suspend.block == nullptr) {
8401		suspend_code = ir_build_coro_suspend(irb, parent_scope, node, nullptr, const_bool_false);
8402		} else {
8403		Scope *child_scope;
	7909	IrInstructionSuspendBegin *begin = ir_build_suspend_begin(irb, parent_scope, node);
	7910	if (node->data.suspend.block != nullptr) {
8404	7911	ScopeSuspend *suspend_scope = create_suspend_scope(irb->codegen, node, parent_scope);
8405		suspend_scope->resume_block = resume_block;
8406		child_scope = &suspend_scope->base;
8407		IrInstruction *save_token = ir_build_coro_save(irb, child_scope, node, irb->exec->coro_handle);
8408		ir_gen_node(irb, node->data.suspend.block, child_scope);
8409		suspend_code = ir_mark_gen(ir_build_coro_suspend(irb, parent_scope, node, save_token, const_bool_false));
8410		}
8411
8412		IrInstructionSwitchBrCase *cases = allocate<IrInstructionSwitchBrCase>(2);
8413		cases[0].value = ir_mark_gen(ir_build_const_u8(irb, parent_scope, node, 0));
8414		cases[0].block = resume_block;
8415		cases[1].value = ir_mark_gen(ir_build_const_u8(irb, parent_scope, node, 1));
8416		cases[1].block = canceled_block;
8417		IrInstructionSwitchBr *switch_br = ir_build_switch_br(irb, parent_scope, node, suspend_code,
8418		irb->exec->coro_suspend_block, 2, cases, const_bool_false, nullptr);
8419		ir_mark_gen(&switch_br->base);
8420
8421		ir_set_cursor_at_end_and_append_block(irb, cleanup_block);
8422		IrBasicBlock *incoming_blocks = allocate<IrBasicBlock >(2);
8423		IrInstruction *incoming_values = allocate<IrInstruction >(2);
8424		incoming_blocks[0] = post_canceled_block;
8425		incoming_values[0] = const_bool_true;
8426		incoming_blocks[1] = post_cancel_awaiter_block;
8427		incoming_values[1] = const_bool_false;
8428		IrInstruction *destroy_ourselves = ir_build_phi(irb, parent_scope, node, 2, incoming_blocks, incoming_values,
8429		nullptr);
8430		ir_gen_defers_for_block(irb, parent_scope, outer_scope, true);
8431		ir_mark_gen(ir_build_cond_br(irb, parent_scope, node, destroy_ourselves, irb->exec->coro_final_cleanup_block, irb->exec->coro_early_final, const_bool_false));
	7912	Scope *child_scope = &suspend_scope->base;
	7913	IrInstruction *susp_res = ir_gen_node(irb, node->data.suspend.block, child_scope);
	7914	ir_mark_gen(ir_build_check_statement_is_void(irb, child_scope, node->data.suspend.block, susp_res));
	7915	}
8432	7916
8433		ir_set_cursor_at_end_and_append_block(irb, resume_block);
8434		return ir_mark_gen(ir_build_const_void(irb, parent_scope, node));
	7917	return ir_build_suspend_finish(irb, parent_scope, node, begin);
8435	7918	}
8436	7919
8437	7920	static IrInstruction ir_gen_node_raw(IrBuilder irb, AstNode node, Scope scope,
...	...	@@ -8523,8 +8006,8 @@ static IrInstruction ir_gen_node_raw(IrBuilder irb, AstNode node, Scope scop
8523	8006	return ir_lval_wrap(irb, scope, ir_gen_array_type(irb, scope, node), lval, result_loc);
8524	8007	case NodeTypePointerType:
8525	8008	return ir_lval_wrap(irb, scope, ir_gen_pointer_type(irb, scope, node), lval, result_loc);
8526		case NodeTypePromiseType:
8527		return ir_lval_wrap(irb, scope, ir_gen_promise_type(irb, scope, node), lval, result_loc);
	8009	case NodeTypeAnyFrameType:
	8010	return ir_lval_wrap(irb, scope, ir_gen_anyframe_type(irb, scope, node), lval, result_loc);
8528	8011	case NodeTypeStringLiteral:
8529	8012	return ir_lval_wrap(irb, scope, ir_gen_string_literal(irb, scope, node), lval, result_loc);
8530	8013	case NodeTypeUndefinedLiteral:
...	...	@@ -8561,12 +8044,10 @@ static IrInstruction ir_gen_node_raw(IrBuilder irb, AstNode node, Scope scop
8561	8044	return ir_lval_wrap(irb, scope, ir_gen_fn_proto(irb, scope, node), lval, result_loc);
8562	8045	case NodeTypeErrorSetDecl:
8563	8046	return ir_lval_wrap(irb, scope, ir_gen_err_set_decl(irb, scope, node), lval, result_loc);
8564		case NodeTypeCancel:
8565		return ir_lval_wrap(irb, scope, ir_gen_cancel(irb, scope, node), lval, result_loc);
8566	8047	case NodeTypeResume:
8567	8048	return ir_lval_wrap(irb, scope, ir_gen_resume(irb, scope, node), lval, result_loc);
8568	8049	case NodeTypeAwaitExpr:
8569		return ir_lval_wrap(irb, scope, ir_gen_await_expr(irb, scope, node), lval, result_loc);
	8050	return ir_gen_await_expr(irb, scope, node, lval, result_loc);
8570	8051	case NodeTypeSuspend:
8571	8052	return ir_lval_wrap(irb, scope, ir_gen_suspend(irb, scope, node), lval, result_loc);
8572	8053	case NodeTypeEnumLiteral:
...	...	@@ -8626,235 +8107,22 @@ bool ir_gen(CodeGen codegen, AstNode node, Scope scope, IrExecutable ir_exec
8626	8107
8627	8108	irb->codegen = codegen;
8628	8109	irb->exec = ir_executable;
	8110	irb->main_block_node = node;
8629	8111
8630	8112	IrBasicBlock *entry_block = ir_create_basic_block(irb, scope, "Entry");
8631	8113	ir_set_cursor_at_end_and_append_block(irb, entry_block);
8632	8114	// Entry block gets a reference because we enter it to begin.
8633	8115	ir_ref_bb(irb->current_basic_block);
8634	8116
8635		ZigFn *fn_entry = exec_fn_entry(irb->exec);
8636
8637		bool is_async = fn_entry != nullptr && fn_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync;
8638		IrInstruction *coro_id;
8639		IrInstruction *u8_ptr_type;
8640		IrInstruction *const_bool_false;
8641		IrInstruction *coro_promise_ptr;
8642		IrInstruction *err_ret_trace_ptr;
8643		ZigType *return_type;
8644		Buf *result_ptr_field_name;
8645		ZigVar *coro_size_var;
8646		if (is_async) {
8647		// create the coro promise
8648		Scope *coro_scope = create_coro_prelude_scope(irb->codegen, node, scope);
8649		const_bool_false = ir_build_const_bool(irb, coro_scope, node, false);
8650		ZigVar *promise_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false);
8651
8652		return_type = fn_entry->type_entry->data.fn.fn_type_id.return_type;
8653		IrInstruction *undef = ir_build_const_undefined(irb, coro_scope, node);
8654		// TODO mark this var decl as "no safety" e.g. disable initializing the undef value to 0xaa
8655		ZigType *coro_frame_type = get_promise_frame_type(irb->codegen, return_type);
8656		IrInstruction *coro_frame_type_value = ir_build_const_type(irb, coro_scope, node, coro_frame_type);
8657		IrInstruction *undef_coro_frame = ir_build_implicit_cast(irb, coro_scope, node, coro_frame_type_value, undef, nullptr);
8658		build_decl_var_and_init(irb, coro_scope, node, promise_var, undef_coro_frame, "promise", const_bool_false);
8659		coro_promise_ptr = ir_build_var_ptr(irb, coro_scope, node, promise_var);
8660
8661		ZigVar *await_handle_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false);
8662		IrInstruction *null_value = ir_build_const_null(irb, coro_scope, node);
8663		IrInstruction *await_handle_type_val = ir_build_const_type(irb, coro_scope, node,
8664		get_optional_type(irb->codegen, irb->codegen->builtin_types.entry_promise));
8665		IrInstruction *null_await_handle = ir_build_implicit_cast(irb, coro_scope, node, await_handle_type_val, null_value, nullptr);
8666		build_decl_var_and_init(irb, coro_scope, node, await_handle_var, null_await_handle, "await_handle", const_bool_false);
8667		irb->exec->await_handle_var_ptr = ir_build_var_ptr(irb, coro_scope, node, await_handle_var);
8668
8669		u8_ptr_type = ir_build_const_type(irb, coro_scope, node,
8670		get_pointer_to_type(irb->codegen, irb->codegen->builtin_types.entry_u8, false));
8671		IrInstruction *promise_as_u8_ptr = ir_build_ptr_cast_src(irb, coro_scope, node, u8_ptr_type,
8672		coro_promise_ptr, false);
8673		coro_id = ir_build_coro_id(irb, coro_scope, node, promise_as_u8_ptr);
8674		coro_size_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false);
8675		IrInstruction *coro_size = ir_build_coro_size(irb, coro_scope, node);
8676		build_decl_var_and_init(irb, coro_scope, node, coro_size_var, coro_size, "coro_size", const_bool_false);
8677		IrInstruction *implicit_allocator_ptr = ir_build_get_implicit_allocator(irb, coro_scope, node,
8678		ImplicitAllocatorIdArg);
8679		irb->exec->coro_allocator_var = ir_create_var(irb, node, coro_scope, nullptr, true, true, true, const_bool_false);
8680		build_decl_var_and_init(irb, coro_scope, node, irb->exec->coro_allocator_var, implicit_allocator_ptr,
8681		"allocator", const_bool_false);
8682		Buf *realloc_field_name = buf_create_from_str(ASYNC_REALLOC_FIELD_NAME);
8683		IrInstruction *realloc_fn_ptr = ir_build_field_ptr(irb, coro_scope, node, implicit_allocator_ptr, realloc_field_name, false);
8684		IrInstruction *realloc_fn = ir_build_load_ptr(irb, coro_scope, node, realloc_fn_ptr);
8685		IrInstruction *maybe_coro_mem_ptr = ir_build_coro_alloc_helper(irb, coro_scope, node, realloc_fn, coro_size);
8686		IrInstruction *alloc_result_is_ok = ir_build_test_nonnull(irb, coro_scope, node, maybe_coro_mem_ptr);
8687		IrBasicBlock *alloc_err_block = ir_create_basic_block(irb, coro_scope, "AllocError");
8688		IrBasicBlock *alloc_ok_block = ir_create_basic_block(irb, coro_scope, "AllocOk");
8689		ir_build_cond_br(irb, coro_scope, node, alloc_result_is_ok, alloc_ok_block, alloc_err_block, const_bool_false);
8690
8691		ir_set_cursor_at_end_and_append_block(irb, alloc_err_block);
8692		// we can return undefined here, because the caller passes a pointer to the error struct field
8693		// in the error union result, and we populate it in case of allocation failure.
8694		ir_build_return(irb, coro_scope, node, undef);
8695
8696		ir_set_cursor_at_end_and_append_block(irb, alloc_ok_block);
8697		IrInstruction *coro_mem_ptr = ir_build_ptr_cast_src(irb, coro_scope, node, u8_ptr_type, maybe_coro_mem_ptr,
8698		false);
8699		irb->exec->coro_handle = ir_build_coro_begin(irb, coro_scope, node, coro_id, coro_mem_ptr);
8700
8701		Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME);
8702		irb->exec->atomic_state_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr,
8703		atomic_state_field_name, false);
8704		IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
8705		ir_build_store_ptr(irb, scope, node, irb->exec->atomic_state_field_ptr, zero);
8706		Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME);
8707		irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name, false);
8708		result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME);
8709		irb->exec->coro_result_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_ptr_field_name, false);
8710		ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_ptr_field_ptr, irb->exec->coro_result_field_ptr);
8711		if (irb->codegen->have_err_ret_tracing) {
8712		// initialize the error return trace
8713		Buf *return_addresses_field_name = buf_create_from_str(RETURN_ADDRESSES_FIELD_NAME);
8714		IrInstruction *return_addresses_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, return_addresses_field_name, false);
8715
8716		Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME);
8717		err_ret_trace_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_field_name, false);
8718		ir_build_mark_err_ret_trace_ptr(irb, scope, node, err_ret_trace_ptr);
8719
8720		// coordinate with builtin.zig
8721		Buf *index_name = buf_create_from_str("index");
8722		IrInstruction *index_ptr = ir_build_field_ptr(irb, scope, node, err_ret_trace_ptr, index_name, false);
8723		ir_build_store_ptr(irb, scope, node, index_ptr, zero);
8724
8725		Buf *instruction_addresses_name = buf_create_from_str("instruction_addresses");
8726		IrInstruction *addrs_slice_ptr = ir_build_field_ptr(irb, scope, node, err_ret_trace_ptr, instruction_addresses_name, false);
8727
8728		IrInstruction *slice_value = ir_build_slice_src(irb, scope, node, return_addresses_ptr, zero, nullptr, false, no_result_loc());
8729		ir_build_store_ptr(irb, scope, node, addrs_slice_ptr, slice_value);
8730		}
8731
8732
8733		irb->exec->coro_early_final = ir_create_basic_block(irb, scope, "CoroEarlyFinal");
8734		irb->exec->coro_normal_final = ir_create_basic_block(irb, scope, "CoroNormalFinal");
8735		irb->exec->coro_suspend_block = ir_create_basic_block(irb, scope, "Suspend");
8736		irb->exec->coro_final_cleanup_block = ir_create_basic_block(irb, scope, "FinalCleanup");
8737		}
8738
8739	8117	IrInstruction *result = ir_gen_node_extra(irb, node, scope, LValNone, nullptr);
8740	8118	assert(result);
8741	8119	if (irb->exec->invalid)
8742	8120	return false;
8743	8121
8744	8122	if (!instr_is_unreachable(result)) {
	8123	ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, result->source_node, result));
8745	8124	// no need for save_err_ret_addr because this cannot return error
8746		ir_gen_async_return(irb, scope, result->source_node, result, true);
8747		}
8748
8749		if (is_async) {
8750		IrBasicBlock *invalid_resume_block = ir_create_basic_block(irb, scope, "InvalidResume");
8751		IrBasicBlock *check_free_block = ir_create_basic_block(irb, scope, "CheckFree");
8752
8753		ir_set_cursor_at_end_and_append_block(irb, irb->exec->coro_early_final);
8754		IrInstruction *const_bool_true = ir_build_const_bool(irb, scope, node, true);
8755		IrInstruction *suspend_code = ir_build_coro_suspend(irb, scope, node, nullptr, const_bool_true);
8756		IrInstructionSwitchBrCase *cases = allocate<IrInstructionSwitchBrCase>(2);
8757		cases[0].value = ir_build_const_u8(irb, scope, node, 0);
8758		cases[0].block = invalid_resume_block;
8759		cases[1].value = ir_build_const_u8(irb, scope, node, 1);
8760		cases[1].block = irb->exec->coro_final_cleanup_block;
8761		ir_build_switch_br(irb, scope, node, suspend_code, irb->exec->coro_suspend_block, 2, cases, const_bool_false, nullptr);
8762
8763		ir_set_cursor_at_end_and_append_block(irb, irb->exec->coro_suspend_block);
8764		ir_build_coro_end(irb, scope, node);
8765		ir_build_return(irb, scope, node, irb->exec->coro_handle);
8766
8767		ir_set_cursor_at_end_and_append_block(irb, invalid_resume_block);
8768		ir_build_unreachable(irb, scope, node);
8769
8770		ir_set_cursor_at_end_and_append_block(irb, irb->exec->coro_normal_final);
8771		if (type_has_bits(return_type)) {
8772		IrInstruction *u8_ptr_type_unknown_len = ir_build_const_type(irb, scope, node,
8773		get_pointer_to_type_extra(irb->codegen, irb->codegen->builtin_types.entry_u8,
8774		false, false, PtrLenUnknown, 0, 0, 0, false));
8775		IrInstruction *result_ptr = ir_build_load_ptr(irb, scope, node, irb->exec->coro_result_ptr_field_ptr);
8776		IrInstruction *result_ptr_as_u8_ptr = ir_build_ptr_cast_src(irb, scope, node, u8_ptr_type_unknown_len,
8777		result_ptr, false);
8778		IrInstruction *return_value_ptr_as_u8_ptr = ir_build_ptr_cast_src(irb, scope, node,
8779		u8_ptr_type_unknown_len, irb->exec->coro_result_field_ptr, false);
8780		IrInstruction *return_type_inst = ir_build_const_type(irb, scope, node,
8781		fn_entry->type_entry->data.fn.fn_type_id.return_type);
8782		IrInstruction *size_of_ret_val = ir_build_size_of(irb, scope, node, return_type_inst);
8783		ir_build_memcpy(irb, scope, node, result_ptr_as_u8_ptr, return_value_ptr_as_u8_ptr, size_of_ret_val);
8784		}
8785		if (irb->codegen->have_err_ret_tracing) {
8786		Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME);
8787		IrInstruction *err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name, false);
8788		IrInstruction *dest_err_ret_trace_ptr = ir_build_load_ptr(irb, scope, node, err_ret_trace_ptr_field_ptr);
8789		ir_build_merge_err_ret_traces(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr, dest_err_ret_trace_ptr);
8790		}
8791		// Before we destroy the coroutine frame, we need to load the target promise into
8792		// a register or local variable which does not get spilled into the frame,
8793		// otherwise llvm tries to access memory inside the destroyed frame.
8794		IrInstruction *unwrapped_await_handle_ptr = ir_build_optional_unwrap_ptr(irb, scope, node,
8795		irb->exec->await_handle_var_ptr, false, false);
8796		IrInstruction *await_handle_in_block = ir_build_load_ptr(irb, scope, node, unwrapped_await_handle_ptr);
8797		ir_build_br(irb, scope, node, check_free_block, const_bool_false);
8798
8799		ir_set_cursor_at_end_and_append_block(irb, irb->exec->coro_final_cleanup_block);
8800		ir_build_br(irb, scope, node, check_free_block, const_bool_false);
8801
8802		ir_set_cursor_at_end_and_append_block(irb, check_free_block);
8803		IrBasicBlock *incoming_blocks = allocate<IrBasicBlock >(2);
8804		IrInstruction *incoming_values = allocate<IrInstruction >(2);
8805		incoming_blocks[0] = irb->exec->coro_final_cleanup_block;
8806		incoming_values[0] = const_bool_false;
8807		incoming_blocks[1] = irb->exec->coro_normal_final;
8808		incoming_values[1] = const_bool_true;
8809		IrInstruction *resume_awaiter = ir_build_phi(irb, scope, node, 2, incoming_blocks, incoming_values, nullptr);
8810
8811		IrBasicBlock *merge_incoming_blocks = allocate<IrBasicBlock >(2);
8812		IrInstruction *merge_incoming_values = allocate<IrInstruction >(2);
8813		merge_incoming_blocks[0] = irb->exec->coro_final_cleanup_block;
8814		merge_incoming_values[0] = ir_build_const_undefined(irb, scope, node);
8815		merge_incoming_blocks[1] = irb->exec->coro_normal_final;
8816		merge_incoming_values[1] = await_handle_in_block;
8817		IrInstruction *awaiter_handle = ir_build_phi(irb, scope, node, 2, merge_incoming_blocks, merge_incoming_values, nullptr);
8818
8819		Buf *shrink_field_name = buf_create_from_str(ASYNC_SHRINK_FIELD_NAME);
8820		IrInstruction *implicit_allocator_ptr = ir_build_get_implicit_allocator(irb, scope, node,
8821		ImplicitAllocatorIdLocalVar);
8822		IrInstruction *shrink_fn_ptr = ir_build_field_ptr(irb, scope, node, implicit_allocator_ptr, shrink_field_name, false);
8823		IrInstruction *shrink_fn = ir_build_load_ptr(irb, scope, node, shrink_fn_ptr);
8824		IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
8825		IrInstruction *coro_mem_ptr_maybe = ir_build_coro_free(irb, scope, node, coro_id, irb->exec->coro_handle);
8826		IrInstruction *u8_ptr_type_unknown_len = ir_build_const_type(irb, scope, node,
8827		get_pointer_to_type_extra(irb->codegen, irb->codegen->builtin_types.entry_u8,
8828		false, false, PtrLenUnknown, 0, 0, 0, false));
8829		IrInstruction *coro_mem_ptr = ir_build_ptr_cast_src(irb, scope, node, u8_ptr_type_unknown_len,
8830		coro_mem_ptr_maybe, false);
8831		IrInstruction *coro_mem_ptr_ref = ir_build_ref(irb, scope, node, coro_mem_ptr, true, false);
8832		IrInstruction *coro_size_ptr = ir_build_var_ptr(irb, scope, node, coro_size_var);
8833		IrInstruction *coro_size = ir_build_load_ptr(irb, scope, node, coro_size_ptr);
8834		IrInstruction *mem_slice = ir_build_slice_src(irb, scope, node, coro_mem_ptr_ref, zero, coro_size, false,
8835		no_result_loc());
8836		size_t arg_count = 5;
8837		IrInstruction *args = allocate<IrInstruction >(arg_count);
8838		args[0] = implicit_allocator_ptr; // self
8839		args[1] = mem_slice; // old_mem
8840		args[2] = ir_build_const_usize(irb, scope, node, 8); // old_align
8841		// TODO: intentional memory leak here. If this is set to 0 then there is an issue where a coroutine
8842		// calls the function and it frees its own stack frame, but then the return value is a slice, which
8843		// is implemented as an sret struct. writing to the return pointer causes invalid memory write.
8844		// We could work around it by having a global helper function which has a void return type
8845		// and calling that instead. But instead this hack will suffice until I rework coroutines to be
8846		// non-allocating. Basically coroutines are not supported right now until they are reworked.
8847		args[3] = ir_build_const_usize(irb, scope, node, 1); // new_size
8848		args[4] = ir_build_const_usize(irb, scope, node, 1); // new_align
8849		ir_build_call_src(irb, scope, node, nullptr, shrink_fn, arg_count, args, false, FnInlineAuto, false, nullptr,
8850		nullptr, no_result_loc());
8851
8852		IrBasicBlock *resume_block = ir_create_basic_block(irb, scope, "Resume");
8853		ir_build_cond_br(irb, scope, node, resume_awaiter, resume_block, irb->exec->coro_suspend_block, const_bool_false);
8854
8855		ir_set_cursor_at_end_and_append_block(irb, resume_block);
8856		ir_gen_resume_target(irb, scope, node, awaiter_handle);
8857		ir_build_br(irb, scope, node, irb->exec->coro_suspend_block, const_bool_false);
	8125	ir_mark_gen(ir_build_return(irb, scope, result->source_node, result));
8858	8126	}
8859	8127
8860	8128	return true;
...	...	@@ -8871,18 +8139,24 @@ bool ir_gen_fn(CodeGen codegen, ZigFn fn_entry) {
8871	8139	return ir_gen(codegen, body_node, fn_entry->child_scope, ir_executable);
8872	8140	}
8873	8141
8874		static void add_call_stack_errors(CodeGen codegen, IrExecutable exec, ErrorMsg *err_msg, int limit) {
	8142	static void ir_add_call_stack_errors(CodeGen codegen, IrExecutable exec, ErrorMsg *err_msg, int limit) {
8875	8143	if (!exec \|\| !exec->source_node \|\| limit < 0) return;
8876	8144	add_error_note(codegen, err_msg, exec->source_node, buf_sprintf("called from here"));
8877	8145
8878		add_call_stack_errors(codegen, exec->parent_exec, err_msg, limit - 1);
	8146	ir_add_call_stack_errors(codegen, exec->parent_exec, err_msg, limit - 1);
	8147	}
	8148
	8149	void ir_add_analysis_trace(IrAnalyze ira, ErrorMsg err_msg, Buf *text) {
	8150	IrInstruction *old_instruction = ira->old_irb.current_basic_block->instruction_list.at(ira->instruction_index);
	8151	add_error_note(ira->codegen, err_msg, old_instruction->source_node, text);
	8152	ir_add_call_stack_errors(ira->codegen, ira->new_irb.exec, err_msg, 10);
8879	8153	}
8880	8154
8881	8155	static ErrorMsg exec_add_error_node(CodeGen codegen, IrExecutable exec, AstNode source_node, Buf *msg) {
8882	8156	invalidate_exec(exec);
8883	8157	ErrorMsg *err_msg = add_node_error(codegen, source_node, msg);
8884	8158	if (exec->parent_exec) {
8885		add_call_stack_errors(codegen, exec, err_msg, 10);
	8159	ir_add_call_stack_errors(codegen, exec, err_msg, 10);
8886	8160	}
8887	8161	return err_msg;
8888	8162	}
...	...	@@ -8946,13 +8220,13 @@ static ConstExprValue ir_exec_const_result(CodeGen codegen, IrExecutable *exec
8946	8220	IrInstruction *instruction = bb->instruction_list.at(i);
8947	8221	if (instruction->id == IrInstructionIdReturn) {
8948	8222	IrInstructionReturn ret_inst = (IrInstructionReturn )instruction;
8949		IrInstruction *value = ret_inst->value;
8950		if (value->value.special == ConstValSpecialRuntime) {
8951		exec_add_error_node(codegen, exec, value->source_node,
	8223	IrInstruction *operand = ret_inst->operand;
	8224	if (operand->value.special == ConstValSpecialRuntime) {
	8225	exec_add_error_node(codegen, exec, operand->source_node,
8952	8226	buf_sprintf("unable to evaluate constant expression"));
8953	8227	return &codegen->invalid_instruction->value;
8954	8228	}
8955		return &value->value;
	8229	return &operand->value;
8956	8230	} else if (ir_has_side_effects(instruction)) {
8957	8231	if (instr_is_comptime(instruction)) {
8958	8232	switch (instruction->id) {
...	...	@@ -10203,12 +9477,6 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze ira, ZigType wanted
10203	9477	return result;
10204	9478	}
10205	9479
10206		if (wanted_type == ira->codegen->builtin_types.entry_promise &&
10207		actual_type->id == ZigTypeIdPromise)
10208		{
10209		return result;
10210		}
10211
10212	9480	// fn
10213	9481	if (wanted_type->id == ZigTypeIdFn &&
10214	9482	actual_type->id == ZigTypeIdFn)
...	...	@@ -10243,20 +9511,6 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze ira, ZigType wanted
10243	9511	return result;
10244	9512	}
10245	9513	}
10246		if (!wanted_type->data.fn.is_generic && wanted_type->data.fn.fn_type_id.cc == CallingConventionAsync) {
10247		ConstCastOnly child = types_match_const_cast_only(ira,
10248		actual_type->data.fn.fn_type_id.async_allocator_type,
10249		wanted_type->data.fn.fn_type_id.async_allocator_type,
10250		source_node, false);
10251		if (child.id == ConstCastResultIdInvalid)
10252		return child;
10253		if (child.id != ConstCastResultIdOk) {
10254		result.id = ConstCastResultIdAsyncAllocatorType;
10255		result.data.async_allocator_type = allocate_nonzero<ConstCastOnly>(1);
10256		*result.data.async_allocator_type = child;
10257		return result;
10258		}
10259		}
10260	9514	if (wanted_type->data.fn.fn_type_id.param_count != actual_type->data.fn.fn_type_id.param_count) {
10261	9515	result.id = ConstCastResultIdFnArgCount;
10262	9516	return result;
...	...	@@ -10561,6 +9815,8 @@ static ZigType ir_resolve_peer_types(IrAnalyze ira, AstNode *source_node, ZigT
10561	9815
10562	9816	ZigType *prev_err_set_type = (err_set_type == nullptr) ? prev_type->data.error_union.err_set_type : err_set_type;
10563	9817	ZigType *cur_err_set_type = cur_type->data.error_union.err_set_type;
	9818	if (prev_err_set_type == cur_err_set_type)
	9819	continue;
10564	9820
10565	9821	if (!resolve_inferred_error_set(ira->codegen, prev_err_set_type, cur_inst->source_node)) {
10566	9822	return ira->codegen->builtin_types.entry_invalid;
...	...	@@ -11206,7 +10462,7 @@ static IrBasicBlock ir_get_new_bb_runtime(IrAnalyze ira, IrBasicBlock *old_bb,
11206	10462	}
11207	10463
11208	10464	static void ir_start_bb(IrAnalyze ira, IrBasicBlock old_bb, IrBasicBlock *const_predecessor_bb) {
11209		ir_assert(!old_bb->suspended, old_bb->instruction_list.at(0));
	10465	ir_assert(!old_bb->suspended, (old_bb->instruction_list.length != 0) ? old_bb->instruction_list.at(0) : nullptr);
11210	10466	ira->instruction_index = 0;
11211	10467	ira->old_irb.current_basic_block = old_bb;
11212	10468	ira->const_predecessor_bb = const_predecessor_bb;
...	...	@@ -11729,6 +10985,33 @@ static IrInstruction ir_analyze_err_set_cast(IrAnalyze ira, IrInstruction *sou
11729	10985	return result;
11730	10986	}
11731	10987
	10988	static IrInstruction ir_analyze_frame_ptr_to_anyframe(IrAnalyze ira, IrInstruction *source_instr,
	10989	IrInstruction value, ZigType wanted_type)
	10990	{
	10991	if (instr_is_comptime(value)) {
	10992	zig_panic("TODO comptime frame pointer");
	10993	}
	10994
	10995	IrInstruction *result = ir_build_cast(&ira->new_irb, source_instr->scope, source_instr->source_node,
	10996	wanted_type, value, CastOpBitCast);
	10997	result->value.type = wanted_type;
	10998	return result;
	10999	}
	11000
	11001	static IrInstruction ir_analyze_anyframe_to_anyframe(IrAnalyze ira, IrInstruction *source_instr,
	11002	IrInstruction value, ZigType wanted_type)
	11003	{
	11004	if (instr_is_comptime(value)) {
	11005	zig_panic("TODO comptime anyframe->T to anyframe");
	11006	}
	11007
	11008	IrInstruction *result = ir_build_cast(&ira->new_irb, source_instr->scope, source_instr->source_node,
	11009	wanted_type, value, CastOpBitCast);
	11010	result->value.type = wanted_type;
	11011	return result;
	11012	}
	11013
	11014
11732	11015	static IrInstruction ir_analyze_err_wrap_code(IrAnalyze ira, IrInstruction source_instr, IrInstruction value,
11733	11016	ZigType wanted_type, ResultLoc result_loc)
11734	11017	{
...	...	@@ -12576,12 +11859,10 @@ static IrInstruction ir_analyze_int_to_c_ptr(IrAnalyze ira, IrInstruction *sou
12576	11859	static bool is_pointery_and_elem_is_not_pointery(ZigType *ty) {
12577	11860	if (ty->id == ZigTypeIdPointer) return ty->data.pointer.child_type->id != ZigTypeIdPointer;
12578	11861	if (ty->id == ZigTypeIdFn) return true;
12579		if (ty->id == ZigTypeIdPromise) return true;
12580	11862	if (ty->id == ZigTypeIdOptional) {
12581	11863	ZigType *ptr_ty = ty->data.maybe.child_type;
12582	11864	if (ptr_ty->id == ZigTypeIdPointer) return ptr_ty->data.pointer.child_type->id != ZigTypeIdPointer;
12583	11865	if (ptr_ty->id == ZigTypeIdFn) return true;
12584		if (ptr_ty->id == ZigTypeIdPromise) return true;
12585	11866	}
12586	11867	return false;
12587	11868	}
...	...	@@ -12829,6 +12110,29 @@ static IrInstruction ir_analyze_cast(IrAnalyze ira, IrInstruction *source_inst
12829	12110	}
12830	12111	}
12831	12112
	12113	// *@Frame(func) to anyframe->T or anyframe
	12114	if (actual_type->id == ZigTypeIdPointer && actual_type->data.pointer.ptr_len == PtrLenSingle &&
	12115	actual_type->data.pointer.child_type->id == ZigTypeIdFnFrame && wanted_type->id == ZigTypeIdAnyFrame)
	12116	{
	12117	bool ok = true;
	12118	if (wanted_type->data.any_frame.result_type != nullptr) {
	12119	ZigFn *fn = actual_type->data.pointer.child_type->data.frame.fn;
	12120	ZigType *fn_return_type = fn->type_entry->data.fn.fn_type_id.return_type;
	12121	if (wanted_type->data.any_frame.result_type != fn_return_type) {
	12122	ok = false;
	12123	}
	12124	}
	12125	if (ok) {
	12126	return ir_analyze_frame_ptr_to_anyframe(ira, source_instr, value, wanted_type);
	12127	}
	12128	}
	12129
	12130	// anyframe->T to anyframe
	12131	if (actual_type->id == ZigTypeIdAnyFrame && actual_type->data.any_frame.result_type != nullptr &&
	12132	wanted_type->id == ZigTypeIdAnyFrame && wanted_type->data.any_frame.result_type == nullptr)
	12133	{
	12134	return ir_analyze_anyframe_to_anyframe(ira, source_instr, value, wanted_type);
	12135	}
12832	12136
12833	12137	// cast from null literal to maybe type
12834	12138	if (wanted_type->id == ZigTypeIdOptional &&
...	...	@@ -13333,11 +12637,11 @@ static IrInstruction *ir_analyze_instruction_add_implicit_return_type(IrAnalyze
13333	12637	}
13334	12638
13335	12639	static IrInstruction ir_analyze_instruction_return(IrAnalyze ira, IrInstructionReturn *instruction) {
13336		IrInstruction *value = instruction->value->child;
13337		if (type_is_invalid(value->value.type))
	12640	IrInstruction *operand = instruction->operand->child;
	12641	if (type_is_invalid(operand->value.type))
13338	12642	return ir_unreach_error(ira);
13339	12643
13340		if (!instr_is_comptime(value) && handle_is_ptr(ira->explicit_return_type)) {
	12644	if (!instr_is_comptime(operand) && handle_is_ptr(ira->explicit_return_type)) {
13341	12645	// result location mechanism took care of it.
13342	12646	IrInstruction *result = ir_build_return(&ira->new_irb, instruction->base.scope,
13343	12647	instruction->base.source_node, nullptr);
...	...	@@ -13345,8 +12649,8 @@ static IrInstruction ir_analyze_instruction_return(IrAnalyze ira, IrInstructio
13345	12649	return ir_finish_anal(ira, result);
13346	12650	}
13347	12651
13348		IrInstruction *casted_value = ir_implicit_cast(ira, value, ira->explicit_return_type);
13349		if (type_is_invalid(casted_value->value.type)) {
	12652	IrInstruction *casted_operand = ir_implicit_cast(ira, operand, ira->explicit_return_type);
	12653	if (type_is_invalid(casted_operand->value.type)) {
13350	12654	AstNode *source_node = ira->explicit_return_type_source_node;
13351	12655	if (source_node != nullptr) {
13352	12656	ErrorMsg *msg = ira->codegen->errors.last();
...	...	@@ -13356,15 +12660,16 @@ static IrInstruction ir_analyze_instruction_return(IrAnalyze ira, IrInstructio
13356	12660	return ir_unreach_error(ira);
13357	12661	}
13358	12662
13359		if (casted_value->value.special == ConstValSpecialRuntime &&
13360		casted_value->value.type->id == ZigTypeIdPointer &&
13361		casted_value->value.data.rh_ptr == RuntimeHintPtrStack)
	12663	if (casted_operand->value.special == ConstValSpecialRuntime &&
	12664	casted_operand->value.type->id == ZigTypeIdPointer &&
	12665	casted_operand->value.data.rh_ptr == RuntimeHintPtrStack)
13362	12666	{
13363		ir_add_error(ira, casted_value, buf_sprintf("function returns address of local variable"));
	12667	ir_add_error(ira, casted_operand, buf_sprintf("function returns address of local variable"));
13364	12668	return ir_unreach_error(ira);
13365	12669	}
	12670
13366	12671	IrInstruction *result = ir_build_return(&ira->new_irb, instruction->base.scope,
13367		instruction->base.source_node, casted_value);
	12672	instruction->base.source_node, casted_operand);
13368	12673	result->value.type = ira->codegen->builtin_types.entry_unreachable;
13369	12674	return ir_finish_anal(ira, result);
13370	12675	}
...	...	@@ -13658,9 +12963,9 @@ static IrInstruction ir_analyze_bin_op_cmp(IrAnalyze ira, IrInstructionBinOp *
13658	12963	case ZigTypeIdOpaque:
13659	12964	case ZigTypeIdBoundFn:
13660	12965	case ZigTypeIdArgTuple:
13661		case ZigTypeIdPromise:
13662	12966	case ZigTypeIdEnum:
13663	12967	case ZigTypeIdEnumLiteral:
	12968	case ZigTypeIdAnyFrame:
13664	12969	operator_allowed = is_equality_cmp;
13665	12970	break;
13666	12971
...	...	@@ -13675,6 +12980,7 @@ static IrInstruction ir_analyze_bin_op_cmp(IrAnalyze ira, IrInstructionBinOp *
13675	12980	case ZigTypeIdNull:
13676	12981	case ZigTypeIdErrorUnion:
13677	12982	case ZigTypeIdUnion:
	12983	case ZigTypeIdFnFrame:
13678	12984	operator_allowed = false;
13679	12985	break;
13680	12986	case ZigTypeIdOptional:
...	...	@@ -15039,7 +14345,8 @@ static IrInstruction ir_analyze_instruction_export(IrAnalyze ira, IrInstructio
15039	14345	case ZigTypeIdBoundFn:
15040	14346	case ZigTypeIdArgTuple:
15041	14347	case ZigTypeIdOpaque:
15042		case ZigTypeIdPromise:
	14348	case ZigTypeIdFnFrame:
	14349	case ZigTypeIdAnyFrame:
15043	14350	ir_add_error(ira, target,
15044	14351	buf_sprintf("invalid export target '%s'", buf_ptr(&type_value->name)));
15045	14352	break;
...	...	@@ -15063,8 +14370,9 @@ static IrInstruction ir_analyze_instruction_export(IrAnalyze ira, IrInstructio
15063	14370	case ZigTypeIdBoundFn:
15064	14371	case ZigTypeIdArgTuple:
15065	14372	case ZigTypeIdOpaque:
15066		case ZigTypeIdPromise:
15067	14373	case ZigTypeIdEnumLiteral:
	14374	case ZigTypeIdFnFrame:
	14375	case ZigTypeIdAnyFrame:
15068	14376	ir_add_error(ira, target,
15069	14377	buf_sprintf("invalid export target type '%s'", buf_ptr(&target->value.type->name)));
15070	14378	break;
...	...	@@ -15091,8 +14399,8 @@ static bool exec_has_err_ret_trace(CodeGen g, IrExecutable exec) {
15091	14399	static IrInstruction ir_analyze_instruction_error_return_trace(IrAnalyze ira,
15092	14400	IrInstructionErrorReturnTrace *instruction)
15093	14401	{
	14402	ZigType *ptr_to_stack_trace_type = get_pointer_to_type(ira->codegen, get_stack_trace_type(ira->codegen), false);
15094	14403	if (instruction->optional == IrInstructionErrorReturnTrace::Null) {
15095		ZigType *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(ira->codegen);
15096	14404	ZigType *optional_type = get_optional_type(ira->codegen, ptr_to_stack_trace_type);
15097	14405	if (!exec_has_err_ret_trace(ira->codegen, ira->new_irb.exec)) {
15098	14406	IrInstruction *result = ir_const(ira, &instruction->base, optional_type);
...	...	@@ -15110,7 +14418,7 @@ static IrInstruction ir_analyze_instruction_error_return_trace(IrAnalyze ira,
15110	14418	assert(ira->codegen->have_err_ret_tracing);
15111	14419	IrInstruction *new_instruction = ir_build_error_return_trace(&ira->new_irb, instruction->base.scope,
15112	14420	instruction->base.source_node, instruction->optional);
15113		new_instruction->value.type = get_ptr_to_stack_trace_type(ira->codegen);
	14421	new_instruction->value.type = ptr_to_stack_trace_type;
15114	14422	return new_instruction;
15115	14423	}
15116	14424	}
...	...	@@ -15142,42 +14450,6 @@ static IrInstruction ir_analyze_instruction_error_union(IrAnalyze ira,
15142	14450	return ir_const_type(ira, &instruction->base, result_type);
15143	14451	}
15144	14452
15145		IrInstruction ir_get_implicit_allocator(IrAnalyze ira, IrInstruction *source_instr, ImplicitAllocatorId id) {
15146		ZigFn *parent_fn_entry = exec_fn_entry(ira->new_irb.exec);
15147		if (parent_fn_entry == nullptr) {
15148		ir_add_error(ira, source_instr, buf_sprintf("no implicit allocator available"));
15149		return ira->codegen->invalid_instruction;
15150		}
15151
15152		FnTypeId *parent_fn_type = &parent_fn_entry->type_entry->data.fn.fn_type_id;
15153		if (parent_fn_type->cc != CallingConventionAsync) {
15154		ir_add_error(ira, source_instr, buf_sprintf("async function call from non-async caller requires allocator parameter"));
15155		return ira->codegen->invalid_instruction;
15156		}
15157
15158		assert(parent_fn_type->async_allocator_type != nullptr);
15159
15160		switch (id) {
15161		case ImplicitAllocatorIdArg:
15162		{
15163		IrInstruction *result = ir_build_get_implicit_allocator(&ira->new_irb, source_instr->scope,
15164		source_instr->source_node, ImplicitAllocatorIdArg);
15165		result->value.type = parent_fn_type->async_allocator_type;
15166		return result;
15167		}
15168		case ImplicitAllocatorIdLocalVar:
15169		{
15170		ZigVar *coro_allocator_var = ira->old_irb.exec->coro_allocator_var;
15171		assert(coro_allocator_var != nullptr);
15172		IrInstruction *var_ptr_inst = ir_get_var_ptr(ira, source_instr, coro_allocator_var);
15173		IrInstruction *result = ir_get_deref(ira, source_instr, var_ptr_inst, nullptr);
15174		assert(result->value.type != nullptr);
15175		return result;
15176		}
15177		}
15178		zig_unreachable();
15179		}
15180
15181	14453	static IrInstruction ir_analyze_alloca(IrAnalyze ira, IrInstruction source_inst, ZigType var_type,
15182	14454	uint32_t align, const char *name_hint, bool force_comptime)
15183	14455	{
...	...	@@ -15186,7 +14458,7 @@ static IrInstruction ir_analyze_alloca(IrAnalyze ira, IrInstruction *source_in
15186	14458	ConstExprValue *pointee = create_const_vals(1);
15187	14459	pointee->special = ConstValSpecialUndef;
15188	14460
15189		IrInstructionAllocaGen *result = ir_create_alloca_gen(ira, source_inst, align, name_hint);
	14461	IrInstructionAllocaGen *result = ir_build_alloca_gen(ira, source_inst, align, name_hint);
15190	14462	result->base.value.special = ConstValSpecialStatic;
15191	14463	result->base.value.data.x_ptr.special = ConstPtrSpecialRef;
15192	14464	result->base.value.data.x_ptr.mut = force_comptime ? ConstPtrMutComptimeVar : ConstPtrMutInfer;
...	...	@@ -15283,7 +14555,7 @@ static IrInstruction ir_resolve_result_raw(IrAnalyze ira, IrInstruction *suspe
15283	14555	return nullptr;
15284	14556	}
15285	14557	// need to return a result location and don't have one. use a stack allocation
15286		IrInstructionAllocaGen *alloca_gen = ir_create_alloca_gen(ira, suspend_source_instr, 0, "");
	14558	IrInstructionAllocaGen *alloca_gen = ir_build_alloca_gen(ira, suspend_source_instr, 0, "");
15287	14559	if ((err = type_resolve(ira->codegen, value_type, ResolveStatusZeroBitsKnown)))
15288	14560	return ira->codegen->invalid_instruction;
15289	14561	alloca_gen->base.value.type = get_pointer_to_type_extra(ira->codegen, value_type, false, false,
...	...	@@ -15353,8 +14625,12 @@ static IrInstruction ir_resolve_result_raw(IrAnalyze ira, IrInstruction *suspe
15353	14625	if ((err = type_resolve(ira->codegen, ira->explicit_return_type, ResolveStatusZeroBitsKnown))) {
15354	14626	return ira->codegen->invalid_instruction;
15355	14627	}
15356		if (!type_has_bits(ira->explicit_return_type) \|\| !handle_is_ptr(ira->explicit_return_type))
15357		return nullptr;
	14628	if (!type_has_bits(ira->explicit_return_type) \|\| !handle_is_ptr(ira->explicit_return_type)) {
	14629	ZigFn *fn_entry = exec_fn_entry(ira->new_irb.exec);
	14630	if (fn_entry == nullptr \|\| fn_entry->inferred_async_node == nullptr) {
	14631	return nullptr;
	14632	}
	14633	}
15358	14634
15359	14635	ZigType *ptr_return_type = get_pointer_to_type(ira->codegen, ira->explicit_return_type, false);
15360	14636	result_loc->written = true;
...	...	@@ -15616,48 +14892,43 @@ static IrInstruction ir_analyze_instruction_reset_result(IrAnalyze ira, IrInst
15616	14892
15617	14893	static IrInstruction ir_analyze_async_call(IrAnalyze ira, IrInstructionCallSrc call_instruction, ZigFn fn_entry,
15618	14894	ZigType fn_type, IrInstruction fn_ref, IrInstruction **casted_args, size_t arg_count,
15619		IrInstruction *async_allocator_inst)
	14895	IrInstruction *casted_new_stack)
15620	14896	{
15621		Buf *realloc_field_name = buf_create_from_str(ASYNC_REALLOC_FIELD_NAME);
15622		ir_assert(async_allocator_inst->value.type->id == ZigTypeIdPointer, &call_instruction->base);
15623		ZigType *container_type = async_allocator_inst->value.type->data.pointer.child_type;
15624		IrInstruction *field_ptr_inst = ir_analyze_container_field_ptr(ira, realloc_field_name, &call_instruction->base,
15625		async_allocator_inst, container_type, false);
15626		if (type_is_invalid(field_ptr_inst->value.type)) {
15627		return ira->codegen->invalid_instruction;
15628		}
15629		ZigType *ptr_to_realloc_fn_type = field_ptr_inst->value.type;
15630		ir_assert(ptr_to_realloc_fn_type->id == ZigTypeIdPointer, &call_instruction->base);
	14897	if (casted_new_stack != nullptr) {
	14898	// this is an @asyncCall
15631	14899
15632		ZigType *realloc_fn_type = ptr_to_realloc_fn_type->data.pointer.child_type;
15633		if (realloc_fn_type->id != ZigTypeIdFn) {
15634		ir_add_error(ira, &call_instruction->base,
15635		buf_sprintf("expected reallocation function, found '%s'", buf_ptr(&realloc_fn_type->name)));
15636		return ira->codegen->invalid_instruction;
15637		}
	14900	if (fn_type->data.fn.fn_type_id.cc != CallingConventionAsync) {
	14901	ir_add_error(ira, fn_ref,
	14902	buf_sprintf("expected async function, found '%s'", buf_ptr(&fn_type->name)));
	14903	return ira->codegen->invalid_instruction;
	14904	}
	14905
	14906	IrInstruction *ret_ptr = call_instruction->args[call_instruction->arg_count]->child;
	14907	if (type_is_invalid(ret_ptr->value.type))
	14908	return ira->codegen->invalid_instruction;
	14909
	14910	ZigType *anyframe_type = get_any_frame_type(ira->codegen, fn_type->data.fn.fn_type_id.return_type);
15638	14911
15639		ZigType *realloc_fn_return_type = realloc_fn_type->data.fn.fn_type_id.return_type;
15640		if (realloc_fn_return_type->id != ZigTypeIdErrorUnion) {
15641		ir_add_error(ira, fn_ref,
15642		buf_sprintf("expected allocation function to return error union, but it returns '%s'", buf_ptr(&realloc_fn_return_type->name)));
	14912	IrInstructionCallGen *call_gen = ir_build_call_gen(ira, &call_instruction->base, nullptr, fn_ref,
	14913	arg_count, casted_args, FnInlineAuto, true, casted_new_stack, ret_ptr, anyframe_type);
	14914	return &call_gen->base;
	14915	} else if (fn_entry == nullptr) {
	14916	ir_add_error(ira, fn_ref, buf_sprintf("function is not comptime-known; @asyncCall required"));
15643	14917	return ira->codegen->invalid_instruction;
15644	14918	}
15645		ZigType *alloc_fn_error_set_type = realloc_fn_return_type->data.error_union.err_set_type;
15646		ZigType *return_type = fn_type->data.fn.fn_type_id.return_type;
15647		ZigType *promise_type = get_promise_type(ira->codegen, return_type);
15648		ZigType *async_return_type = get_error_union_type(ira->codegen, alloc_fn_error_set_type, promise_type);
15649	14919
15650		IrInstruction *result_loc = ir_resolve_result(ira, &call_instruction->base, no_result_loc(),
15651		async_return_type, nullptr, true, true, false);
15652		if (type_is_invalid(result_loc->value.type) \|\| instr_is_unreachable(result_loc)) {
	14920	ZigType *frame_type = get_fn_frame_type(ira->codegen, fn_entry);
	14921	IrInstruction *result_loc = ir_resolve_result(ira, &call_instruction->base, call_instruction->result_loc,
	14922	frame_type, nullptr, true, true, false);
	14923	if (result_loc != nullptr && (type_is_invalid(result_loc->value.type) \|\| instr_is_unreachable(result_loc))) {
15653	14924	return result_loc;
15654	14925	}
15655
15656		return ir_build_call_gen(ira, &call_instruction->base, fn_entry, fn_ref, arg_count,
15657		casted_args, FnInlineAuto, true, async_allocator_inst, nullptr, result_loc,
15658		async_return_type);
	14926	result_loc = ir_implicit_cast(ira, result_loc, get_pointer_to_type(ira->codegen, frame_type, false));
	14927	if (type_is_invalid(result_loc->value.type))
	14928	return ira->codegen->invalid_instruction;
	14929	return &ir_build_call_gen(ira, &call_instruction->base, fn_entry, fn_ref, arg_count,
	14930	casted_args, FnInlineAuto, true, nullptr, result_loc, frame_type)->base;
15659	14931	}
15660
15661	14932	static bool ir_analyze_fn_call_inline_arg(IrAnalyze ira, AstNode fn_proto_node,
15662	14933	IrInstruction arg, Scope exec_scope, size_t next_proto_i)
15663	14934	{
...	...	@@ -16004,20 +15275,6 @@ static IrInstruction ir_analyze_fn_call(IrAnalyze ira, IrInstructionCallSrc *c
16004	15275	}
16005	15276	return ira->codegen->invalid_instruction;
16006	15277	}
16007		if (fn_type_id->cc == CallingConventionAsync && !call_instruction->is_async) {
16008		ErrorMsg *msg = ir_add_error(ira, fn_ref, buf_sprintf("must use async keyword to call async function"));
16009		if (fn_proto_node) {
16010		add_error_note(ira->codegen, msg, fn_proto_node, buf_sprintf("declared here"));
16011		}
16012		return ira->codegen->invalid_instruction;
16013		}
16014		if (fn_type_id->cc != CallingConventionAsync && call_instruction->is_async) {
16015		ErrorMsg *msg = ir_add_error(ira, fn_ref, buf_sprintf("cannot use async keyword to call non-async function"));
16016		if (fn_proto_node) {
16017		add_error_note(ira->codegen, msg, fn_proto_node, buf_sprintf("declared here"));
16018		}
16019		return ira->codegen->invalid_instruction;
16020		}
16021	15278
16022	15279
16023	15280	if (fn_type_id->is_var_args) {
...	...	@@ -16354,33 +15611,6 @@ static IrInstruction ir_analyze_fn_call(IrAnalyze ira, IrInstructionCallSrc *c
16354	15611	break;
16355	15612	}
16356	15613	}
16357		IrInstruction *async_allocator_inst = nullptr;
16358		if (call_instruction->is_async) {
16359		AstNode *async_allocator_type_node = fn_proto_node->data.fn_proto.async_allocator_type;
16360		if (async_allocator_type_node != nullptr) {
16361		ZigType *async_allocator_type = ir_analyze_type_expr(ira, impl_fn->child_scope, async_allocator_type_node);
16362		if (type_is_invalid(async_allocator_type))
16363		return ira->codegen->invalid_instruction;
16364		inst_fn_type_id.async_allocator_type = async_allocator_type;
16365		}
16366		IrInstruction *uncasted_async_allocator_inst;
16367		if (call_instruction->async_allocator == nullptr) {
16368		uncasted_async_allocator_inst = ir_get_implicit_allocator(ira, &call_instruction->base,
16369		ImplicitAllocatorIdLocalVar);
16370		if (type_is_invalid(uncasted_async_allocator_inst->value.type))
16371		return ira->codegen->invalid_instruction;
16372		} else {
16373		uncasted_async_allocator_inst = call_instruction->async_allocator->child;
16374		if (type_is_invalid(uncasted_async_allocator_inst->value.type))
16375		return ira->codegen->invalid_instruction;
16376		}
16377		if (inst_fn_type_id.async_allocator_type == nullptr) {
16378		inst_fn_type_id.async_allocator_type = uncasted_async_allocator_inst->value.type;
16379		}
16380		async_allocator_inst = ir_implicit_cast(ira, uncasted_async_allocator_inst, inst_fn_type_id.async_allocator_type);
16381		if (type_is_invalid(async_allocator_inst->value.type))
16382		return ira->codegen->invalid_instruction;
16383		}
16384	15614
16385	15615	auto existing_entry = ira->codegen->generic_table.put_unique(generic_id, impl_fn);
16386	15616	if (existing_entry) {
...	...	@@ -16423,17 +15653,23 @@ static IrInstruction ir_analyze_fn_call(IrAnalyze ira, IrInstructionCallSrc *c
16423	15653	size_t impl_param_count = impl_fn_type_id->param_count;
16424	15654	if (call_instruction->is_async) {
16425	15655	IrInstruction *result = ir_analyze_async_call(ira, call_instruction, impl_fn, impl_fn->type_entry,
16426		fn_ref, casted_args, impl_param_count, async_allocator_inst);
	15656	nullptr, casted_args, impl_param_count, casted_new_stack);
16427	15657	return ir_finish_anal(ira, result);
16428	15658	}
16429	15659
16430		assert(async_allocator_inst == nullptr);
16431		IrInstruction *new_call_instruction = ir_build_call_gen(ira, &call_instruction->base,
	15660	if (impl_fn_type_id->cc == CallingConventionAsync && parent_fn_entry->inferred_async_node == nullptr) {
	15661	parent_fn_entry->inferred_async_node = fn_ref->source_node;
	15662	parent_fn_entry->inferred_async_fn = impl_fn;
	15663	}
	15664
	15665	IrInstructionCallGen *new_call_instruction = ir_build_call_gen(ira, &call_instruction->base,
16432	15666	impl_fn, nullptr, impl_param_count, casted_args, fn_inline,
16433		call_instruction->is_async, nullptr, casted_new_stack, result_loc,
	15667	false, casted_new_stack, result_loc,
16434	15668	impl_fn_type_id->return_type);
16435	15669
16436		return ir_finish_anal(ira, new_call_instruction);
	15670	parent_fn_entry->call_list.append(new_call_instruction);
	15671
	15672	return ir_finish_anal(ira, &new_call_instruction->base);
16437	15673	}
16438	15674
16439	15675	ZigFn *parent_fn_entry = exec_fn_entry(ira->new_irb.exec);
...	...	@@ -16475,20 +15711,56 @@ static IrInstruction ir_analyze_fn_call(IrAnalyze ira, IrInstructionCallSrc *c
16475	15711	IrInstruction *old_arg = call_instruction->args[call_i]->child;
16476	15712	if (type_is_invalid(old_arg->value.type))
16477	15713	return ira->codegen->invalid_instruction;
16478		IrInstruction *casted_arg;
16479		if (next_arg_index < src_param_count) {
16480		ZigType *param_type = fn_type_id->param_info[next_arg_index].type;
16481		if (type_is_invalid(param_type))
16482		return ira->codegen->invalid_instruction;
16483		casted_arg = ir_implicit_cast(ira, old_arg, param_type);
16484		if (type_is_invalid(casted_arg->value.type))
16485		return ira->codegen->invalid_instruction;
	15714
	15715	if (old_arg->value.type->id == ZigTypeIdArgTuple) {
	15716	for (size_t arg_tuple_i = old_arg->value.data.x_arg_tuple.start_index;
	15717	arg_tuple_i < old_arg->value.data.x_arg_tuple.end_index; arg_tuple_i += 1)
	15718	{
	15719	ZigVar *arg_var = get_fn_var_by_index(parent_fn_entry, arg_tuple_i);
	15720	if (arg_var == nullptr) {
	15721	ir_add_error(ira, old_arg,
	15722	buf_sprintf("compiler bug: var args can't handle void. https://github.com/ziglang/zig/issues/557"));
	15723	return ira->codegen->invalid_instruction;
	15724	}
	15725	IrInstruction *arg_var_ptr_inst = ir_get_var_ptr(ira, old_arg, arg_var);
	15726	if (type_is_invalid(arg_var_ptr_inst->value.type))
	15727	return ira->codegen->invalid_instruction;
	15728
	15729	IrInstruction *arg_tuple_arg = ir_get_deref(ira, old_arg, arg_var_ptr_inst, nullptr);
	15730	if (type_is_invalid(arg_tuple_arg->value.type))
	15731	return ira->codegen->invalid_instruction;
	15732
	15733	IrInstruction *casted_arg;
	15734	if (next_arg_index < src_param_count) {
	15735	ZigType *param_type = fn_type_id->param_info[next_arg_index].type;
	15736	if (type_is_invalid(param_type))
	15737	return ira->codegen->invalid_instruction;
	15738	casted_arg = ir_implicit_cast(ira, arg_tuple_arg, param_type);
	15739	if (type_is_invalid(casted_arg->value.type))
	15740	return ira->codegen->invalid_instruction;
	15741	} else {
	15742	casted_arg = arg_tuple_arg;
	15743	}
	15744
	15745	casted_args[next_arg_index] = casted_arg;
	15746	next_arg_index += 1;
	15747	}
16486	15748	} else {
16487		casted_arg = old_arg;
16488		}
	15749	IrInstruction *casted_arg;
	15750	if (next_arg_index < src_param_count) {
	15751	ZigType *param_type = fn_type_id->param_info[next_arg_index].type;
	15752	if (type_is_invalid(param_type))
	15753	return ira->codegen->invalid_instruction;
	15754	casted_arg = ir_implicit_cast(ira, old_arg, param_type);
	15755	if (type_is_invalid(casted_arg->value.type))
	15756	return ira->codegen->invalid_instruction;
	15757	} else {
	15758	casted_arg = old_arg;
	15759	}
16489	15760
16490		casted_args[next_arg_index] = casted_arg;
16491		next_arg_index += 1;
	15761	casted_args[next_arg_index] = casted_arg;
	15762	next_arg_index += 1;
	15763	}
16492	15764	}
16493	15765
16494	15766	assert(next_arg_index == call_param_count);
...	...	@@ -16497,32 +15769,21 @@ static IrInstruction ir_analyze_fn_call(IrAnalyze ira, IrInstructionCallSrc *c
16497	15769	if (type_is_invalid(return_type))
16498	15770	return ira->codegen->invalid_instruction;
16499	15771
16500		if (call_instruction->is_async) {
16501		IrInstruction *uncasted_async_allocator_inst;
16502		if (call_instruction->async_allocator == nullptr) {
16503		uncasted_async_allocator_inst = ir_get_implicit_allocator(ira, &call_instruction->base,
16504		ImplicitAllocatorIdLocalVar);
16505		if (type_is_invalid(uncasted_async_allocator_inst->value.type))
16506		return ira->codegen->invalid_instruction;
16507		} else {
16508		uncasted_async_allocator_inst = call_instruction->async_allocator->child;
16509		if (type_is_invalid(uncasted_async_allocator_inst->value.type))
16510		return ira->codegen->invalid_instruction;
16511
16512		}
16513		IrInstruction *async_allocator_inst = ir_implicit_cast(ira, uncasted_async_allocator_inst, fn_type_id->async_allocator_type);
16514		if (type_is_invalid(async_allocator_inst->value.type))
16515		return ira->codegen->invalid_instruction;
	15772	if (fn_entry != nullptr && fn_entry->fn_inline == FnInlineAlways && fn_inline == FnInlineNever) {
	15773	ir_add_error(ira, &call_instruction->base,
	15774	buf_sprintf("no-inline call of inline function"));
	15775	return ira->codegen->invalid_instruction;
	15776	}
16516	15777
	15778	if (call_instruction->is_async) {
16517	15779	IrInstruction *result = ir_analyze_async_call(ira, call_instruction, fn_entry, fn_type, fn_ref,
16518		casted_args, call_param_count, async_allocator_inst);
	15780	casted_args, call_param_count, casted_new_stack);
16519	15781	return ir_finish_anal(ira, result);
16520	15782	}
16521	15783
16522		if (fn_entry != nullptr && fn_entry->fn_inline == FnInlineAlways && fn_inline == FnInlineNever) {
16523		ir_add_error(ira, &call_instruction->base,
16524		buf_sprintf("no-inline call of inline function"));
16525		return ira->codegen->invalid_instruction;
	15784	if (fn_type_id->cc == CallingConventionAsync && parent_fn_entry->inferred_async_node == nullptr) {
	15785	parent_fn_entry->inferred_async_node = fn_ref->source_node;
	15786	parent_fn_entry->inferred_async_fn = fn_entry;
16526	15787	}
16527	15788
16528	15789	IrInstruction *result_loc;
...	...	@@ -16536,10 +15797,11 @@ static IrInstruction ir_analyze_fn_call(IrAnalyze ira, IrInstructionCallSrc *c
16536	15797	result_loc = nullptr;
16537	15798	}
16538	15799
16539		IrInstruction *new_call_instruction = ir_build_call_gen(ira, &call_instruction->base, fn_entry, fn_ref,
16540		call_param_count, casted_args, fn_inline, false, nullptr, casted_new_stack,
	15800	IrInstructionCallGen *new_call_instruction = ir_build_call_gen(ira, &call_instruction->base, fn_entry, fn_ref,
	15801	call_param_count, casted_args, fn_inline, false, casted_new_stack,
16541	15802	result_loc, return_type);
16542		return ir_finish_anal(ira, new_call_instruction);
	15803	parent_fn_entry->call_list.append(new_call_instruction);
	15804	return ir_finish_anal(ira, &new_call_instruction->base);
16543	15805	}
16544	15806
16545	15807	static IrInstruction ir_analyze_instruction_call(IrAnalyze ira, IrInstructionCallSrc *call_instruction) {
...	...	@@ -16684,7 +15946,7 @@ static Error ir_read_const_ptr(IrAnalyze ira, CodeGen codegen, AstNode *source
16684	15946	zig_unreachable();
16685	15947	}
16686	15948
16687		static IrInstruction ir_analyze_maybe(IrAnalyze ira, IrInstructionUnOp *un_op_instruction) {
	15949	static IrInstruction ir_analyze_optional_type(IrAnalyze ira, IrInstructionUnOp *un_op_instruction) {
16688	15950	Error err;
16689	15951	IrInstruction *value = un_op_instruction->value->child;
16690	15952	ZigType *type_entry = ir_resolve_type(ira, value);
...	...	@@ -16718,8 +15980,10 @@ static IrInstruction ir_analyze_maybe(IrAnalyze ira, IrInstructionUnOp *un_op_
16718	15980	case ZigTypeIdFn:
16719	15981	case ZigTypeIdBoundFn:
16720	15982	case ZigTypeIdArgTuple:
16721		case ZigTypeIdPromise:
	15983	case ZigTypeIdFnFrame:
	15984	case ZigTypeIdAnyFrame:
16722	15985	return ir_const_type(ira, &un_op_instruction->base, get_optional_type(ira->codegen, type_entry));
	15986
16723	15987	case ZigTypeIdUnreachable:
16724	15988	case ZigTypeIdOpaque:
16725	15989	ir_add_error_node(ira, un_op_instruction->base.source_node,
...	...	@@ -16883,7 +16147,7 @@ static IrInstruction ir_analyze_instruction_un_op(IrAnalyze ira, IrInstruction
16883	16147	return result;
16884	16148	}
16885	16149	case IrUnOpOptional:
16886		return ir_analyze_maybe(ira, instruction);
	16150	return ir_analyze_optional_type(ira, instruction);
16887	16151	}
16888	16152	zig_unreachable();
16889	16153	}
...	...	@@ -18443,6 +17707,20 @@ static IrInstruction ir_analyze_instruction_set_float_mode(IrAnalyze ira,
18443	17707	return ir_const_void(ira, &instruction->base);
18444	17708	}
18445	17709
	17710	static IrInstruction ir_analyze_instruction_any_frame_type(IrAnalyze ira,
	17711	IrInstructionAnyFrameType *instruction)
	17712	{
	17713	ZigType *payload_type = nullptr;
	17714	if (instruction->payload_type != nullptr) {
	17715	payload_type = ir_resolve_type(ira, instruction->payload_type->child);
	17716	if (type_is_invalid(payload_type))
	17717	return ira->codegen->invalid_instruction;
	17718	}
	17719
	17720	ZigType *any_frame_type = get_any_frame_type(ira->codegen, payload_type);
	17721	return ir_const_type(ira, &instruction->base, any_frame_type);
	17722	}
	17723
18446	17724	static IrInstruction ir_analyze_instruction_slice_type(IrAnalyze ira,
18447	17725	IrInstructionSliceType *slice_type_instruction)
18448	17726	{
...	...	@@ -18490,8 +17768,9 @@ static IrInstruction ir_analyze_instruction_slice_type(IrAnalyze ira,
18490	17768	case ZigTypeIdUnion:
18491	17769	case ZigTypeIdFn:
18492	17770	case ZigTypeIdBoundFn:
18493		case ZigTypeIdPromise:
18494	17771	case ZigTypeIdVector:
	17772	case ZigTypeIdFnFrame:
	17773	case ZigTypeIdAnyFrame:
18495	17774	{
18496	17775	ResolveStatus needed_status = (align_bytes == 0) ?
18497	17776	ResolveStatusZeroBitsKnown : ResolveStatusAlignmentKnown;
...	...	@@ -18605,8 +17884,9 @@ static IrInstruction ir_analyze_instruction_array_type(IrAnalyze ira,
18605	17884	case ZigTypeIdUnion:
18606	17885	case ZigTypeIdFn:
18607	17886	case ZigTypeIdBoundFn:
18608		case ZigTypeIdPromise:
18609	17887	case ZigTypeIdVector:
	17888	case ZigTypeIdFnFrame:
	17889	case ZigTypeIdAnyFrame:
18610	17890	{
18611	17891	if ((err = ensure_complete_type(ira->codegen, child_type)))
18612	17892	return ira->codegen->invalid_instruction;
...	...	@@ -18617,22 +17897,6 @@ static IrInstruction ir_analyze_instruction_array_type(IrAnalyze ira,
18617	17897	zig_unreachable();
18618	17898	}
18619	17899
18620		static IrInstruction ir_analyze_instruction_promise_type(IrAnalyze ira, IrInstructionPromiseType *instruction) {
18621		ZigType *promise_type;
18622
18623		if (instruction->payload_type == nullptr) {
18624		promise_type = ira->codegen->builtin_types.entry_promise;
18625		} else {
18626		ZigType *payload_type = ir_resolve_type(ira, instruction->payload_type->child);
18627		if (type_is_invalid(payload_type))
18628		return ira->codegen->invalid_instruction;
18629
18630		promise_type = get_promise_type(ira->codegen, payload_type);
18631		}
18632
18633		return ir_const_type(ira, &instruction->base, promise_type);
18634		}
18635
18636	17900	static IrInstruction ir_analyze_instruction_size_of(IrAnalyze ira,
18637	17901	IrInstructionSizeOf *size_of_instruction)
18638	17902	{
...	...	@@ -18672,8 +17936,9 @@ static IrInstruction ir_analyze_instruction_size_of(IrAnalyze ira,
18672	17936	case ZigTypeIdEnum:
18673	17937	case ZigTypeIdUnion:
18674	17938	case ZigTypeIdFn:
18675		case ZigTypeIdPromise:
18676	17939	case ZigTypeIdVector:
	17940	case ZigTypeIdFnFrame:
	17941	case ZigTypeIdAnyFrame:
18677	17942	{
18678	17943	uint64_t size_in_bytes = type_size(ira->codegen, type_entry);
18679	17944	return ir_const_unsigned(ira, &size_of_instruction->base, size_in_bytes);
...	...	@@ -19159,7 +18424,6 @@ static IrInstruction ir_analyze_instruction_switch_target(IrAnalyze ira,
19159	18424	case ZigTypeIdComptimeInt:
19160	18425	case ZigTypeIdEnumLiteral:
19161	18426	case ZigTypeIdPointer:
19162		case ZigTypeIdPromise:
19163	18427	case ZigTypeIdFn:
19164	18428	case ZigTypeIdErrorSet: {
19165	18429	if (pointee_val) {
...	...	@@ -19238,6 +18502,8 @@ static IrInstruction ir_analyze_instruction_switch_target(IrAnalyze ira,
19238	18502	case ZigTypeIdArgTuple:
19239	18503	case ZigTypeIdOpaque:
19240	18504	case ZigTypeIdVector:
	18505	case ZigTypeIdFnFrame:
	18506	case ZigTypeIdAnyFrame:
19241	18507	ir_add_error(ira, &switch_target_instruction->base,
19242	18508	buf_sprintf("invalid switch target type '%s'", buf_ptr(&target_type->name)));
19243	18509	return ira->codegen->invalid_instruction;
...	...	@@ -20672,32 +19938,22 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInstruction source_instr
20672	19938
20673	19939	break;
20674	19940	}
20675		case ZigTypeIdPromise:
20676		{
20677		result = create_const_vals(1);
20678		result->special = ConstValSpecialStatic;
20679		result->type = ir_type_info_get_type(ira, "Promise", nullptr);
20680
20681		ConstExprValue *fields = create_const_vals(1);
20682		result->data.x_struct.fields = fields;
20683
20684		// child: ?type
20685		ensure_field_index(result->type, "child", 0);
20686		fields[0].special = ConstValSpecialStatic;
20687		fields[0].type = get_optional_type(ira->codegen, ira->codegen->builtin_types.entry_type);
	19941	case ZigTypeIdAnyFrame: {
	19942	result = create_const_vals(1);
	19943	result->special = ConstValSpecialStatic;
	19944	result->type = ir_type_info_get_type(ira, "AnyFrame", nullptr);
20688	19945
20689		if (type_entry->data.promise.result_type == nullptr)
20690		fields[0].data.x_optional = nullptr;
20691		else {
20692		ConstExprValue *child_type = create_const_vals(1);
20693		child_type->special = ConstValSpecialStatic;
20694		child_type->type = ira->codegen->builtin_types.entry_type;
20695		child_type->data.x_type = type_entry->data.promise.result_type;
20696		fields[0].data.x_optional = child_type;
20697		}
	19946	ConstExprValue *fields = create_const_vals(1);
	19947	result->data.x_struct.fields = fields;
20698	19948
20699		break;
20700		}
	19949	// child: ?type
	19950	ensure_field_index(result->type, "child", 0);
	19951	fields[0].special = ConstValSpecialStatic;
	19952	fields[0].type = get_optional_type(ira->codegen, ira->codegen->builtin_types.entry_type);
	19953	fields[0].data.x_optional = (type_entry->data.any_frame.result_type == nullptr) ? nullptr :
	19954	create_const_type(ira->codegen, type_entry->data.any_frame.result_type);
	19955	break;
	19956	}
20701	19957	case ZigTypeIdEnum:
20702	19958	{
20703	19959	result = create_const_vals(1);
...	...	@@ -21007,7 +20263,7 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInstruction source_instr
21007	20263	result->special = ConstValSpecialStatic;
21008	20264	result->type = ir_type_info_get_type(ira, "Fn", nullptr);
21009	20265
21010		ConstExprValue *fields = create_const_vals(6);
	20266	ConstExprValue *fields = create_const_vals(5);
21011	20267	result->data.x_struct.fields = fields;
21012	20268
21013	20269	// calling_convention: TypeInfo.CallingConvention
...	...	@@ -21038,20 +20294,7 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInstruction source_instr
21038	20294	return_type->special = ConstValSpecialStatic;
21039	20295	return_type->type = ira->codegen->builtin_types.entry_type;
21040	20296	return_type->data.x_type = type_entry->data.fn.fn_type_id.return_type;
21041		fields[3].data.x_optional = return_type;
21042		}
21043		// async_allocator_type: type
21044		ensure_field_index(result->type, "async_allocator_type", 4);
21045		fields[4].special = ConstValSpecialStatic;
21046		fields[4].type = get_optional_type(ira->codegen, ira->codegen->builtin_types.entry_type);
21047		if (type_entry->data.fn.fn_type_id.async_allocator_type == nullptr)
21048		fields[4].data.x_optional = nullptr;
21049		else {
21050		ConstExprValue *async_alloc_type = create_const_vals(1);
21051		async_alloc_type->special = ConstValSpecialStatic;
21052		async_alloc_type->type = ira->codegen->builtin_types.entry_type;
21053		async_alloc_type->data.x_type = type_entry->data.fn.fn_type_id.async_allocator_type;
21054		fields[4].data.x_optional = async_alloc_type;
	20297	fields[3].data.x_optional = return_type;
21055	20298	}
21056	20299	// args: []TypeInfo.FnArg
21057	20300	ZigType *type_info_fn_arg_type = ir_type_info_get_type(ira, "FnArg", nullptr);
...	...	@@ -21067,10 +20310,9 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInstruction source_instr
21067	20310	fn_arg_array->data.x_array.special = ConstArraySpecialNone;
21068	20311	fn_arg_array->data.x_array.data.s_none.elements = create_const_vals(fn_arg_count);
21069	20312
21070		init_const_slice(ira->codegen, &fields[5], fn_arg_array, 0, fn_arg_count, false);
	20313	init_const_slice(ira->codegen, &fields[4], fn_arg_array, 0, fn_arg_count, false);
21071	20314
21072		for (size_t fn_arg_index = 0; fn_arg_index < fn_arg_count; fn_arg_index++)
21073		{
	20315	for (size_t fn_arg_index = 0; fn_arg_index < fn_arg_count; fn_arg_index++) {
21074	20316	FnTypeParamInfo *fn_param_info = &type_entry->data.fn.fn_type_id.param_info[fn_arg_index];
21075	20317	ConstExprValue *fn_arg_val = &fn_arg_array->data.x_array.data.s_none.elements[fn_arg_index];
21076	20318
...	...	@@ -21117,6 +20359,8 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInstruction source_instr
21117	20359
21118	20360	break;
21119	20361	}
	20362	case ZigTypeIdFnFrame:
	20363	zig_panic("TODO @typeInfo for async function frames");
21120	20364	}
21121	20365
21122	20366	assert(result != nullptr);
...	...	@@ -22830,11 +22074,45 @@ static IrInstruction ir_analyze_instruction_frame_address(IrAnalyze ira, IrIns
22830	22074	return result;
22831	22075	}
22832	22076
22833		static IrInstruction ir_analyze_instruction_handle(IrAnalyze ira, IrInstructionHandle *instruction) {
	22077	static IrInstruction ir_analyze_instruction_frame_handle(IrAnalyze ira, IrInstructionFrameHandle *instruction) {
	22078	ZigFn *fn = exec_fn_entry(ira->new_irb.exec);
	22079	ir_assert(fn != nullptr, &instruction->base);
	22080
	22081	if (fn->inferred_async_node == nullptr) {
	22082	fn->inferred_async_node = instruction->base.source_node;
	22083	}
	22084
	22085	ZigType *frame_type = get_fn_frame_type(ira->codegen, fn);
	22086	ZigType *ptr_frame_type = get_pointer_to_type(ira->codegen, frame_type, false);
	22087
22834	22088	IrInstruction *result = ir_build_handle(&ira->new_irb, instruction->base.scope, instruction->base.source_node);
22835		ZigFn *fn_entry = exec_fn_entry(ira->new_irb.exec);
22836		assert(fn_entry != nullptr);
22837		result->value.type = get_promise_type(ira->codegen, fn_entry->type_entry->data.fn.fn_type_id.return_type);
	22089	result->value.type = ptr_frame_type;
	22090	return result;
	22091	}
	22092
	22093	static IrInstruction ir_analyze_instruction_frame_type(IrAnalyze ira, IrInstructionFrameType *instruction) {
	22094	ZigFn *fn = ir_resolve_fn(ira, instruction->fn->child);
	22095	if (fn == nullptr)
	22096	return ira->codegen->invalid_instruction;
	22097
	22098	ZigType *ty = get_fn_frame_type(ira->codegen, fn);
	22099	return ir_const_type(ira, &instruction->base, ty);
	22100	}
	22101
	22102	static IrInstruction ir_analyze_instruction_frame_size(IrAnalyze ira, IrInstructionFrameSizeSrc *instruction) {
	22103	IrInstruction *fn = instruction->fn->child;
	22104	if (type_is_invalid(fn->value.type))
	22105	return ira->codegen->invalid_instruction;
	22106
	22107	if (fn->value.type->id != ZigTypeIdFn) {
	22108	ir_add_error(ira, fn,
	22109	buf_sprintf("expected function, found '%s'", buf_ptr(&fn->value.type->name)));
	22110	return ira->codegen->invalid_instruction;
	22111	}
	22112
	22113	IrInstruction *result = ir_build_frame_size_gen(&ira->new_irb, instruction->base.scope,
	22114	instruction->base.source_node, fn);
	22115	result->value.type = ira->codegen->builtin_types.entry_usize;
22838	22116	return result;
22839	22117	}
22840	22118
...	...	@@ -22869,7 +22147,6 @@ static IrInstruction ir_analyze_instruction_align_of(IrAnalyze ira, IrInstruct
22869	22147	case ZigTypeIdInt:
22870	22148	case ZigTypeIdFloat:
22871	22149	case ZigTypeIdPointer:
22872		case ZigTypeIdPromise:
22873	22150	case ZigTypeIdArray:
22874	22151	case ZigTypeIdStruct:
22875	22152	case ZigTypeIdOptional:
...	...	@@ -22879,6 +22156,8 @@ static IrInstruction ir_analyze_instruction_align_of(IrAnalyze ira, IrInstruct
22879	22156	case ZigTypeIdUnion:
22880	22157	case ZigTypeIdFn:
22881	22158	case ZigTypeIdVector:
	22159	case ZigTypeIdFnFrame:
	22160	case ZigTypeIdAnyFrame:
22882	22161	{
22883	22162	uint64_t align_in_bytes = get_abi_alignment(ira->codegen, type_entry);
22884	22163	return ir_const_unsigned(ira, &instruction->base, align_in_bytes);
...	...	@@ -22993,19 +22272,6 @@ static IrInstruction ir_analyze_instruction_overflow_op(IrAnalyze ira, IrInstr
22993	22272	return result;
22994	22273	}
22995	22274
22996		static IrInstruction ir_analyze_instruction_result_ptr(IrAnalyze ira, IrInstructionResultPtr *instruction) {
22997		IrInstruction *result = instruction->result->child;
22998		if (type_is_invalid(result->value.type))
22999		return result;
23000
23001		if (instruction->result_loc->written && instruction->result_loc->resolved_loc != nullptr &&
23002		!instr_is_comptime(result))
23003		{
23004		return instruction->result_loc->resolved_loc;
23005		}
23006		return ir_get_ref(ira, &instruction->base, result, true, false);
23007		}
23008
23009	22275	static void ir_eval_mul_add(IrAnalyze ira, IrInstructionMulAdd source_instr, ZigType *float_type,
23010	22276	ConstExprValue op1, ConstExprValue op2, ConstExprValue op3, ConstExprValue out_val) {
23011	22277	if (float_type->id == ZigTypeIdComptimeFloat) {
...	...	@@ -23130,11 +22396,16 @@ static IrInstruction ir_analyze_instruction_test_err(IrAnalyze ira, IrInstruct
23130	22396	if (type_is_invalid(base_ptr->value.type))
23131	22397	return ira->codegen->invalid_instruction;
23132	22398
23133		IrInstruction *value = ir_get_deref(ira, &instruction->base, base_ptr, nullptr);
	22399	IrInstruction *value;
	22400	if (instruction->base_ptr_is_payload) {
	22401	value = base_ptr;
	22402	} else {
	22403	value = ir_get_deref(ira, &instruction->base, base_ptr, nullptr);
	22404	}
	22405
23134	22406	ZigType *type_entry = value->value.type;
23135	22407	if (type_is_invalid(type_entry))
23136	22408	return ira->codegen->invalid_instruction;
23137
23138	22409	if (type_entry->id == ZigTypeIdErrorUnion) {
23139	22410	if (instr_is_comptime(value)) {
23140	22411	ConstExprValue *err_union_val = ir_resolve_const(ira, value, UndefBad);
...	...	@@ -23428,18 +22699,6 @@ static IrInstruction ir_analyze_instruction_fn_proto(IrAnalyze ira, IrInstruct
23428	22699	return ira->codegen->invalid_instruction;
23429	22700	}
23430	22701
23431		if (fn_type_id.cc == CallingConventionAsync) {
23432		if (instruction->async_allocator_type_value == nullptr) {
23433		ir_add_error(ira, &instruction->base,
23434		buf_sprintf("async fn proto missing allocator type"));
23435		return ira->codegen->invalid_instruction;
23436		}
23437		IrInstruction *async_allocator_type_value = instruction->async_allocator_type_value->child;
23438		fn_type_id.async_allocator_type = ir_resolve_type(ira, async_allocator_type_value);
23439		if (type_is_invalid(fn_type_id.async_allocator_type))
23440		return ira->codegen->invalid_instruction;
23441		}
23442
23443	22702	return ir_const_type(ira, &instruction->base, get_fn_type(ira->codegen, &fn_type_id));
23444	22703	}
23445	22704
...	...	@@ -23678,7 +22937,7 @@ static IrInstruction ir_analyze_instruction_check_statement_is_void(IrAnalyze
23678	22937	if (type_is_invalid(statement_type))
23679	22938	return ira->codegen->invalid_instruction;
23680	22939
23681		if (statement_type->id != ZigTypeIdVoid) {
	22940	if (statement_type->id != ZigTypeIdVoid && statement_type->id != ZigTypeIdUnreachable) {
23682	22941	ir_add_error(ira, &instruction->base, buf_sprintf("expression value is ignored"));
23683	22942	}
23684	22943
...	...	@@ -23933,7 +23192,6 @@ static void buf_write_value_bytes(CodeGen codegen, uint8_t buf, ConstExprValue
23933	23192	case ZigTypeIdEnumLiteral:
23934	23193	case ZigTypeIdUndefined:
23935	23194	case ZigTypeIdNull:
23936		case ZigTypeIdPromise:
23937	23195	case ZigTypeIdErrorUnion:
23938	23196	case ZigTypeIdErrorSet:
23939	23197	zig_unreachable();
...	...	@@ -24043,6 +23301,10 @@ static void buf_write_value_bytes(CodeGen codegen, uint8_t buf, ConstExprValue
24043	23301	zig_panic("TODO buf_write_value_bytes fn type");
24044	23302	case ZigTypeIdUnion:
24045	23303	zig_panic("TODO buf_write_value_bytes union type");
	23304	case ZigTypeIdFnFrame:
	23305	zig_panic("TODO buf_write_value_bytes async fn frame type");
	23306	case ZigTypeIdAnyFrame:
	23307	zig_panic("TODO buf_write_value_bytes anyframe type");
24046	23308	}
24047	23309	zig_unreachable();
24048	23310	}
...	...	@@ -24087,7 +23349,6 @@ static Error buf_read_value_bytes(IrAnalyze ira, CodeGen codegen, AstNode *sou
24087	23349	case ZigTypeIdEnumLiteral:
24088	23350	case ZigTypeIdUndefined:
24089	23351	case ZigTypeIdNull:
24090		case ZigTypeIdPromise:
24091	23352	zig_unreachable();
24092	23353	case ZigTypeIdVoid:
24093	23354	return ErrorNone;
...	...	@@ -24223,6 +23484,10 @@ static Error buf_read_value_bytes(IrAnalyze ira, CodeGen codegen, AstNode *sou
24223	23484	zig_panic("TODO buf_read_value_bytes fn type");
24224	23485	case ZigTypeIdUnion:
24225	23486	zig_panic("TODO buf_read_value_bytes union type");
	23487	case ZigTypeIdFnFrame:
	23488	zig_panic("TODO buf_read_value_bytes async fn frame type");
	23489	case ZigTypeIdAnyFrame:
	23490	zig_panic("TODO buf_read_value_bytes anyframe type");
24226	23491	}
24227	23492	zig_unreachable();
24228	23493	}
...	...	@@ -24573,184 +23838,6 @@ static IrInstruction ir_analyze_instruction_tag_type(IrAnalyze ira, IrInstruct
24573	23838	}
24574	23839	}
24575	23840
24576		static IrInstruction ir_analyze_instruction_cancel(IrAnalyze ira, IrInstructionCancel *instruction) {
24577		IrInstruction *target_inst = instruction->target->child;
24578		if (type_is_invalid(target_inst->value.type))
24579		return ira->codegen->invalid_instruction;
24580		IrInstruction *casted_target = ir_implicit_cast(ira, target_inst, ira->codegen->builtin_types.entry_promise);
24581		if (type_is_invalid(casted_target->value.type))
24582		return ira->codegen->invalid_instruction;
24583
24584		IrInstruction *result = ir_build_cancel(&ira->new_irb, instruction->base.scope, instruction->base.source_node, casted_target);
24585		result->value.type = ira->codegen->builtin_types.entry_void;
24586		result->value.special = ConstValSpecialStatic;
24587		return result;
24588		}
24589
24590		static IrInstruction ir_analyze_instruction_coro_id(IrAnalyze ira, IrInstructionCoroId *instruction) {
24591		IrInstruction *promise_ptr = instruction->promise_ptr->child;
24592		if (type_is_invalid(promise_ptr->value.type))
24593		return ira->codegen->invalid_instruction;
24594
24595		IrInstruction *result = ir_build_coro_id(&ira->new_irb, instruction->base.scope, instruction->base.source_node,
24596		promise_ptr);
24597		result->value.type = ira->codegen->builtin_types.entry_usize;
24598		return result;
24599		}
24600
24601		static IrInstruction ir_analyze_instruction_coro_alloc(IrAnalyze ira, IrInstructionCoroAlloc *instruction) {
24602		IrInstruction *coro_id = instruction->coro_id->child;
24603		if (type_is_invalid(coro_id->value.type))
24604		return ira->codegen->invalid_instruction;
24605
24606		IrInstruction *result = ir_build_coro_alloc(&ira->new_irb, instruction->base.scope, instruction->base.source_node,
24607		coro_id);
24608		result->value.type = ira->codegen->builtin_types.entry_bool;
24609		return result;
24610		}
24611
24612		static IrInstruction ir_analyze_instruction_coro_size(IrAnalyze ira, IrInstructionCoroSize *instruction) {
24613		IrInstruction *result = ir_build_coro_size(&ira->new_irb, instruction->base.scope, instruction->base.source_node);
24614		result->value.type = ira->codegen->builtin_types.entry_usize;
24615		return result;
24616		}
24617
24618		static IrInstruction ir_analyze_instruction_coro_begin(IrAnalyze ira, IrInstructionCoroBegin *instruction) {
24619		IrInstruction *coro_id = instruction->coro_id->child;
24620		if (type_is_invalid(coro_id->value.type))
24621		return ira->codegen->invalid_instruction;
24622
24623		IrInstruction *coro_mem_ptr = instruction->coro_mem_ptr->child;
24624		if (type_is_invalid(coro_mem_ptr->value.type))
24625		return ira->codegen->invalid_instruction;
24626
24627		ZigFn *fn_entry = exec_fn_entry(ira->new_irb.exec);
24628		ir_assert(fn_entry != nullptr, &instruction->base);
24629		IrInstruction *result = ir_build_coro_begin(&ira->new_irb, instruction->base.scope, instruction->base.source_node,
24630		coro_id, coro_mem_ptr);
24631		result->value.type = get_promise_type(ira->codegen, fn_entry->type_entry->data.fn.fn_type_id.return_type);
24632		return result;
24633		}
24634
24635		static IrInstruction ir_analyze_instruction_get_implicit_allocator(IrAnalyze ira, IrInstructionGetImplicitAllocator *instruction) {
24636		return ir_get_implicit_allocator(ira, &instruction->base, instruction->id);
24637		}
24638
24639		static IrInstruction ir_analyze_instruction_coro_alloc_fail(IrAnalyze ira, IrInstructionCoroAllocFail *instruction) {
24640		IrInstruction *err_val = instruction->err_val->child;
24641		if (type_is_invalid(err_val->value.type))
24642		return ir_unreach_error(ira);
24643
24644		IrInstruction *result = ir_build_coro_alloc_fail(&ira->new_irb, instruction->base.scope, instruction->base.source_node, err_val);
24645		result->value.type = ira->codegen->builtin_types.entry_unreachable;
24646		return ir_finish_anal(ira, result);
24647		}
24648
24649		static IrInstruction ir_analyze_instruction_coro_suspend(IrAnalyze ira, IrInstructionCoroSuspend *instruction) {
24650		IrInstruction *save_point = nullptr;
24651		if (instruction->save_point != nullptr) {
24652		save_point = instruction->save_point->child;
24653		if (type_is_invalid(save_point->value.type))
24654		return ira->codegen->invalid_instruction;
24655		}
24656
24657		IrInstruction *is_final = instruction->is_final->child;
24658		if (type_is_invalid(is_final->value.type))
24659		return ira->codegen->invalid_instruction;
24660
24661		IrInstruction *result = ir_build_coro_suspend(&ira->new_irb, instruction->base.scope,
24662		instruction->base.source_node, save_point, is_final);
24663		result->value.type = ira->codegen->builtin_types.entry_u8;
24664		return result;
24665		}
24666
24667		static IrInstruction ir_analyze_instruction_coro_end(IrAnalyze ira, IrInstructionCoroEnd *instruction) {
24668		IrInstruction *result = ir_build_coro_end(&ira->new_irb, instruction->base.scope,
24669		instruction->base.source_node);
24670		result->value.type = ira->codegen->builtin_types.entry_void;
24671		return result;
24672		}
24673
24674		static IrInstruction ir_analyze_instruction_coro_free(IrAnalyze ira, IrInstructionCoroFree *instruction) {
24675		IrInstruction *coro_id = instruction->coro_id->child;
24676		if (type_is_invalid(coro_id->value.type))
24677		return ira->codegen->invalid_instruction;
24678
24679		IrInstruction *coro_handle = instruction->coro_handle->child;
24680		if (type_is_invalid(coro_handle->value.type))
24681		return ira->codegen->invalid_instruction;
24682
24683		IrInstruction *result = ir_build_coro_free(&ira->new_irb, instruction->base.scope,
24684		instruction->base.source_node, coro_id, coro_handle);
24685		ZigType *ptr_type = get_pointer_to_type(ira->codegen, ira->codegen->builtin_types.entry_u8, false);
24686		result->value.type = get_optional_type(ira->codegen, ptr_type);
24687		return result;
24688		}
24689
24690		static IrInstruction ir_analyze_instruction_coro_resume(IrAnalyze ira, IrInstructionCoroResume *instruction) {
24691		IrInstruction *awaiter_handle = instruction->awaiter_handle->child;
24692		if (type_is_invalid(awaiter_handle->value.type))
24693		return ira->codegen->invalid_instruction;
24694
24695		IrInstruction *casted_target = ir_implicit_cast(ira, awaiter_handle, ira->codegen->builtin_types.entry_promise);
24696		if (type_is_invalid(casted_target->value.type))
24697		return ira->codegen->invalid_instruction;
24698
24699		IrInstruction *result = ir_build_coro_resume(&ira->new_irb, instruction->base.scope,
24700		instruction->base.source_node, casted_target);
24701		result->value.type = ira->codegen->builtin_types.entry_void;
24702		return result;
24703		}
24704
24705		static IrInstruction ir_analyze_instruction_coro_save(IrAnalyze ira, IrInstructionCoroSave *instruction) {
24706		IrInstruction *coro_handle = instruction->coro_handle->child;
24707		if (type_is_invalid(coro_handle->value.type))
24708		return ira->codegen->invalid_instruction;
24709
24710		IrInstruction *result = ir_build_coro_save(&ira->new_irb, instruction->base.scope,
24711		instruction->base.source_node, coro_handle);
24712		result->value.type = ira->codegen->builtin_types.entry_usize;
24713		return result;
24714		}
24715
24716		static IrInstruction ir_analyze_instruction_coro_promise(IrAnalyze ira, IrInstructionCoroPromise *instruction) {
24717		IrInstruction *coro_handle = instruction->coro_handle->child;
24718		if (type_is_invalid(coro_handle->value.type))
24719		return ira->codegen->invalid_instruction;
24720
24721		if (coro_handle->value.type->id != ZigTypeIdPromise \|\|
24722		coro_handle->value.type->data.promise.result_type == nullptr)
24723		{
24724		ir_add_error(ira, &instruction->base, buf_sprintf("expected promise->T, found '%s'",
24725		buf_ptr(&coro_handle->value.type->name)));
24726		return ira->codegen->invalid_instruction;
24727		}
24728
24729		ZigType *coro_frame_type = get_promise_frame_type(ira->codegen,
24730		coro_handle->value.type->data.promise.result_type);
24731
24732		IrInstruction *result = ir_build_coro_promise(&ira->new_irb, instruction->base.scope,
24733		instruction->base.source_node, coro_handle);
24734		result->value.type = get_pointer_to_type(ira->codegen, coro_frame_type, false);
24735		return result;
24736		}
24737
24738		static IrInstruction ir_analyze_instruction_coro_alloc_helper(IrAnalyze ira, IrInstructionCoroAllocHelper *instruction) {
24739		IrInstruction *realloc_fn = instruction->realloc_fn->child;
24740		if (type_is_invalid(realloc_fn->value.type))
24741		return ira->codegen->invalid_instruction;
24742
24743		IrInstruction *coro_size = instruction->coro_size->child;
24744		if (type_is_invalid(coro_size->value.type))
24745		return ira->codegen->invalid_instruction;
24746
24747		IrInstruction *result = ir_build_coro_alloc_helper(&ira->new_irb, instruction->base.scope,
24748		instruction->base.source_node, realloc_fn, coro_size);
24749		ZigType *u8_ptr_type = get_pointer_to_type(ira->codegen, ira->codegen->builtin_types.entry_u8, false);
24750		result->value.type = get_optional_type(ira->codegen, u8_ptr_type);
24751		return result;
24752		}
24753
24754	23841	static ZigType ir_resolve_atomic_operand_type(IrAnalyze ira, IrInstruction *op) {
24755	23842	ZigType *operand_type = ir_resolve_type(ira, op);
24756	23843	if (type_is_invalid(operand_type))
...	...	@@ -24882,65 +23969,6 @@ static IrInstruction ir_analyze_instruction_atomic_load(IrAnalyze ira, IrInstr
24882	23969	return result;
24883	23970	}
24884	23971
24885		static IrInstruction ir_analyze_instruction_promise_result_type(IrAnalyze ira, IrInstructionPromiseResultType *instruction) {
24886		ZigType *promise_type = ir_resolve_type(ira, instruction->promise_type->child);
24887		if (type_is_invalid(promise_type))
24888		return ira->codegen->invalid_instruction;
24889
24890		if (promise_type->id != ZigTypeIdPromise \|\| promise_type->data.promise.result_type == nullptr) {
24891		ir_add_error(ira, &instruction->base, buf_sprintf("expected promise->T, found '%s'",
24892		buf_ptr(&promise_type->name)));
24893		return ira->codegen->invalid_instruction;
24894		}
24895
24896		return ir_const_type(ira, &instruction->base, promise_type->data.promise.result_type);
24897		}
24898
24899		static IrInstruction ir_analyze_instruction_await_bookkeeping(IrAnalyze ira, IrInstructionAwaitBookkeeping *instruction) {
24900		ZigType *promise_result_type = ir_resolve_type(ira, instruction->promise_result_type->child);
24901		if (type_is_invalid(promise_result_type))
24902		return ira->codegen->invalid_instruction;
24903
24904		ZigFn *fn_entry = exec_fn_entry(ira->new_irb.exec);
24905		ir_assert(fn_entry != nullptr, &instruction->base);
24906
24907		if (type_can_fail(promise_result_type)) {
24908		fn_entry->calls_or_awaits_errorable_fn = true;
24909		}
24910
24911		return ir_const_void(ira, &instruction->base);
24912		}
24913
24914		static IrInstruction ir_analyze_instruction_merge_err_ret_traces(IrAnalyze ira,
24915		IrInstructionMergeErrRetTraces *instruction)
24916		{
24917		IrInstruction *coro_promise_ptr = instruction->coro_promise_ptr->child;
24918		if (type_is_invalid(coro_promise_ptr->value.type))
24919		return ira->codegen->invalid_instruction;
24920
24921		ir_assert(coro_promise_ptr->value.type->id == ZigTypeIdPointer, &instruction->base);
24922		ZigType *promise_frame_type = coro_promise_ptr->value.type->data.pointer.child_type;
24923		ir_assert(promise_frame_type->id == ZigTypeIdStruct, &instruction->base);
24924		ZigType *promise_result_type = promise_frame_type->data.structure.fields[1].type_entry;
24925
24926		if (!type_can_fail(promise_result_type)) {
24927		return ir_const_void(ira, &instruction->base);
24928		}
24929
24930		IrInstruction *src_err_ret_trace_ptr = instruction->src_err_ret_trace_ptr->child;
24931		if (type_is_invalid(src_err_ret_trace_ptr->value.type))
24932		return ira->codegen->invalid_instruction;
24933
24934		IrInstruction *dest_err_ret_trace_ptr = instruction->dest_err_ret_trace_ptr->child;
24935		if (type_is_invalid(dest_err_ret_trace_ptr->value.type))
24936		return ira->codegen->invalid_instruction;
24937
24938		IrInstruction *result = ir_build_merge_err_ret_traces(&ira->new_irb, instruction->base.scope,
24939		instruction->base.source_node, coro_promise_ptr, src_err_ret_trace_ptr, dest_err_ret_trace_ptr);
24940		result->value.type = ira->codegen->builtin_types.entry_void;
24941		return result;
24942		}
24943
24944	23972	static IrInstruction ir_analyze_instruction_save_err_ret_addr(IrAnalyze ira, IrInstructionSaveErrRetAddr *instruction) {
24945	23973	IrInstruction *result = ir_build_save_err_ret_addr(&ira->new_irb, instruction->base.scope,
24946	23974	instruction->base.source_node);
...	...	@@ -24948,17 +23976,6 @@ static IrInstruction ir_analyze_instruction_save_err_ret_addr(IrAnalyze ira, I
24948	23976	return result;
24949	23977	}
24950	23978
24951		static IrInstruction ir_analyze_instruction_mark_err_ret_trace_ptr(IrAnalyze ira, IrInstructionMarkErrRetTracePtr *instruction) {
24952		IrInstruction *err_ret_trace_ptr = instruction->err_ret_trace_ptr->child;
24953		if (type_is_invalid(err_ret_trace_ptr->value.type))
24954		return ira->codegen->invalid_instruction;
24955
24956		IrInstruction *result = ir_build_mark_err_ret_trace_ptr(&ira->new_irb, instruction->base.scope,
24957		instruction->base.source_node, err_ret_trace_ptr);
24958		result->value.type = ira->codegen->builtin_types.entry_void;
24959		return result;
24960		}
24961
24962	23979	static void ir_eval_float_op(IrAnalyze ira, IrInstructionFloatOp source_instr, ZigType *float_type,
24963	23980	ConstExprValue op, ConstExprValue out_val) {
24964	23981	assert(ira && source_instr && float_type && out_val && op);
...	...	@@ -25485,6 +24502,162 @@ static IrInstruction ir_analyze_instruction_union_init_named_field(IrAnalyze i
25485	24502	union_type, field_name, field_result_loc, result_loc);
25486	24503	}
25487	24504
	24505	static IrInstruction ir_analyze_instruction_suspend_begin(IrAnalyze ira, IrInstructionSuspendBegin *instruction) {
	24506	IrInstructionSuspendBegin *result = ir_build_suspend_begin(&ira->new_irb, instruction->base.scope,
	24507	instruction->base.source_node);
	24508	return &result->base;
	24509	}
	24510
	24511	static IrInstruction ir_analyze_instruction_suspend_finish(IrAnalyze ira,
	24512	IrInstructionSuspendFinish *instruction)
	24513	{
	24514	IrInstruction *begin_base = instruction->begin->base.child;
	24515	if (type_is_invalid(begin_base->value.type))
	24516	return ira->codegen->invalid_instruction;
	24517	ir_assert(begin_base->id == IrInstructionIdSuspendBegin, &instruction->base);
	24518	IrInstructionSuspendBegin begin = reinterpret_cast<IrInstructionSuspendBegin >(begin_base);
	24519
	24520	ZigFn *fn_entry = exec_fn_entry(ira->new_irb.exec);
	24521	ir_assert(fn_entry != nullptr, &instruction->base);
	24522
	24523	if (fn_entry->inferred_async_node == nullptr) {
	24524	fn_entry->inferred_async_node = instruction->base.source_node;
	24525	}
	24526
	24527	return ir_build_suspend_finish(&ira->new_irb, instruction->base.scope, instruction->base.source_node, begin);
	24528	}
	24529
	24530	static IrInstruction analyze_frame_ptr_to_anyframe_T(IrAnalyze ira, IrInstruction *source_instr,
	24531	IrInstruction *frame_ptr)
	24532	{
	24533	if (type_is_invalid(frame_ptr->value.type))
	24534	return ira->codegen->invalid_instruction;
	24535
	24536	ZigType *result_type;
	24537	IrInstruction *frame;
	24538	if (frame_ptr->value.type->id == ZigTypeIdPointer &&
	24539	frame_ptr->value.type->data.pointer.ptr_len == PtrLenSingle &&
	24540	frame_ptr->value.type->data.pointer.child_type->id == ZigTypeIdFnFrame)
	24541	{
	24542	result_type = frame_ptr->value.type->data.pointer.child_type->data.frame.fn->type_entry->data.fn.fn_type_id.return_type;
	24543	frame = frame_ptr;
	24544	} else {
	24545	frame = ir_get_deref(ira, source_instr, frame_ptr, nullptr);
	24546	if (frame->value.type->id == ZigTypeIdPointer &&
	24547	frame->value.type->data.pointer.ptr_len == PtrLenSingle &&
	24548	frame->value.type->data.pointer.child_type->id == ZigTypeIdFnFrame)
	24549	{
	24550	result_type = frame->value.type->data.pointer.child_type->data.frame.fn->type_entry->data.fn.fn_type_id.return_type;
	24551	} else if (frame->value.type->id != ZigTypeIdAnyFrame \|\|
	24552	frame->value.type->data.any_frame.result_type == nullptr)
	24553	{
	24554	ir_add_error(ira, source_instr,
	24555	buf_sprintf("expected anyframe->T, found '%s'", buf_ptr(&frame->value.type->name)));
	24556	return ira->codegen->invalid_instruction;
	24557	} else {
	24558	result_type = frame->value.type->data.any_frame.result_type;
	24559	}
	24560	}
	24561
	24562	ZigType *any_frame_type = get_any_frame_type(ira->codegen, result_type);
	24563	IrInstruction *casted_frame = ir_implicit_cast(ira, frame, any_frame_type);
	24564	if (type_is_invalid(casted_frame->value.type))
	24565	return ira->codegen->invalid_instruction;
	24566
	24567	return casted_frame;
	24568	}
	24569
	24570	static IrInstruction ir_analyze_instruction_await(IrAnalyze ira, IrInstructionAwaitSrc *instruction) {
	24571	IrInstruction *frame = analyze_frame_ptr_to_anyframe_T(ira, &instruction->base, instruction->frame->child);
	24572	if (type_is_invalid(frame->value.type))
	24573	return ira->codegen->invalid_instruction;
	24574
	24575	ZigType *result_type = frame->value.type->data.any_frame.result_type;
	24576
	24577	ZigFn *fn_entry = exec_fn_entry(ira->new_irb.exec);
	24578	ir_assert(fn_entry != nullptr, &instruction->base);
	24579
	24580	if (fn_entry->inferred_async_node == nullptr) {
	24581	fn_entry->inferred_async_node = instruction->base.source_node;
	24582	}
	24583
	24584	if (type_can_fail(result_type)) {
	24585	fn_entry->calls_or_awaits_errorable_fn = true;
	24586	}
	24587
	24588	IrInstruction *result_loc;
	24589	if (type_has_bits(result_type)) {
	24590	result_loc = ir_resolve_result(ira, &instruction->base, instruction->result_loc,
	24591	result_type, nullptr, true, true, true);
	24592	if (result_loc != nullptr && (type_is_invalid(result_loc->value.type) \|\| instr_is_unreachable(result_loc)))
	24593	return result_loc;
	24594	} else {
	24595	result_loc = nullptr;
	24596	}
	24597
	24598	IrInstruction *result = ir_build_await_gen(ira, &instruction->base, frame, result_type, result_loc);
	24599	return ir_finish_anal(ira, result);
	24600	}
	24601
	24602	static IrInstruction ir_analyze_instruction_resume(IrAnalyze ira, IrInstructionResume *instruction) {
	24603	IrInstruction *frame_ptr = instruction->frame->child;
	24604	if (type_is_invalid(frame_ptr->value.type))
	24605	return ira->codegen->invalid_instruction;
	24606
	24607	IrInstruction *frame;
	24608	if (frame_ptr->value.type->id == ZigTypeIdPointer &&
	24609	frame_ptr->value.type->data.pointer.ptr_len == PtrLenSingle &&
	24610	frame_ptr->value.type->data.pointer.child_type->id == ZigTypeIdFnFrame)
	24611	{
	24612	frame = frame_ptr;
	24613	} else {
	24614	frame = ir_get_deref(ira, &instruction->base, frame_ptr, nullptr);
	24615	}
	24616
	24617	ZigType *any_frame_type = get_any_frame_type(ira->codegen, nullptr);
	24618	IrInstruction *casted_frame = ir_implicit_cast(ira, frame, any_frame_type);
	24619	if (type_is_invalid(casted_frame->value.type))
	24620	return ira->codegen->invalid_instruction;
	24621
	24622	return ir_build_resume(&ira->new_irb, instruction->base.scope, instruction->base.source_node, casted_frame);
	24623	}
	24624
	24625	static IrInstruction ir_analyze_instruction_spill_begin(IrAnalyze ira, IrInstructionSpillBegin *instruction) {
	24626	if (ir_should_inline(ira->new_irb.exec, instruction->base.scope))
	24627	return ir_const_void(ira, &instruction->base);
	24628
	24629	IrInstruction *operand = instruction->operand->child;
	24630	if (type_is_invalid(operand->value.type))
	24631	return ira->codegen->invalid_instruction;
	24632
	24633	if (!type_has_bits(operand->value.type))
	24634	return ir_const_void(ira, &instruction->base);
	24635
	24636	ir_assert(instruction->spill_id == SpillIdRetErrCode, &instruction->base);
	24637	ira->new_irb.exec->need_err_code_spill = true;
	24638
	24639	IrInstructionSpillBegin *result = ir_build_spill_begin(&ira->new_irb, instruction->base.scope,
	24640	instruction->base.source_node, operand, instruction->spill_id);
	24641	return &result->base;
	24642	}
	24643
	24644	static IrInstruction ir_analyze_instruction_spill_end(IrAnalyze ira, IrInstructionSpillEnd *instruction) {
	24645	IrInstruction *operand = instruction->begin->operand->child;
	24646	if (type_is_invalid(operand->value.type))
	24647	return ira->codegen->invalid_instruction;
	24648
	24649	if (ir_should_inline(ira->new_irb.exec, instruction->base.scope) \|\| !type_has_bits(operand->value.type))
	24650	return operand;
	24651
	24652	ir_assert(instruction->begin->base.child->id == IrInstructionIdSpillBegin, &instruction->base);
	24653	IrInstructionSpillBegin begin = reinterpret_cast<IrInstructionSpillBegin >(instruction->begin->base.child);
	24654
	24655	IrInstruction *result = ir_build_spill_end(&ira->new_irb, instruction->base.scope,
	24656	instruction->base.source_node, begin);
	24657	result->value.type = operand->value.type;
	24658	return result;
	24659	}
	24660
25488	24661	static IrInstruction ir_analyze_instruction_base(IrAnalyze ira, IrInstruction *instruction) {
25489	24662	switch (instruction->id) {
25490	24663	case IrInstructionIdInvalid:
...	...	@@ -25512,6 +24685,8 @@ static IrInstruction ir_analyze_instruction_base(IrAnalyze ira, IrInstruction
25512	24685	case IrInstructionIdSliceGen:
25513	24686	case IrInstructionIdRefGen:
25514	24687	case IrInstructionIdTestErrGen:
	24688	case IrInstructionIdFrameSizeGen:
	24689	case IrInstructionIdAwaitGen:
25515	24690	zig_unreachable();
25516	24691
25517	24692	case IrInstructionIdReturn:
...	...	@@ -25552,6 +24727,8 @@ static IrInstruction ir_analyze_instruction_base(IrAnalyze ira, IrInstruction
25552	24727	return ir_analyze_instruction_set_runtime_safety(ira, (IrInstructionSetRuntimeSafety *)instruction);
25553	24728	case IrInstructionIdSetFloatMode:
25554	24729	return ir_analyze_instruction_set_float_mode(ira, (IrInstructionSetFloatMode *)instruction);
	24730	case IrInstructionIdAnyFrameType:
	24731	return ir_analyze_instruction_any_frame_type(ira, (IrInstructionAnyFrameType *)instruction);
25555	24732	case IrInstructionIdSliceType:
25556	24733	return ir_analyze_instruction_slice_type(ira, (IrInstructionSliceType *)instruction);
25557	24734	case IrInstructionIdGlobalAsm:
...	...	@@ -25560,8 +24737,6 @@ static IrInstruction ir_analyze_instruction_base(IrAnalyze ira, IrInstruction
25560	24737	return ir_analyze_instruction_asm(ira, (IrInstructionAsm *)instruction);
25561	24738	case IrInstructionIdArrayType:
25562	24739	return ir_analyze_instruction_array_type(ira, (IrInstructionArrayType *)instruction);
25563		case IrInstructionIdPromiseType:
25564		return ir_analyze_instruction_promise_type(ira, (IrInstructionPromiseType *)instruction);
25565	24740	case IrInstructionIdSizeOf:
25566	24741	return ir_analyze_instruction_size_of(ira, (IrInstructionSizeOf *)instruction);
25567	24742	case IrInstructionIdTestNonNull:
...	...	@@ -25660,8 +24835,12 @@ static IrInstruction ir_analyze_instruction_base(IrAnalyze ira, IrInstruction
25660	24835	return ir_analyze_instruction_return_address(ira, (IrInstructionReturnAddress *)instruction);
25661	24836	case IrInstructionIdFrameAddress:
25662	24837	return ir_analyze_instruction_frame_address(ira, (IrInstructionFrameAddress *)instruction);
25663		case IrInstructionIdHandle:
25664		return ir_analyze_instruction_handle(ira, (IrInstructionHandle *)instruction);
	24838	case IrInstructionIdFrameHandle:
	24839	return ir_analyze_instruction_frame_handle(ira, (IrInstructionFrameHandle *)instruction);
	24840	case IrInstructionIdFrameType:
	24841	return ir_analyze_instruction_frame_type(ira, (IrInstructionFrameType *)instruction);
	24842	case IrInstructionIdFrameSizeSrc:
	24843	return ir_analyze_instruction_frame_size(ira, (IrInstructionFrameSizeSrc *)instruction);
25665	24844	case IrInstructionIdAlignOf:
25666	24845	return ir_analyze_instruction_align_of(ira, (IrInstructionAlignOf *)instruction);
25667	24846	case IrInstructionIdOverflowOp:
...	...	@@ -25716,8 +24895,6 @@ static IrInstruction ir_analyze_instruction_base(IrAnalyze ira, IrInstruction
25716	24895	return ir_analyze_instruction_resolve_result(ira, (IrInstructionResolveResult *)instruction);
25717	24896	case IrInstructionIdResetResult:
25718	24897	return ir_analyze_instruction_reset_result(ira, (IrInstructionResetResult *)instruction);
25719		case IrInstructionIdResultPtr:
25720		return ir_analyze_instruction_result_ptr(ira, (IrInstructionResultPtr *)instruction);
25721	24898	case IrInstructionIdOpaqueType:
25722	24899	return ir_analyze_instruction_opaque_type(ira, (IrInstructionOpaqueType *)instruction);
25723	24900	case IrInstructionIdSetAlignStack:
...	...	@@ -25732,50 +24909,14 @@ static IrInstruction ir_analyze_instruction_base(IrAnalyze ira, IrInstruction
25732	24909	return ir_analyze_instruction_error_return_trace(ira, (IrInstructionErrorReturnTrace *)instruction);
25733	24910	case IrInstructionIdErrorUnion:
25734	24911	return ir_analyze_instruction_error_union(ira, (IrInstructionErrorUnion *)instruction);
25735		case IrInstructionIdCancel:
25736		return ir_analyze_instruction_cancel(ira, (IrInstructionCancel *)instruction);
25737		case IrInstructionIdCoroId:
25738		return ir_analyze_instruction_coro_id(ira, (IrInstructionCoroId *)instruction);
25739		case IrInstructionIdCoroAlloc:
25740		return ir_analyze_instruction_coro_alloc(ira, (IrInstructionCoroAlloc *)instruction);
25741		case IrInstructionIdCoroSize:
25742		return ir_analyze_instruction_coro_size(ira, (IrInstructionCoroSize *)instruction);
25743		case IrInstructionIdCoroBegin:
25744		return ir_analyze_instruction_coro_begin(ira, (IrInstructionCoroBegin *)instruction);
25745		case IrInstructionIdGetImplicitAllocator:
25746		return ir_analyze_instruction_get_implicit_allocator(ira, (IrInstructionGetImplicitAllocator *)instruction);
25747		case IrInstructionIdCoroAllocFail:
25748		return ir_analyze_instruction_coro_alloc_fail(ira, (IrInstructionCoroAllocFail *)instruction);
25749		case IrInstructionIdCoroSuspend:
25750		return ir_analyze_instruction_coro_suspend(ira, (IrInstructionCoroSuspend *)instruction);
25751		case IrInstructionIdCoroEnd:
25752		return ir_analyze_instruction_coro_end(ira, (IrInstructionCoroEnd *)instruction);
25753		case IrInstructionIdCoroFree:
25754		return ir_analyze_instruction_coro_free(ira, (IrInstructionCoroFree *)instruction);
25755		case IrInstructionIdCoroResume:
25756		return ir_analyze_instruction_coro_resume(ira, (IrInstructionCoroResume *)instruction);
25757		case IrInstructionIdCoroSave:
25758		return ir_analyze_instruction_coro_save(ira, (IrInstructionCoroSave *)instruction);
25759		case IrInstructionIdCoroPromise:
25760		return ir_analyze_instruction_coro_promise(ira, (IrInstructionCoroPromise *)instruction);
25761		case IrInstructionIdCoroAllocHelper:
25762		return ir_analyze_instruction_coro_alloc_helper(ira, (IrInstructionCoroAllocHelper *)instruction);
25763	24912	case IrInstructionIdAtomicRmw:
25764	24913	return ir_analyze_instruction_atomic_rmw(ira, (IrInstructionAtomicRmw *)instruction);
25765	24914	case IrInstructionIdAtomicLoad:
25766	24915	return ir_analyze_instruction_atomic_load(ira, (IrInstructionAtomicLoad *)instruction);
25767		case IrInstructionIdPromiseResultType:
25768		return ir_analyze_instruction_promise_result_type(ira, (IrInstructionPromiseResultType *)instruction);
25769		case IrInstructionIdAwaitBookkeeping:
25770		return ir_analyze_instruction_await_bookkeeping(ira, (IrInstructionAwaitBookkeeping *)instruction);
25771	24916	case IrInstructionIdSaveErrRetAddr:
25772	24917	return ir_analyze_instruction_save_err_ret_addr(ira, (IrInstructionSaveErrRetAddr *)instruction);
25773	24918	case IrInstructionIdAddImplicitReturnType:
25774	24919	return ir_analyze_instruction_add_implicit_return_type(ira, (IrInstructionAddImplicitReturnType *)instruction);
25775		case IrInstructionIdMergeErrRetTraces:
25776		return ir_analyze_instruction_merge_err_ret_traces(ira, (IrInstructionMergeErrRetTraces *)instruction);
25777		case IrInstructionIdMarkErrRetTracePtr:
25778		return ir_analyze_instruction_mark_err_ret_trace_ptr(ira, (IrInstructionMarkErrRetTracePtr *)instruction);
25779	24920	case IrInstructionIdFloatOp:
25780	24921	return ir_analyze_instruction_float_op(ira, (IrInstructionFloatOp *)instruction);
25781	24922	case IrInstructionIdMulAdd:
...	...	@@ -25802,6 +24943,18 @@ static IrInstruction ir_analyze_instruction_base(IrAnalyze ira, IrInstruction
25802	24943	return ir_analyze_instruction_bit_cast_src(ira, (IrInstructionBitCastSrc *)instruction);
25803	24944	case IrInstructionIdUnionInitNamedField:
25804	24945	return ir_analyze_instruction_union_init_named_field(ira, (IrInstructionUnionInitNamedField *)instruction);
	24946	case IrInstructionIdSuspendBegin:
	24947	return ir_analyze_instruction_suspend_begin(ira, (IrInstructionSuspendBegin *)instruction);
	24948	case IrInstructionIdSuspendFinish:
	24949	return ir_analyze_instruction_suspend_finish(ira, (IrInstructionSuspendFinish *)instruction);
	24950	case IrInstructionIdResume:
	24951	return ir_analyze_instruction_resume(ira, (IrInstructionResume *)instruction);
	24952	case IrInstructionIdAwaitSrc:
	24953	return ir_analyze_instruction_await(ira, (IrInstructionAwaitSrc *)instruction);
	24954	case IrInstructionIdSpillBegin:
	24955	return ir_analyze_instruction_spill_begin(ira, (IrInstructionSpillBegin *)instruction);
	24956	case IrInstructionIdSpillEnd:
	24957	return ir_analyze_instruction_spill_end(ira, (IrInstructionSpillEnd *)instruction);
25805	24958	}
25806	24959	zig_unreachable();
25807	24960	}
...	...	@@ -25818,9 +24971,7 @@ ZigType ir_analyze(CodeGen codegen, IrExecutable old_exec, IrExecutable new_
25818	24971	old_exec->analysis = ira;
25819	24972	ira->codegen = codegen;
25820	24973
25821		ZigFn *fn_entry = exec_fn_entry(old_exec);
25822		bool is_async = fn_entry != nullptr && fn_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync;
25823		ira->explicit_return_type = is_async ? get_promise_type(codegen, expected_type) : expected_type;
	24974	ira->explicit_return_type = expected_type;
25824	24975	ira->explicit_return_type_source_node = expected_type_source_node;
25825	24976
25826	24977	ira->old_irb.codegen = codegen;
...	...	@@ -25918,19 +25069,8 @@ bool ir_has_side_effects(IrInstruction *instruction) {
25918	25069	case IrInstructionIdPtrType:
25919	25070	case IrInstructionIdSetAlignStack:
25920	25071	case IrInstructionIdExport:
25921		case IrInstructionIdCancel:
25922		case IrInstructionIdCoroId:
25923		case IrInstructionIdCoroBegin:
25924		case IrInstructionIdCoroAllocFail:
25925		case IrInstructionIdCoroEnd:
25926		case IrInstructionIdCoroResume:
25927		case IrInstructionIdCoroSave:
25928		case IrInstructionIdCoroAllocHelper:
25929		case IrInstructionIdAwaitBookkeeping:
25930	25072	case IrInstructionIdSaveErrRetAddr:
25931	25073	case IrInstructionIdAddImplicitReturnType:
25932		case IrInstructionIdMergeErrRetTraces:
25933		case IrInstructionIdMarkErrRetTracePtr:
25934	25074	case IrInstructionIdAtomicRmw:
25935	25075	case IrInstructionIdCmpxchgGen:
25936	25076	case IrInstructionIdCmpxchgSrc:
...	...	@@ -25945,6 +25085,12 @@ bool ir_has_side_effects(IrInstruction *instruction) {
25945	25085	case IrInstructionIdOptionalWrap:
25946	25086	case IrInstructionIdVectorToArray:
25947	25087	case IrInstructionIdResetResult:
	25088	case IrInstructionIdSuspendBegin:
	25089	case IrInstructionIdSuspendFinish:
	25090	case IrInstructionIdResume:
	25091	case IrInstructionIdAwaitSrc:
	25092	case IrInstructionIdAwaitGen:
	25093	case IrInstructionIdSpillBegin:
25948	25094	return true;
25949	25095
25950	25096	case IrInstructionIdPhi:
...	...	@@ -25963,8 +25109,8 @@ bool ir_has_side_effects(IrInstruction *instruction) {
25963	25109	case IrInstructionIdTypeOf:
25964	25110	case IrInstructionIdStructFieldPtr:
25965	25111	case IrInstructionIdArrayType:
25966		case IrInstructionIdPromiseType:
25967	25112	case IrInstructionIdSliceType:
	25113	case IrInstructionIdAnyFrameType:
25968	25114	case IrInstructionIdSizeOf:
25969	25115	case IrInstructionIdTestNonNull:
25970	25116	case IrInstructionIdOptionalUnwrapPtr:
...	...	@@ -25990,7 +25136,10 @@ bool ir_has_side_effects(IrInstruction *instruction) {
25990	25136	case IrInstructionIdAlignOf:
25991	25137	case IrInstructionIdReturnAddress:
25992	25138	case IrInstructionIdFrameAddress:
25993		case IrInstructionIdHandle:
	25139	case IrInstructionIdFrameHandle:
	25140	case IrInstructionIdFrameType:
	25141	case IrInstructionIdFrameSizeSrc:
	25142	case IrInstructionIdFrameSizeGen:
25994	25143	case IrInstructionIdTestErrSrc:
25995	25144	case IrInstructionIdTestErrGen:
25996	25145	case IrInstructionIdFnProto:
...	...	@@ -26023,13 +25172,6 @@ bool ir_has_side_effects(IrInstruction *instruction) {
26023	25172	case IrInstructionIdTagType:
26024	25173	case IrInstructionIdErrorReturnTrace:
26025	25174	case IrInstructionIdErrorUnion:
26026		case IrInstructionIdGetImplicitAllocator:
26027		case IrInstructionIdCoroAlloc:
26028		case IrInstructionIdCoroSize:
26029		case IrInstructionIdCoroSuspend:
26030		case IrInstructionIdCoroFree:
26031		case IrInstructionIdCoroPromise:
26032		case IrInstructionIdPromiseResultType:
26033	25175	case IrInstructionIdFloatOp:
26034	25176	case IrInstructionIdMulAdd:
26035	25177	case IrInstructionIdAtomicLoad:
...	...	@@ -26046,7 +25188,7 @@ bool ir_has_side_effects(IrInstruction *instruction) {
26046	25188	case IrInstructionIdHasDecl:
26047	25189	case IrInstructionIdAllocaSrc:
26048	25190	case IrInstructionIdAllocaGen:
26049		case IrInstructionIdResultPtr:
	25191	case IrInstructionIdSpillEnd:
26050	25192	return false;
26051	25193
26052	25194	case IrInstructionIdAsm:

src/ir.hpp+2

...	...	@@ -28,4 +28,6 @@ ConstExprValue const_ptr_pointee(IrAnalyze ira, CodeGen *codegen, ConstExprVal
28	28	AstNode *source_node);
29	29	const char *float_op_to_name(BuiltinFnId op, bool llvm_name);
30	30
	31	void ir_add_analysis_trace(IrAnalyze ira, ErrorMsg err_msg, Buf *text);
	32
31	33	#endif

src/ir_print.cpp+109-224

...	...	@@ -64,11 +64,9 @@ static void ir_print_other_block(IrPrint irp, IrBasicBlock bb) {
64	64	}
65	65	}
66	66
67		static void ir_print_return(IrPrint irp, IrInstructionReturn return_instruction) {
	67	static void ir_print_return(IrPrint irp, IrInstructionReturn instruction) {
68	68	fprintf(irp->f, "return ");
69		if (return_instruction->value != nullptr) {
70		ir_print_other_instruction(irp, return_instruction->value);
71		}
	69	ir_print_other_instruction(irp, instruction->operand);
72	70	}
73	71
74	72	static void ir_print_const(IrPrint irp, IrInstructionConst const_instruction) {
...	...	@@ -257,13 +255,7 @@ static void ir_print_result_loc(IrPrint irp, ResultLoc result_loc) {
257	255
258	256	static void ir_print_call_src(IrPrint irp, IrInstructionCallSrc call_instruction) {
259	257	if (call_instruction->is_async) {
260		fprintf(irp->f, "async");
261		if (call_instruction->async_allocator != nullptr) {
262		fprintf(irp->f, "<");
263		ir_print_other_instruction(irp, call_instruction->async_allocator);
264		fprintf(irp->f, ">");
265		}
266		fprintf(irp->f, " ");
	258	fprintf(irp->f, "async ");
267	259	}
268	260	if (call_instruction->fn_entry) {
269	261	fprintf(irp->f, "%s", buf_ptr(&call_instruction->fn_entry->symbol_name));
...	...	@@ -284,13 +276,7 @@ static void ir_print_call_src(IrPrint irp, IrInstructionCallSrc call_instructi
284	276
285	277	static void ir_print_call_gen(IrPrint irp, IrInstructionCallGen call_instruction) {
286	278	if (call_instruction->is_async) {
287		fprintf(irp->f, "async");
288		if (call_instruction->async_allocator != nullptr) {
289		fprintf(irp->f, "<");
290		ir_print_other_instruction(irp, call_instruction->async_allocator);
291		fprintf(irp->f, ">");
292		}
293		fprintf(irp->f, " ");
	279	fprintf(irp->f, "async ");
294	280	}
295	281	if (call_instruction->fn_entry) {
296	282	fprintf(irp->f, "%s", buf_ptr(&call_instruction->fn_entry->symbol_name));
...	...	@@ -477,20 +463,21 @@ static void ir_print_array_type(IrPrint irp, IrInstructionArrayType instructio
477	463	ir_print_other_instruction(irp, instruction->child_type);
478	464	}
479	465
480		static void ir_print_promise_type(IrPrint irp, IrInstructionPromiseType instruction) {
481		fprintf(irp->f, "promise");
482		if (instruction->payload_type != nullptr) {
483		fprintf(irp->f, "->");
484		ir_print_other_instruction(irp, instruction->payload_type);
485		}
486		}
487
488	466	static void ir_print_slice_type(IrPrint irp, IrInstructionSliceType instruction) {
489	467	const char *const_kw = instruction->is_const ? "const " : "";
490	468	fprintf(irp->f, "[]%s", const_kw);
491	469	ir_print_other_instruction(irp, instruction->child_type);
492	470	}
493	471
	472	static void ir_print_any_frame_type(IrPrint irp, IrInstructionAnyFrameType instruction) {
	473	if (instruction->payload_type == nullptr) {
	474	fprintf(irp->f, "anyframe");
	475	} else {
	476	fprintf(irp->f, "anyframe->");
	477	ir_print_other_instruction(irp, instruction->payload_type);
	478	}
	479	}
	480
494	481	static void ir_print_global_asm(IrPrint irp, IrInstructionGlobalAsm instruction) {
495	482	fprintf(irp->f, "asm(\"%s\")", buf_ptr(instruction->asm_code));
496	483	}
...	...	@@ -926,8 +913,26 @@ static void ir_print_frame_address(IrPrint irp, IrInstructionFrameAddress inst
926	913	fprintf(irp->f, "@frameAddress()");
927	914	}
928	915
929		static void ir_print_handle(IrPrint irp, IrInstructionHandle instruction) {
930		fprintf(irp->f, "@handle()");
	916	static void ir_print_handle(IrPrint irp, IrInstructionFrameHandle instruction) {
	917	fprintf(irp->f, "@frame()");
	918	}
	919
	920	static void ir_print_frame_type(IrPrint irp, IrInstructionFrameType instruction) {
	921	fprintf(irp->f, "@Frame(");
	922	ir_print_other_instruction(irp, instruction->fn);
	923	fprintf(irp->f, ")");
	924	}
	925
	926	static void ir_print_frame_size_src(IrPrint irp, IrInstructionFrameSizeSrc instruction) {
	927	fprintf(irp->f, "@frameSize(");
	928	ir_print_other_instruction(irp, instruction->fn);
	929	fprintf(irp->f, ")");
	930	}
	931
	932	static void ir_print_frame_size_gen(IrPrint irp, IrInstructionFrameSizeGen instruction) {
	933	fprintf(irp->f, "@frameSize(");
	934	ir_print_other_instruction(irp, instruction->fn);
	935	fprintf(irp->f, ")");
931	936	}
932	937
933	938	static void ir_print_return_address(IrPrint irp, IrInstructionReturnAddress instruction) {
...	...	@@ -1322,14 +1327,6 @@ static void ir_print_reset_result(IrPrint irp, IrInstructionResetResult instru
1322	1327	fprintf(irp->f, ")");
1323	1328	}
1324	1329
1325		static void ir_print_result_ptr(IrPrint irp, IrInstructionResultPtr instruction) {
1326		fprintf(irp->f, "ResultPtr(");
1327		ir_print_result_loc(irp, instruction->result_loc);
1328		fprintf(irp->f, ",");
1329		ir_print_other_instruction(irp, instruction->result);
1330		fprintf(irp->f, ")");
1331		}
1332
1333	1330	static void ir_print_opaque_type(IrPrint irp, IrInstructionOpaqueType instruction) {
1334	1331	fprintf(irp->f, "@OpaqueType()");
1335	1332	}
...	...	@@ -1391,110 +1388,6 @@ static void ir_print_error_union(IrPrint irp, IrInstructionErrorUnion instruct
1391	1388	ir_print_other_instruction(irp, instruction->payload);
1392	1389	}
1393	1390
1394		static void ir_print_cancel(IrPrint irp, IrInstructionCancel instruction) {
1395		fprintf(irp->f, "cancel ");
1396		ir_print_other_instruction(irp, instruction->target);
1397		}
1398
1399		static void ir_print_get_implicit_allocator(IrPrint irp, IrInstructionGetImplicitAllocator instruction) {
1400		fprintf(irp->f, "@getImplicitAllocator(");
1401		switch (instruction->id) {
1402		case ImplicitAllocatorIdArg:
1403		fprintf(irp->f, "Arg");
1404		break;
1405		case ImplicitAllocatorIdLocalVar:
1406		fprintf(irp->f, "LocalVar");
1407		break;
1408		}
1409		fprintf(irp->f, ")");
1410		}
1411
1412		static void ir_print_coro_id(IrPrint irp, IrInstructionCoroId instruction) {
1413		fprintf(irp->f, "@coroId(");
1414		ir_print_other_instruction(irp, instruction->promise_ptr);
1415		fprintf(irp->f, ")");
1416		}
1417
1418		static void ir_print_coro_alloc(IrPrint irp, IrInstructionCoroAlloc instruction) {
1419		fprintf(irp->f, "@coroAlloc(");
1420		ir_print_other_instruction(irp, instruction->coro_id);
1421		fprintf(irp->f, ")");
1422		}
1423
1424		static void ir_print_coro_size(IrPrint irp, IrInstructionCoroSize instruction) {
1425		fprintf(irp->f, "@coroSize()");
1426		}
1427
1428		static void ir_print_coro_begin(IrPrint irp, IrInstructionCoroBegin instruction) {
1429		fprintf(irp->f, "@coroBegin(");
1430		ir_print_other_instruction(irp, instruction->coro_id);
1431		fprintf(irp->f, ",");
1432		ir_print_other_instruction(irp, instruction->coro_mem_ptr);
1433		fprintf(irp->f, ")");
1434		}
1435
1436		static void ir_print_coro_alloc_fail(IrPrint irp, IrInstructionCoroAllocFail instruction) {
1437		fprintf(irp->f, "@coroAllocFail(");
1438		ir_print_other_instruction(irp, instruction->err_val);
1439		fprintf(irp->f, ")");
1440		}
1441
1442		static void ir_print_coro_suspend(IrPrint irp, IrInstructionCoroSuspend instruction) {
1443		fprintf(irp->f, "@coroSuspend(");
1444		if (instruction->save_point != nullptr) {
1445		ir_print_other_instruction(irp, instruction->save_point);
1446		} else {
1447		fprintf(irp->f, "null");
1448		}
1449		fprintf(irp->f, ",");
1450		ir_print_other_instruction(irp, instruction->is_final);
1451		fprintf(irp->f, ")");
1452		}
1453
1454		static void ir_print_coro_end(IrPrint irp, IrInstructionCoroEnd instruction) {
1455		fprintf(irp->f, "@coroEnd()");
1456		}
1457
1458		static void ir_print_coro_free(IrPrint irp, IrInstructionCoroFree instruction) {
1459		fprintf(irp->f, "@coroFree(");
1460		ir_print_other_instruction(irp, instruction->coro_id);
1461		fprintf(irp->f, ",");
1462		ir_print_other_instruction(irp, instruction->coro_handle);
1463		fprintf(irp->f, ")");
1464		}
1465
1466		static void ir_print_coro_resume(IrPrint irp, IrInstructionCoroResume instruction) {
1467		fprintf(irp->f, "@coroResume(");
1468		ir_print_other_instruction(irp, instruction->awaiter_handle);
1469		fprintf(irp->f, ")");
1470		}
1471
1472		static void ir_print_coro_save(IrPrint irp, IrInstructionCoroSave instruction) {
1473		fprintf(irp->f, "@coroSave(");
1474		ir_print_other_instruction(irp, instruction->coro_handle);
1475		fprintf(irp->f, ")");
1476		}
1477
1478		static void ir_print_coro_promise(IrPrint irp, IrInstructionCoroPromise instruction) {
1479		fprintf(irp->f, "@coroPromise(");
1480		ir_print_other_instruction(irp, instruction->coro_handle);
1481		fprintf(irp->f, ")");
1482		}
1483
1484		static void ir_print_promise_result_type(IrPrint irp, IrInstructionPromiseResultType instruction) {
1485		fprintf(irp->f, "@PromiseResultType(");
1486		ir_print_other_instruction(irp, instruction->promise_type);
1487		fprintf(irp->f, ")");
1488		}
1489
1490		static void ir_print_coro_alloc_helper(IrPrint irp, IrInstructionCoroAllocHelper instruction) {
1491		fprintf(irp->f, "@coroAllocHelper(");
1492		ir_print_other_instruction(irp, instruction->realloc_fn);
1493		fprintf(irp->f, ",");
1494		ir_print_other_instruction(irp, instruction->coro_size);
1495		fprintf(irp->f, ")");
1496		}
1497
1498	1391	static void ir_print_atomic_rmw(IrPrint irp, IrInstructionAtomicRmw instruction) {
1499	1392	fprintf(irp->f, "@atomicRmw(");
1500	1393	if (instruction->operand_type != nullptr) {
...	...	@@ -1539,12 +1432,6 @@ static void ir_print_atomic_load(IrPrint irp, IrInstructionAtomicLoad instruct
1539	1432	fprintf(irp->f, ")");
1540	1433	}
1541	1434
1542		static void ir_print_await_bookkeeping(IrPrint irp, IrInstructionAwaitBookkeeping instruction) {
1543		fprintf(irp->f, "@awaitBookkeeping(");
1544		ir_print_other_instruction(irp, instruction->promise_result_type);
1545		fprintf(irp->f, ")");
1546		}
1547
1548	1435	static void ir_print_save_err_ret_addr(IrPrint irp, IrInstructionSaveErrRetAddr instruction) {
1549	1436	fprintf(irp->f, "@saveErrRetAddr()");
1550	1437	}
...	...	@@ -1555,22 +1442,6 @@ static void ir_print_add_implicit_return_type(IrPrint *irp, IrInstructionAddImpl
1555	1442	fprintf(irp->f, ")");
1556	1443	}
1557	1444
1558		static void ir_print_merge_err_ret_traces(IrPrint irp, IrInstructionMergeErrRetTraces instruction) {
1559		fprintf(irp->f, "@mergeErrRetTraces(");
1560		ir_print_other_instruction(irp, instruction->coro_promise_ptr);
1561		fprintf(irp->f, ",");
1562		ir_print_other_instruction(irp, instruction->src_err_ret_trace_ptr);
1563		fprintf(irp->f, ",");
1564		ir_print_other_instruction(irp, instruction->dest_err_ret_trace_ptr);
1565		fprintf(irp->f, ")");
1566		}
1567
1568		static void ir_print_mark_err_ret_trace_ptr(IrPrint irp, IrInstructionMarkErrRetTracePtr instruction) {
1569		fprintf(irp->f, "@markErrRetTracePtr(");
1570		ir_print_other_instruction(irp, instruction->err_ret_trace_ptr);
1571		fprintf(irp->f, ")");
1572		}
1573
1574	1445	static void ir_print_float_op(IrPrint irp, IrInstructionFloatOp instruction) {
1575	1446
1576	1447	fprintf(irp->f, "@%s(", float_op_to_name(instruction->op, false));
...	...	@@ -1638,6 +1509,47 @@ static void ir_print_union_init_named_field(IrPrint *irp, IrInstructionUnionInit
1638	1509	fprintf(irp->f, ")");
1639	1510	}
1640	1511
	1512	static void ir_print_suspend_begin(IrPrint irp, IrInstructionSuspendBegin instruction) {
	1513	fprintf(irp->f, "@suspendBegin()");
	1514	}
	1515
	1516	static void ir_print_suspend_finish(IrPrint irp, IrInstructionSuspendFinish instruction) {
	1517	fprintf(irp->f, "@suspendFinish()");
	1518	}
	1519
	1520	static void ir_print_resume(IrPrint irp, IrInstructionResume instruction) {
	1521	fprintf(irp->f, "resume ");
	1522	ir_print_other_instruction(irp, instruction->frame);
	1523	}
	1524
	1525	static void ir_print_await_src(IrPrint irp, IrInstructionAwaitSrc instruction) {
	1526	fprintf(irp->f, "@await(");
	1527	ir_print_other_instruction(irp, instruction->frame);
	1528	fprintf(irp->f, ",");
	1529	ir_print_result_loc(irp, instruction->result_loc);
	1530	fprintf(irp->f, ")");
	1531	}
	1532
	1533	static void ir_print_await_gen(IrPrint irp, IrInstructionAwaitGen instruction) {
	1534	fprintf(irp->f, "@await(");
	1535	ir_print_other_instruction(irp, instruction->frame);
	1536	fprintf(irp->f, ",");
	1537	ir_print_other_instruction(irp, instruction->result_loc);
	1538	fprintf(irp->f, ")");
	1539	}
	1540
	1541	static void ir_print_spill_begin(IrPrint irp, IrInstructionSpillBegin instruction) {
	1542	fprintf(irp->f, "@spillBegin(");
	1543	ir_print_other_instruction(irp, instruction->operand);
	1544	fprintf(irp->f, ")");
	1545	}
	1546
	1547	static void ir_print_spill_end(IrPrint irp, IrInstructionSpillEnd instruction) {
	1548	fprintf(irp->f, "@spillEnd(");
	1549	ir_print_other_instruction(irp, &instruction->begin->base);
	1550	fprintf(irp->f, ")");
	1551	}
	1552
1641	1553	static void ir_print_instruction(IrPrint irp, IrInstruction instruction) {
1642	1554	ir_print_prefix(irp, instruction);
1643	1555	switch (instruction->id) {
...	...	@@ -1727,12 +1639,12 @@ static void ir_print_instruction(IrPrint irp, IrInstruction instruction) {
1727	1639	case IrInstructionIdArrayType:
1728	1640	ir_print_array_type(irp, (IrInstructionArrayType *)instruction);
1729	1641	break;
1730		case IrInstructionIdPromiseType:
1731		ir_print_promise_type(irp, (IrInstructionPromiseType *)instruction);
1732		break;
1733	1642	case IrInstructionIdSliceType:
1734	1643	ir_print_slice_type(irp, (IrInstructionSliceType *)instruction);
1735	1644	break;
	1645	case IrInstructionIdAnyFrameType:
	1646	ir_print_any_frame_type(irp, (IrInstructionAnyFrameType *)instruction);
	1647	break;
1736	1648	case IrInstructionIdGlobalAsm:
1737	1649	ir_print_global_asm(irp, (IrInstructionGlobalAsm *)instruction);
1738	1650	break;
...	...	@@ -1886,8 +1798,17 @@ static void ir_print_instruction(IrPrint irp, IrInstruction instruction) {
1886	1798	case IrInstructionIdFrameAddress:
1887	1799	ir_print_frame_address(irp, (IrInstructionFrameAddress *)instruction);
1888	1800	break;
1889		case IrInstructionIdHandle:
1890		ir_print_handle(irp, (IrInstructionHandle *)instruction);
	1801	case IrInstructionIdFrameHandle:
	1802	ir_print_handle(irp, (IrInstructionFrameHandle *)instruction);
	1803	break;
	1804	case IrInstructionIdFrameType:
	1805	ir_print_frame_type(irp, (IrInstructionFrameType *)instruction);
	1806	break;
	1807	case IrInstructionIdFrameSizeSrc:
	1808	ir_print_frame_size_src(irp, (IrInstructionFrameSizeSrc *)instruction);
	1809	break;
	1810	case IrInstructionIdFrameSizeGen:
	1811	ir_print_frame_size_gen(irp, (IrInstructionFrameSizeGen *)instruction);
1891	1812	break;
1892	1813	case IrInstructionIdAlignOf:
1893	1814	ir_print_align_of(irp, (IrInstructionAlignOf *)instruction);
...	...	@@ -2006,9 +1927,6 @@ static void ir_print_instruction(IrPrint irp, IrInstruction instruction) {
2006	1927	case IrInstructionIdResetResult:
2007	1928	ir_print_reset_result(irp, (IrInstructionResetResult *)instruction);
2008	1929	break;
2009		case IrInstructionIdResultPtr:
2010		ir_print_result_ptr(irp, (IrInstructionResultPtr *)instruction);
2011		break;
2012	1930	case IrInstructionIdOpaqueType:
2013	1931	ir_print_opaque_type(irp, (IrInstructionOpaqueType *)instruction);
2014	1932	break;
...	...	@@ -2030,69 +1948,15 @@ static void ir_print_instruction(IrPrint irp, IrInstruction instruction) {
2030	1948	case IrInstructionIdErrorUnion:
2031	1949	ir_print_error_union(irp, (IrInstructionErrorUnion *)instruction);
2032	1950	break;
2033		case IrInstructionIdCancel:
2034		ir_print_cancel(irp, (IrInstructionCancel *)instruction);
2035		break;
2036		case IrInstructionIdGetImplicitAllocator:
2037		ir_print_get_implicit_allocator(irp, (IrInstructionGetImplicitAllocator *)instruction);
2038		break;
2039		case IrInstructionIdCoroId:
2040		ir_print_coro_id(irp, (IrInstructionCoroId *)instruction);
2041		break;
2042		case IrInstructionIdCoroAlloc:
2043		ir_print_coro_alloc(irp, (IrInstructionCoroAlloc *)instruction);
2044		break;
2045		case IrInstructionIdCoroSize:
2046		ir_print_coro_size(irp, (IrInstructionCoroSize *)instruction);
2047		break;
2048		case IrInstructionIdCoroBegin:
2049		ir_print_coro_begin(irp, (IrInstructionCoroBegin *)instruction);
2050		break;
2051		case IrInstructionIdCoroAllocFail:
2052		ir_print_coro_alloc_fail(irp, (IrInstructionCoroAllocFail *)instruction);
2053		break;
2054		case IrInstructionIdCoroSuspend:
2055		ir_print_coro_suspend(irp, (IrInstructionCoroSuspend *)instruction);
2056		break;
2057		case IrInstructionIdCoroEnd:
2058		ir_print_coro_end(irp, (IrInstructionCoroEnd *)instruction);
2059		break;
2060		case IrInstructionIdCoroFree:
2061		ir_print_coro_free(irp, (IrInstructionCoroFree *)instruction);
2062		break;
2063		case IrInstructionIdCoroResume:
2064		ir_print_coro_resume(irp, (IrInstructionCoroResume *)instruction);
2065		break;
2066		case IrInstructionIdCoroSave:
2067		ir_print_coro_save(irp, (IrInstructionCoroSave *)instruction);
2068		break;
2069		case IrInstructionIdCoroAllocHelper:
2070		ir_print_coro_alloc_helper(irp, (IrInstructionCoroAllocHelper *)instruction);
2071		break;
2072	1951	case IrInstructionIdAtomicRmw:
2073	1952	ir_print_atomic_rmw(irp, (IrInstructionAtomicRmw *)instruction);
2074	1953	break;
2075		case IrInstructionIdCoroPromise:
2076		ir_print_coro_promise(irp, (IrInstructionCoroPromise *)instruction);
2077		break;
2078		case IrInstructionIdPromiseResultType:
2079		ir_print_promise_result_type(irp, (IrInstructionPromiseResultType *)instruction);
2080		break;
2081		case IrInstructionIdAwaitBookkeeping:
2082		ir_print_await_bookkeeping(irp, (IrInstructionAwaitBookkeeping *)instruction);
2083		break;
2084	1954	case IrInstructionIdSaveErrRetAddr:
2085	1955	ir_print_save_err_ret_addr(irp, (IrInstructionSaveErrRetAddr *)instruction);
2086	1956	break;
2087	1957	case IrInstructionIdAddImplicitReturnType:
2088	1958	ir_print_add_implicit_return_type(irp, (IrInstructionAddImplicitReturnType *)instruction);
2089	1959	break;
2090		case IrInstructionIdMergeErrRetTraces:
2091		ir_print_merge_err_ret_traces(irp, (IrInstructionMergeErrRetTraces *)instruction);
2092		break;
2093		case IrInstructionIdMarkErrRetTracePtr:
2094		ir_print_mark_err_ret_trace_ptr(irp, (IrInstructionMarkErrRetTracePtr *)instruction);
2095		break;
2096	1960	case IrInstructionIdFloatOp:
2097	1961	ir_print_float_op(irp, (IrInstructionFloatOp *)instruction);
2098	1962	break;
...	...	@@ -2147,6 +2011,27 @@ static void ir_print_instruction(IrPrint irp, IrInstruction instruction) {
2147	2011	case IrInstructionIdUnionInitNamedField:
2148	2012	ir_print_union_init_named_field(irp, (IrInstructionUnionInitNamedField *)instruction);
2149	2013	break;
	2014	case IrInstructionIdSuspendBegin:
	2015	ir_print_suspend_begin(irp, (IrInstructionSuspendBegin *)instruction);
	2016	break;
	2017	case IrInstructionIdSuspendFinish:
	2018	ir_print_suspend_finish(irp, (IrInstructionSuspendFinish *)instruction);
	2019	break;
	2020	case IrInstructionIdResume:
	2021	ir_print_resume(irp, (IrInstructionResume *)instruction);
	2022	break;
	2023	case IrInstructionIdAwaitSrc:
	2024	ir_print_await_src(irp, (IrInstructionAwaitSrc *)instruction);
	2025	break;
	2026	case IrInstructionIdAwaitGen:
	2027	ir_print_await_gen(irp, (IrInstructionAwaitGen *)instruction);
	2028	break;
	2029	case IrInstructionIdSpillBegin:
	2030	ir_print_spill_begin(irp, (IrInstructionSpillBegin *)instruction);
	2031	break;
	2032	case IrInstructionIdSpillEnd:
	2033	ir_print_spill_end(irp, (IrInstructionSpillEnd *)instruction);
	2034	break;
2150	2035	}
2151	2036	fprintf(irp->f, "\n");
2152	2037	}

src/parser.cpp+11-35

...	...	@@ -282,8 +282,8 @@ static AstNode *ast_parse_prefix_op_expr(
282	282	case NodeTypeAwaitExpr:
283	283	right = &prefix->data.await_expr.expr;
284	284	break;
285		case NodeTypePromiseType:
286		right = &prefix->data.promise_type.payload_type;
	285	case NodeTypeAnyFrameType:
	286	right = &prefix->data.anyframe_type.payload_type;
287	287	break;
288	288	case NodeTypeArrayType:
289	289	right = &prefix->data.array_type.child_type;
...	...	@@ -1167,7 +1167,6 @@ static AstNode ast_parse_prefix_expr(ParseContext pc) {
1167	1167	// <- AsmExpr
1168	1168	// / IfExpr
1169	1169	// / KEYWORD_break BreakLabel? Expr?
1170		// / KEYWORD_cancel Expr
1171	1170	// / KEYWORD_comptime Expr
1172	1171	// / KEYWORD_continue BreakLabel?
1173	1172	// / KEYWORD_resume Expr
...	...	@@ -1195,14 +1194,6 @@ static AstNode ast_parse_primary_expr(ParseContext pc) {
1195	1194	return res;
1196	1195	}
1197	1196
1198		Token *cancel = eat_token_if(pc, TokenIdKeywordCancel);
1199		if (cancel != nullptr) {
1200		AstNode *expr = ast_expect(pc, ast_parse_expr);
1201		AstNode *res = ast_create_node(pc, NodeTypeCancel, cancel);
1202		res->data.cancel_expr.expr = expr;
1203		return res;
1204		}
1205
1206	1197	Token *comptime = eat_token_if(pc, TokenIdKeywordCompTime);
1207	1198	if (comptime != nullptr) {
1208	1199	AstNode *expr = ast_expect(pc, ast_parse_expr);
...	...	@@ -1643,9 +1634,9 @@ static AstNode ast_parse_primary_type_expr(ParseContext pc) {
1643	1634	if (null != nullptr)
1644	1635	return ast_create_node(pc, NodeTypeNullLiteral, null);
1645	1636
1646		Token *promise = eat_token_if(pc, TokenIdKeywordPromise);
1647		if (promise != nullptr)
1648		return ast_create_node(pc, NodeTypePromiseType, promise);
	1637	Token *anyframe = eat_token_if(pc, TokenIdKeywordAnyFrame);
	1638	if (anyframe != nullptr)
	1639	return ast_create_node(pc, NodeTypeAnyFrameType, anyframe);
1649	1640
1650	1641	Token *true_token = eat_token_if(pc, TokenIdKeywordTrue);
1651	1642	if (true_token != nullptr) {
...	...	@@ -2042,11 +2033,6 @@ static Optional<AstNodeFnProto> ast_parse_fn_cc(ParseContext *pc) {
2042	2033	}
2043	2034	if (eat_token_if(pc, TokenIdKeywordAsync) != nullptr) {
2044	2035	res.cc = CallingConventionAsync;
2045		if (eat_token_if(pc, TokenIdCmpLessThan) == nullptr)
2046		return Optional<AstNodeFnProto>::some(res);
2047
2048		res.async_allocator_type = ast_expect(pc, ast_parse_type_expr);
2049		expect_token(pc, TokenIdCmpGreaterThan);
2050	2036	return Optional<AstNodeFnProto>::some(res);
2051	2037	}
2052	2038
...	...	@@ -2522,7 +2508,7 @@ static AstNode ast_parse_prefix_op(ParseContext pc) {
2522	2508
2523	2509	// PrefixTypeOp
2524	2510	// <- QUESTIONMARK
2525		// / KEYWORD_promise MINUSRARROW
	2511	// / KEYWORD_anyframe MINUSRARROW
2526	2512	// / ArrayTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile)*
2527	2513	// / PtrTypeStart (KEYWORD_align LPAREN Expr (COLON INTEGER COLON INTEGER)? RPAREN / KEYWORD_const / KEYWORD_volatile)*
2528	2514	static AstNode ast_parse_prefix_type_op(ParseContext pc) {
...	...	@@ -2533,10 +2519,10 @@ static AstNode ast_parse_prefix_type_op(ParseContext pc) {
2533	2519	return res;
2534	2520	}
2535	2521
2536		Token *promise = eat_token_if(pc, TokenIdKeywordPromise);
2537		if (promise != nullptr) {
	2522	Token *anyframe = eat_token_if(pc, TokenIdKeywordAnyFrame);
	2523	if (anyframe != nullptr) {
2538	2524	if (eat_token_if(pc, TokenIdArrow) != nullptr) {
2539		AstNode *res = ast_create_node(pc, NodeTypePromiseType, promise);
	2525	AstNode *res = ast_create_node(pc, NodeTypeAnyFrameType, anyframe);
2540	2526	return res;
2541	2527	}
2542	2528
...	...	@@ -2680,11 +2666,6 @@ static AstNode ast_parse_async_prefix(ParseContext pc) {
2680	2666	AstNode *res = ast_create_node(pc, NodeTypeFnCallExpr, async);
2681	2667	res->data.fn_call_expr.is_async = true;
2682	2668	res->data.fn_call_expr.seen = false;
2683		if (eat_token_if(pc, TokenIdCmpLessThan) != nullptr) {
2684		AstNode *prefix_expr = ast_expect(pc, ast_parse_prefix_expr);
2685		expect_token(pc, TokenIdCmpGreaterThan);
2686		res->data.fn_call_expr.async_allocator = prefix_expr;
2687		}
2688	2669
2689	2670	return res;
2690	2671	}
...	...	@@ -2858,7 +2839,6 @@ void ast_visit_node_children(AstNode node, void (visit)(AstNode *, void cont
2858	2839	visit_node_list(&node->data.fn_proto.params, visit, context);
2859	2840	visit_field(&node->data.fn_proto.align_expr, visit, context);
2860	2841	visit_field(&node->data.fn_proto.section_expr, visit, context);
2861		visit_field(&node->data.fn_proto.async_allocator_type, visit, context);
2862	2842	break;
2863	2843	case NodeTypeFnDef:
2864	2844	visit_field(&node->data.fn_def.fn_proto, visit, context);
...	...	@@ -2918,7 +2898,6 @@ void ast_visit_node_children(AstNode node, void (visit)(AstNode *, void cont
2918	2898	case NodeTypeFnCallExpr:
2919	2899	visit_field(&node->data.fn_call_expr.fn_ref_expr, visit, context);
2920	2900	visit_node_list(&node->data.fn_call_expr.params, visit, context);
2921		visit_field(&node->data.fn_call_expr.async_allocator, visit, context);
2922	2901	break;
2923	2902	case NodeTypeArrayAccessExpr:
2924	2903	visit_field(&node->data.array_access_expr.array_ref_expr, visit, context);
...	...	@@ -3034,8 +3013,8 @@ void ast_visit_node_children(AstNode node, void (visit)(AstNode *, void cont
3034	3013	case NodeTypeInferredArrayType:
3035	3014	visit_field(&node->data.array_type.child_type, visit, context);
3036	3015	break;
3037		case NodeTypePromiseType:
3038		visit_field(&node->data.promise_type.payload_type, visit, context);
	3016	case NodeTypeAnyFrameType:
	3017	visit_field(&node->data.anyframe_type.payload_type, visit, context);
3039	3018	break;
3040	3019	case NodeTypeErrorType:
3041	3020	// none
...	...	@@ -3047,9 +3026,6 @@ void ast_visit_node_children(AstNode node, void (visit)(AstNode *, void cont
3047	3026	case NodeTypeErrorSetDecl:
3048	3027	visit_node_list(&node->data.err_set_decl.decls, visit, context);
3049	3028	break;
3050		case NodeTypeCancel:
3051		visit_field(&node->data.cancel_expr.expr, visit, context);
3052		break;
3053	3029	case NodeTypeResume:
3054	3030	visit_field(&node->data.resume_expr.expr, visit, context);
3055	3031	break;

src/target.cpp+4

...	...	@@ -1759,3 +1759,7 @@ bool target_supports_libunwind(const ZigTarget *target) {
1759	1759	return true;
1760	1760	}
1761	1761
	1762
	1763	unsigned target_fn_align(const ZigTarget *target) {
	1764	return 16;
	1765	}

src/target.hpp+2

...	...	@@ -197,4 +197,6 @@ uint32_t target_arch_largest_atomic_bits(ZigLLVM_ArchType arch);
197	197	size_t target_libc_count(void);
198	198	void target_libc_enum(size_t index, ZigTarget *out_target);
199	199
	200	unsigned target_fn_align(const ZigTarget *target);
	201
200	202	#endif

src/tokenizer.cpp+2-4

...	...	@@ -109,11 +109,11 @@ static const struct ZigKeyword zig_keywords[] = {
109	109	{"align", TokenIdKeywordAlign},
110	110	{"allowzero", TokenIdKeywordAllowZero},
111	111	{"and", TokenIdKeywordAnd},
	112	{"anyframe", TokenIdKeywordAnyFrame},
112	113	{"asm", TokenIdKeywordAsm},
113	114	{"async", TokenIdKeywordAsync},
114	115	{"await", TokenIdKeywordAwait},
115	116	{"break", TokenIdKeywordBreak},
116		{"cancel", TokenIdKeywordCancel},
117	117	{"catch", TokenIdKeywordCatch},
118	118	{"comptime", TokenIdKeywordCompTime},
119	119	{"const", TokenIdKeywordConst},
...	...	@@ -136,7 +136,6 @@ static const struct ZigKeyword zig_keywords[] = {
136	136	{"or", TokenIdKeywordOr},
137	137	{"orelse", TokenIdKeywordOrElse},
138	138	{"packed", TokenIdKeywordPacked},
139		{"promise", TokenIdKeywordPromise},
140	139	{"pub", TokenIdKeywordPub},
141	140	{"resume", TokenIdKeywordResume},
142	141	{"return", TokenIdKeywordReturn},
...	...	@@ -1531,9 +1530,9 @@ const char * token_name(TokenId id) {
1531	1530	case TokenIdKeywordAwait: return "await";
1532	1531	case TokenIdKeywordResume: return "resume";
1533	1532	case TokenIdKeywordSuspend: return "suspend";
1534		case TokenIdKeywordCancel: return "cancel";
1535	1533	case TokenIdKeywordAlign: return "align";
1536	1534	case TokenIdKeywordAnd: return "and";
	1535	case TokenIdKeywordAnyFrame: return "anyframe";
1537	1536	case TokenIdKeywordAsm: return "asm";
1538	1537	case TokenIdKeywordBreak: return "break";
1539	1538	case TokenIdKeywordCatch: return "catch";
...	...	@@ -1558,7 +1557,6 @@ const char * token_name(TokenId id) {
1558	1557	case TokenIdKeywordOr: return "or";
1559	1558	case TokenIdKeywordOrElse: return "orelse";
1560	1559	case TokenIdKeywordPacked: return "packed";
1561		case TokenIdKeywordPromise: return "promise";
1562	1560	case TokenIdKeywordPub: return "pub";
1563	1561	case TokenIdKeywordReturn: return "return";
1564	1562	case TokenIdKeywordLinkSection: return "linksection";

src/tokenizer.hpp+1-2

...	...	@@ -53,11 +53,11 @@ enum TokenId {
53	53	TokenIdKeywordAlign,
54	54	TokenIdKeywordAllowZero,
55	55	TokenIdKeywordAnd,
	56	TokenIdKeywordAnyFrame,
56	57	TokenIdKeywordAsm,
57	58	TokenIdKeywordAsync,
58	59	TokenIdKeywordAwait,
59	60	TokenIdKeywordBreak,
60		TokenIdKeywordCancel,
61	61	TokenIdKeywordCatch,
62	62	TokenIdKeywordCompTime,
63	63	TokenIdKeywordConst,
...	...	@@ -81,7 +81,6 @@ enum TokenId {
81	81	TokenIdKeywordOr,
82	82	TokenIdKeywordOrElse,
83	83	TokenIdKeywordPacked,
84		TokenIdKeywordPromise,
85	84	TokenIdKeywordPub,
86	85	TokenIdKeywordResume,
87	86	TokenIdKeywordReturn,

src/zig_llvm.cpp+8-3

...	...	@@ -42,7 +42,6 @@
42	42	#include <llvm/Support/TargetRegistry.h>
43	43	#include <llvm/Target/TargetMachine.h>
44	44	#include <llvm/Target/CodeGenCWrappers.h>
45		#include <llvm/Transforms/Coroutines.h>
46	45	#include <llvm/Transforms/IPO.h>
47	46	#include <llvm/Transforms/IPO/AlwaysInliner.h>
48	47	#include <llvm/Transforms/IPO/PassManagerBuilder.h>
...	...	@@ -203,8 +202,6 @@ bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMM
203	202	PMBuilder->Inliner = createFunctionInliningPass(PMBuilder->OptLevel, PMBuilder->SizeLevel, false);
204	203	}
205	204
206		addCoroutinePassesToExtensionPoints(*PMBuilder);
207
208	205	// Set up the per-function pass manager.
209	206	legacy::FunctionPassManager FPM = legacy::FunctionPassManager(module);
210	207	auto tliwp = new(std::nothrow) TargetLibraryInfoWrapperPass(tlii);
...	...	@@ -898,6 +895,14 @@ LLVMValueRef ZigLLVMBuildAShrExact(LLVMBuilderRef builder, LLVMValueRef LHS, LLV
898	895	return wrap(unwrap(builder)->CreateAShr(unwrap(LHS), unwrap(RHS), name, true));
899	896	}
900	897
	898	void ZigLLVMSetTailCall(LLVMValueRef Call) {
	899	unwrap<CallInst>(Call)->setTailCallKind(CallInst::TCK_MustTail);
	900	}
	901
	902	void ZigLLVMFunctionSetPrefixData(LLVMValueRef function, LLVMValueRef data) {
	903	unwrap<Function>(function)->setPrefixData(unwrap<Constant>(data));
	904	}
	905
901	906
902	907	class MyOStream: public raw_ostream {
903	908	public:

src/zig_llvm.h+2

...	...	@@ -211,6 +211,8 @@ ZIG_EXTERN_C LLVMValueRef ZigLLVMInsertDeclare(struct ZigLLVMDIBuilder *dibuilde
211	211	ZIG_EXTERN_C struct ZigLLVMDILocation ZigLLVMGetDebugLoc(unsigned line, unsigned col, struct ZigLLVMDIScope scope);
212	212
213	213	ZIG_EXTERN_C void ZigLLVMSetFastMath(LLVMBuilderRef builder_wrapped, bool on_state);
	214	ZIG_EXTERN_C void ZigLLVMSetTailCall(LLVMValueRef Call);
	215	ZIG_EXTERN_C void ZigLLVMFunctionSetPrefixData(LLVMValueRef fn, LLVMValueRef data);
214	216
215	217	ZIG_EXTERN_C void ZigLLVMAddFunctionAttr(LLVMValueRef fn, const char attr_name, const char attr_value);
216	218	ZIG_EXTERN_C void ZigLLVMAddFunctionAttrCold(LLVMValueRef fn);

std/event/channel.zig+45-72

...	...	@@ -2,8 +2,6 @@ const std = @import("../std.zig");
2	2	const builtin = @import("builtin");
3	3	const assert = std.debug.assert;
4	4	const testing = std.testing;
5		const AtomicRmwOp = builtin.AtomicRmwOp;
6		const AtomicOrder = builtin.AtomicOrder;
7	5	const Loop = std.event.Loop;
8	6
9	7	/// many producer, many consumer, thread-safe, runtime configurable buffer size
...	...	@@ -77,24 +75,20 @@ pub fn Channel(comptime T: type) type {
77	75	/// must be called when all calls to put and get have suspended and no more calls occur
78	76	pub fn destroy(self: *SelfChannel) void {
79	77	while (self.getters.get()) \|get_node\| {
80		cancel get_node.data.tick_node.data;
	78	resume get_node.data.tick_node.data;
81	79	}
82	80	while (self.putters.get()) \|put_node\| {
83		cancel put_node.data.tick_node.data;
	81	resume put_node.data.tick_node.data;
84	82	}
85	83	self.loop.allocator.free(self.buffer_nodes);
86	84	self.loop.allocator.destroy(self);
87	85	}
88	86
89		/// puts a data item in the channel. The promise completes when the value has been added to the
	87	/// puts a data item in the channel. The function returns when the value has been added to the
90	88	/// buffer, or in the case of a zero size buffer, when the item has been retrieved by a getter.
91		pub async fn put(self: *SelfChannel, data: T) void {
92		// TODO fix this workaround
93		suspend {
94		resume @handle();
95		}
96
97		var my_tick_node = Loop.NextTickNode.init(@handle());
	89	/// Or when the channel is destroyed.
	90	pub fn put(self: *SelfChannel, data: T) void {
	91	var my_tick_node = Loop.NextTickNode.init(@frame());
98	92	var queue_node = std.atomic.Queue(PutNode).Node.init(PutNode{
99	93	.tick_node = &my_tick_node,
100	94	.data = data,
...	...	@@ -102,35 +96,29 @@ pub fn Channel(comptime T: type) type {
102	96
103	97	// TODO test canceling a put()
104	98	errdefer {
105		_ = @atomicRmw(usize, &self.put_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	99	_ = @atomicRmw(usize, &self.put_count, .Sub, 1, .SeqCst);
106	100	const need_dispatch = !self.putters.remove(&queue_node);
107	101	self.loop.cancelOnNextTick(&my_tick_node);
108	102	if (need_dispatch) {
109	103	// oops we made the put_count incorrect for a period of time. fix by dispatching.
110		_ = @atomicRmw(usize, &self.put_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	104	_ = @atomicRmw(usize, &self.put_count, .Add, 1, .SeqCst);
111	105	self.dispatch();
112	106	}
113	107	}
114	108	suspend {
115	109	self.putters.put(&queue_node);
116		_ = @atomicRmw(usize, &self.put_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	110	_ = @atomicRmw(usize, &self.put_count, .Add, 1, .SeqCst);
117	111
118	112	self.dispatch();
119	113	}
120	114	}
121	115
122		/// await this function to get an item from the channel. If the buffer is empty, the promise will
	116	/// await this function to get an item from the channel. If the buffer is empty, the frame will
123	117	/// complete when the next item is put in the channel.
124	118	pub async fn get(self: *SelfChannel) T {
125		// TODO fix this workaround
126		suspend {
127		resume @handle();
128		}
129
130		// TODO integrate this function with named return values
131		// so we can get rid of this extra result copy
	119	// TODO https://github.com/ziglang/zig/issues/2765
132	120	var result: T = undefined;
133		var my_tick_node = Loop.NextTickNode.init(@handle());
	121	var my_tick_node = Loop.NextTickNode.init(@frame());
134	122	var queue_node = std.atomic.Queue(GetNode).Node.init(GetNode{
135	123	.tick_node = &my_tick_node,
136	124	.data = GetNode.Data{
...	...	@@ -140,19 +128,19 @@ pub fn Channel(comptime T: type) type {
140	128
141	129	// TODO test canceling a get()
142	130	errdefer {
143		_ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	131	_ = @atomicRmw(usize, &self.get_count, .Sub, 1, .SeqCst);
144	132	const need_dispatch = !self.getters.remove(&queue_node);
145	133	self.loop.cancelOnNextTick(&my_tick_node);
146	134	if (need_dispatch) {
147	135	// oops we made the get_count incorrect for a period of time. fix by dispatching.
148		_ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	136	_ = @atomicRmw(usize, &self.get_count, .Add, 1, .SeqCst);
149	137	self.dispatch();
150	138	}
151	139	}
152	140
153	141	suspend {
154	142	self.getters.put(&queue_node);
155		_ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	143	_ = @atomicRmw(usize, &self.get_count, .Add, 1, .SeqCst);
156	144
157	145	self.dispatch();
158	146	}
...	...	@@ -173,15 +161,10 @@ pub fn Channel(comptime T: type) type {
173	161	/// Await is necessary for locking purposes. The function will be resumed after checking the channel
174	162	/// for data and will not wait for data to be available.
175	163	pub async fn getOrNull(self: *SelfChannel) ?T {
176		// TODO fix this workaround
177		suspend {
178		resume @handle();
179		}
180
181	164	// TODO integrate this function with named return values
182	165	// so we can get rid of this extra result copy
183	166	var result: ?T = null;
184		var my_tick_node = Loop.NextTickNode.init(@handle());
	167	var my_tick_node = Loop.NextTickNode.init(@frame());
185	168	var or_null_node = std.atomic.Queue(*std.atomic.Queue(GetNode).Node).Node.init(undefined);
186	169	var queue_node = std.atomic.Queue(GetNode).Node.init(GetNode{
187	170	.tick_node = &my_tick_node,
...	...	@@ -197,19 +180,19 @@ pub fn Channel(comptime T: type) type {
197	180	// TODO test canceling getOrNull
198	181	errdefer {
199	182	_ = self.or_null_queue.remove(&or_null_node);
200		_ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	183	_ = @atomicRmw(usize, &self.get_count, .Sub, 1, .SeqCst);
201	184	const need_dispatch = !self.getters.remove(&queue_node);
202	185	self.loop.cancelOnNextTick(&my_tick_node);
203	186	if (need_dispatch) {
204	187	// oops we made the get_count incorrect for a period of time. fix by dispatching.
205		_ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	188	_ = @atomicRmw(usize, &self.get_count, .Add, 1, .SeqCst);
206	189	self.dispatch();
207	190	}
208	191	}
209	192
210	193	suspend {
211	194	self.getters.put(&queue_node);
212		_ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	195	_ = @atomicRmw(usize, &self.get_count, .Add, 1, .SeqCst);
213	196	self.or_null_queue.put(&or_null_node);
214	197
215	198	self.dispatch();
...	...	@@ -219,21 +202,21 @@ pub fn Channel(comptime T: type) type {
219	202
220	203	fn dispatch(self: *SelfChannel) void {
221	204	// set the "need dispatch" flag
222		_ = @atomicRmw(u8, &self.need_dispatch, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
	205	_ = @atomicRmw(u8, &self.need_dispatch, .Xchg, 1, .SeqCst);
223	206
224	207	lock: while (true) {
225	208	// set the lock flag
226		const prev_lock = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
	209	const prev_lock = @atomicRmw(u8, &self.dispatch_lock, .Xchg, 1, .SeqCst);
227	210	if (prev_lock != 0) return;
228	211
229	212	// clear the need_dispatch flag since we're about to do it
230		_ = @atomicRmw(u8, &self.need_dispatch, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	213	_ = @atomicRmw(u8, &self.need_dispatch, .Xchg, 0, .SeqCst);
231	214
232	215	while (true) {
233	216	one_dispatch: {
234	217	// later we correct these extra subtractions
235		var get_count = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
236		var put_count = @atomicRmw(usize, &self.put_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	218	var get_count = @atomicRmw(usize, &self.get_count, .Sub, 1, .SeqCst);
	219	var put_count = @atomicRmw(usize, &self.put_count, .Sub, 1, .SeqCst);
237	220
238	221	// transfer self.buffer to self.getters
239	222	while (self.buffer_len != 0) {
...	...	@@ -252,7 +235,7 @@ pub fn Channel(comptime T: type) type {
252	235	self.loop.onNextTick(get_node.tick_node);
253	236	self.buffer_len -= 1;
254	237
255		get_count = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	238	get_count = @atomicRmw(usize, &self.get_count, .Sub, 1, .SeqCst);
256	239	}
257	240
258	241	// direct transfer self.putters to self.getters
...	...	@@ -272,8 +255,8 @@ pub fn Channel(comptime T: type) type {
272	255	self.loop.onNextTick(get_node.tick_node);
273	256	self.loop.onNextTick(put_node.tick_node);
274	257
275		get_count = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
276		put_count = @atomicRmw(usize, &self.put_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	258	get_count = @atomicRmw(usize, &self.get_count, .Sub, 1, .SeqCst);
	259	put_count = @atomicRmw(usize, &self.put_count, .Sub, 1, .SeqCst);
277	260	}
278	261
279	262	// transfer self.putters to self.buffer
...	...	@@ -285,13 +268,13 @@ pub fn Channel(comptime T: type) type {
285	268	self.buffer_index +%= 1;
286	269	self.buffer_len += 1;
287	270
288		put_count = @atomicRmw(usize, &self.put_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	271	put_count = @atomicRmw(usize, &self.put_count, .Sub, 1, .SeqCst);
289	272	}
290	273	}
291	274
292	275	// undo the extra subtractions
293		_ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
294		_ = @atomicRmw(usize, &self.put_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	276	_ = @atomicRmw(usize, &self.get_count, .Add, 1, .SeqCst);
	277	_ = @atomicRmw(usize, &self.put_count, .Add, 1, .SeqCst);
295	278
296	279	// All the "get or null" functions should resume now.
297	280	var remove_count: usize = 0;
...	...	@@ -300,18 +283,18 @@ pub fn Channel(comptime T: type) type {
300	283	self.loop.onNextTick(or_null_node.data.data.tick_node);
301	284	}
302	285	if (remove_count != 0) {
303		_ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Sub, remove_count, AtomicOrder.SeqCst);
	286	_ = @atomicRmw(usize, &self.get_count, .Sub, remove_count, .SeqCst);
304	287	}
305	288
306	289	// clear need-dispatch flag
307		const need_dispatch = @atomicRmw(u8, &self.need_dispatch, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	290	const need_dispatch = @atomicRmw(u8, &self.need_dispatch, .Xchg, 0, .SeqCst);
308	291	if (need_dispatch != 0) continue;
309	292
310		const my_lock = @atomicRmw(u8, &self.dispatch_lock, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	293	const my_lock = @atomicRmw(u8, &self.dispatch_lock, .Xchg, 0, .SeqCst);
311	294	assert(my_lock != 0);
312	295
313	296	// we have to check again now that we unlocked
314		if (@atomicLoad(u8, &self.need_dispatch, AtomicOrder.SeqCst) != 0) continue :lock;
	297	if (@atomicLoad(u8, &self.need_dispatch, .SeqCst) != 0) continue :lock;
315	298
316	299	return;
317	300	}
...	...	@@ -324,51 +307,41 @@ test "std.event.Channel" {
324	307	// https://github.com/ziglang/zig/issues/1908
325	308	if (builtin.single_threaded) return error.SkipZigTest;
326	309
327		const allocator = std.heap.direct_allocator;
328
329	310	var loop: Loop = undefined;
330	311	// TODO make a multi threaded test
331		try loop.initSingleThreaded(allocator);
	312	try loop.initSingleThreaded(std.heap.direct_allocator);
332	313	defer loop.deinit();
333	314
334	315	const channel = try Channel(i32).create(&loop, 0);
335	316	defer channel.destroy();
336	317
337		const handle = try async<allocator> testChannelGetter(&loop, channel);
338		defer cancel handle;
339
340		const putter = try async<allocator> testChannelPutter(channel);
341		defer cancel putter;
	318	const handle = async testChannelGetter(&loop, channel);
	319	const putter = async testChannelPutter(channel);
342	320
343	321	loop.run();
344	322	}
345	323
346	324	async fn testChannelGetter(loop: Loop, channel: Channel(i32)) void {
347		errdefer @panic("test failed");
348
349		const value1_promise = try async channel.get();
350		const value1 = await value1_promise;
	325	const value1 = channel.get();
351	326	testing.expect(value1 == 1234);
352	327
353		const value2_promise = try async channel.get();
354		const value2 = await value2_promise;
	328	const value2 = channel.get();
355	329	testing.expect(value2 == 4567);
356	330
357		const value3_promise = try async channel.getOrNull();
358		const value3 = await value3_promise;
	331	const value3 = channel.getOrNull();
359	332	testing.expect(value3 == null);
360	333
361		const last_put = try async testPut(channel, 4444);
362		const value4 = await try async channel.getOrNull();
	334	const last_put = async testPut(channel, 4444);
	335	const value4 = channel.getOrNull();
363	336	testing.expect(value4.? == 4444);
364	337	await last_put;
365	338	}
366	339
367	340	async fn testChannelPutter(channel: *Channel(i32)) void {
368		await (async channel.put(1234) catch @panic("out of memory"));
369		await (async channel.put(4567) catch @panic("out of memory"));
	341	channel.put(1234);
	342	channel.put(4567);
370	343	}
371	344
372	345	async fn testPut(channel: *Channel(i32), value: i32) void {
373		await (async channel.put(value) catch @panic("out of memory"));
	346	channel.put(value);
374	347	}

std/event/fs.zig+640-694

...	...	@@ -76,17 +76,13 @@ pub const Request = struct {
76	76
77	77	pub const PWriteVError = error{OutOfMemory} \|\| File.WriteError;
78	78
79		/// data - just the inner references - must live until pwritev promise completes.
	79	/// data - just the inner references - must live until pwritev frame completes.
80	80	pub async fn pwritev(loop: *Loop, fd: fd_t, data: []const []const u8, offset: usize) PWriteVError!void {
81		// workaround for https://github.com/ziglang/zig/issues/1194
82		suspend {
83		resume @handle();
84		}
85	81	switch (builtin.os) {
86		builtin.Os.macosx,
87		builtin.Os.linux,
88		builtin.Os.freebsd,
89		builtin.Os.netbsd,
	82	.macosx,
	83	.linux,
	84	.freebsd,
	85	.netbsd,
90	86	=> {
91	87	const iovecs = try loop.allocator.alloc(os.iovec_const, data.len);
92	88	defer loop.allocator.free(iovecs);
...	...	@@ -100,7 +96,7 @@ pub async fn pwritev(loop: *Loop, fd: fd_t, data: []const []const u8, offset: us
100	96
101	97	return await (async pwritevPosix(loop, fd, iovecs, offset) catch unreachable);
102	98	},
103		builtin.Os.windows => {
	99	.windows => {
104	100	const data_copy = try std.mem.dupe(loop.allocator, []const u8, data);
105	101	defer loop.allocator.free(data_copy);
106	102	return await (async pwritevWindows(loop, fd, data, offset) catch unreachable);
...	...	@@ -109,7 +105,7 @@ pub async fn pwritev(loop: *Loop, fd: fd_t, data: []const []const u8, offset: us
109	105	}
110	106	}
111	107
112		/// data must outlive the returned promise
	108	/// data must outlive the returned frame
113	109	pub async fn pwritevWindows(loop: *Loop, fd: fd_t, data: []const []const u8, offset: usize) os.WindowsWriteError!void {
114	110	if (data.len == 0) return;
115	111	if (data.len == 1) return await (async pwriteWindows(loop, fd, data[0], offset) catch unreachable);
...	...	@@ -123,15 +119,10 @@ pub async fn pwritevWindows(loop: *Loop, fd: fd_t, data: []const []const u8, off
123	119	}
124	120
125	121	pub async fn pwriteWindows(loop: *Loop, fd: fd_t, data: []const u8, offset: u64) os.WindowsWriteError!void {
126		// workaround for https://github.com/ziglang/zig/issues/1194
127		suspend {
128		resume @handle();
129		}
130
131	122	var resume_node = Loop.ResumeNode.Basic{
132	123	.base = Loop.ResumeNode{
133	124	.id = Loop.ResumeNode.Id.Basic,
134		.handle = @handle(),
	125	.handle = @frame(),
135	126	.overlapped = windows.OVERLAPPED{
136	127	.Internal = 0,
137	128	.InternalHigh = 0,
...	...	@@ -166,18 +157,13 @@ pub async fn pwriteWindows(loop: *Loop, fd: fd_t, data: []const u8, offset: u64)
166	157	}
167	158	}
168	159
169		/// iovecs must live until pwritev promise completes.
	160	/// iovecs must live until pwritev frame completes.
170	161	pub async fn pwritevPosix(
171	162	loop: *Loop,
172	163	fd: fd_t,
173	164	iovecs: []const os.iovec_const,
174	165	offset: usize,
175	166	) os.WriteError!void {
176		// workaround for https://github.com/ziglang/zig/issues/1194
177		suspend {
178		resume @handle();
179		}
180
181	167	var req_node = RequestNode{
182	168	.prev = null,
183	169	.next = null,
...	...	@@ -194,7 +180,7 @@ pub async fn pwritevPosix(
194	180	.TickNode = Loop.NextTickNode{
195	181	.prev = null,
196	182	.next = null,
197		.data = @handle(),
	183	.data = @frame(),
198	184	},
199	185	},
200	186	},
...	...	@@ -211,19 +197,14 @@ pub async fn pwritevPosix(
211	197
212	198	pub const PReadVError = error{OutOfMemory} \|\| File.ReadError;
213	199
214		/// data - just the inner references - must live until preadv promise completes.
	200	/// data - just the inner references - must live until preadv frame completes.
215	201	pub async fn preadv(loop: *Loop, fd: fd_t, data: []const []u8, offset: usize) PReadVError!usize {
216		// workaround for https://github.com/ziglang/zig/issues/1194
217		suspend {
218		resume @handle();
219		}
220
221	202	assert(data.len != 0);
222	203	switch (builtin.os) {
223		builtin.Os.macosx,
224		builtin.Os.linux,
225		builtin.Os.freebsd,
226		builtin.Os.netbsd,
	204	.macosx,
	205	.linux,
	206	.freebsd,
	207	.netbsd,
227	208	=> {
228	209	const iovecs = try loop.allocator.alloc(os.iovec, data.len);
229	210	defer loop.allocator.free(iovecs);
...	...	@@ -237,7 +218,7 @@ pub async fn preadv(loop: *Loop, fd: fd_t, data: []const []u8, offset: usize) PR
237	218
238	219	return await (async preadvPosix(loop, fd, iovecs, offset) catch unreachable);
239	220	},
240		builtin.Os.windows => {
	221	.windows => {
241	222	const data_copy = try std.mem.dupe(loop.allocator, []u8, data);
242	223	defer loop.allocator.free(data_copy);
243	224	return await (async preadvWindows(loop, fd, data_copy, offset) catch unreachable);
...	...	@@ -246,7 +227,7 @@ pub async fn preadv(loop: *Loop, fd: fd_t, data: []const []u8, offset: usize) PR
246	227	}
247	228	}
248	229
249		/// data must outlive the returned promise
	230	/// data must outlive the returned frame
250	231	pub async fn preadvWindows(loop: *Loop, fd: fd_t, data: []const []u8, offset: u64) !usize {
251	232	assert(data.len != 0);
252	233	if (data.len == 1) return await (async preadWindows(loop, fd, data[0], offset) catch unreachable);
...	...	@@ -272,15 +253,10 @@ pub async fn preadvWindows(loop: *Loop, fd: fd_t, data: []const []u8, offset: u6
272	253	}
273	254
274	255	pub async fn preadWindows(loop: *Loop, fd: fd_t, data: []u8, offset: u64) !usize {
275		// workaround for https://github.com/ziglang/zig/issues/1194
276		suspend {
277		resume @handle();
278		}
279
280	256	var resume_node = Loop.ResumeNode.Basic{
281	257	.base = Loop.ResumeNode{
282	258	.id = Loop.ResumeNode.Id.Basic,
283		.handle = @handle(),
	259	.handle = @frame(),
284	260	.overlapped = windows.OVERLAPPED{
285	261	.Internal = 0,
286	262	.InternalHigh = 0,
...	...	@@ -314,18 +290,13 @@ pub async fn preadWindows(loop: *Loop, fd: fd_t, data: []u8, offset: u64) !usize
314	290	return usize(bytes_transferred);
315	291	}
316	292
317		/// iovecs must live until preadv promise completes
	293	/// iovecs must live until preadv frame completes
318	294	pub async fn preadvPosix(
319	295	loop: *Loop,
320	296	fd: fd_t,
321	297	iovecs: []const os.iovec,
322	298	offset: usize,
323	299	) os.ReadError!usize {
324		// workaround for https://github.com/ziglang/zig/issues/1194
325		suspend {
326		resume @handle();
327		}
328
329	300	var req_node = RequestNode{
330	301	.prev = null,
331	302	.next = null,
...	...	@@ -342,7 +313,7 @@ pub async fn preadvPosix(
342	313	.TickNode = Loop.NextTickNode{
343	314	.prev = null,
344	315	.next = null,
345		.data = @handle(),
	316	.data = @frame(),
346	317	},
347	318	},
348	319	},
...	...	@@ -363,11 +334,6 @@ pub async fn openPosix(
363	334	flags: u32,
364	335	mode: File.Mode,
365	336	) File.OpenError!fd_t {
366		// workaround for https://github.com/ziglang/zig/issues/1194
367		suspend {
368		resume @handle();
369		}
370
371	337	const path_c = try std.os.toPosixPath(path);
372	338
373	339	var req_node = RequestNode{
...	...	@@ -386,7 +352,7 @@ pub async fn openPosix(
386	352	.TickNode = Loop.NextTickNode{
387	353	.prev = null,
388	354	.next = null,
389		.data = @handle(),
	355	.data = @frame(),
390	356	},
391	357	},
392	358	},
...	...	@@ -403,12 +369,12 @@ pub async fn openPosix(
403	369
404	370	pub async fn openRead(loop: *Loop, path: []const u8) File.OpenError!fd_t {
405	371	switch (builtin.os) {
406		builtin.Os.macosx, builtin.Os.linux, builtin.Os.freebsd, builtin.Os.netbsd => {
	372	.macosx, .linux, .freebsd, .netbsd => {
407	373	const flags = os.O_LARGEFILE \| os.O_RDONLY \| os.O_CLOEXEC;
408	374	return await (async openPosix(loop, path, flags, File.default_mode) catch unreachable);
409	375	},
410	376
411		builtin.Os.windows => return windows.CreateFile(
	377	.windows => return windows.CreateFile(
412	378	path,
413	379	windows.GENERIC_READ,
414	380	windows.FILE_SHARE_READ,
...	...	@@ -431,15 +397,15 @@ pub async fn openWrite(loop: *Loop, path: []const u8) File.OpenError!fd_t {
431	397	/// Creates if does not exist. Truncates the file if it exists.
432	398	pub async fn openWriteMode(loop: *Loop, path: []const u8, mode: File.Mode) File.OpenError!fd_t {
433	399	switch (builtin.os) {
434		builtin.Os.macosx,
435		builtin.Os.linux,
436		builtin.Os.freebsd,
437		builtin.Os.netbsd,
	400	.macosx,
	401	.linux,
	402	.freebsd,
	403	.netbsd,
438	404	=> {
439	405	const flags = os.O_LARGEFILE \| os.O_WRONLY \| os.O_CREAT \| os.O_CLOEXEC \| os.O_TRUNC;
440	406	return await (async openPosix(loop, path, flags, File.default_mode) catch unreachable);
441	407	},
442		builtin.Os.windows => return windows.CreateFile(
	408	.windows => return windows.CreateFile(
443	409	path,
444	410	windows.GENERIC_WRITE,
445	411	windows.FILE_SHARE_WRITE \| windows.FILE_SHARE_READ \| windows.FILE_SHARE_DELETE,
...	...	@@ -459,12 +425,12 @@ pub async fn openReadWrite(
459	425	mode: File.Mode,
460	426	) File.OpenError!fd_t {
461	427	switch (builtin.os) {
462		builtin.Os.macosx, builtin.Os.linux, builtin.Os.freebsd, builtin.Os.netbsd => {
	428	.macosx, .linux, .freebsd, .netbsd => {
463	429	const flags = os.O_LARGEFILE \| os.O_RDWR \| os.O_CREAT \| os.O_CLOEXEC;
464	430	return await (async openPosix(loop, path, flags, mode) catch unreachable);
465	431	},
466	432
467		builtin.Os.windows => return windows.CreateFile(
	433	.windows => return windows.CreateFile(
468	434	path,
469	435	windows.GENERIC_WRITE \| windows.GENERIC_READ,
470	436	windows.FILE_SHARE_WRITE \| windows.FILE_SHARE_READ \| windows.FILE_SHARE_DELETE,
...	...	@@ -489,9 +455,9 @@ pub const CloseOperation = struct {
489	455	os_data: OsData,
490	456
491	457	const OsData = switch (builtin.os) {
492		builtin.Os.linux, builtin.Os.macosx, builtin.Os.freebsd, builtin.Os.netbsd => OsDataPosix,
	458	.linux, .macosx, .freebsd, .netbsd => OsDataPosix,
493	459
494		builtin.Os.windows => struct {
	460	.windows => struct {
495	461	handle: ?fd_t,
496	462	},
497	463
...	...	@@ -508,8 +474,8 @@ pub const CloseOperation = struct {
508	474	self.* = CloseOperation{
509	475	.loop = loop,
510	476	.os_data = switch (builtin.os) {
511		builtin.Os.linux, builtin.Os.macosx, builtin.Os.freebsd, builtin.Os.netbsd => initOsDataPosix(self),
512		builtin.Os.windows => OsData{ .handle = null },
	477	.linux, .macosx, .freebsd, .netbsd => initOsDataPosix(self),
	478	.windows => OsData{ .handle = null },
513	479	else => @compileError("Unsupported OS"),
514	480	},
515	481	};
...	...	@@ -535,10 +501,10 @@ pub const CloseOperation = struct {
535	501	/// Defer this after creating.
536	502	pub fn finish(self: *CloseOperation) void {
537	503	switch (builtin.os) {
538		builtin.Os.linux,
539		builtin.Os.macosx,
540		builtin.Os.freebsd,
541		builtin.Os.netbsd,
	504	.linux,
	505	.macosx,
	506	.freebsd,
	507	.netbsd,
542	508	=> {
543	509	if (self.os_data.have_fd) {
544	510	self.loop.posixFsRequest(&self.os_data.close_req_node);
...	...	@@ -546,7 +512,7 @@ pub const CloseOperation = struct {
546	512	self.loop.allocator.destroy(self);
547	513	}
548	514	},
549		builtin.Os.windows => {
	515	.windows => {
550	516	if (self.os_data.handle) \|handle\| {
551	517	os.close(handle);
552	518	}
...	...	@@ -558,15 +524,15 @@ pub const CloseOperation = struct {
558	524
559	525	pub fn setHandle(self: *CloseOperation, handle: fd_t) void {
560	526	switch (builtin.os) {
561		builtin.Os.linux,
562		builtin.Os.macosx,
563		builtin.Os.freebsd,
564		builtin.Os.netbsd,
	527	.linux,
	528	.macosx,
	529	.freebsd,
	530	.netbsd,
565	531	=> {
566	532	self.os_data.close_req_node.data.msg.Close.fd = handle;
567	533	self.os_data.have_fd = true;
568	534	},
569		builtin.Os.windows => {
	535	.windows => {
570	536	self.os_data.handle = handle;
571	537	},
572	538	else => @compileError("Unsupported OS"),
...	...	@@ -576,14 +542,14 @@ pub const CloseOperation = struct {
576	542	/// Undo a `setHandle`.
577	543	pub fn clearHandle(self: *CloseOperation) void {
578	544	switch (builtin.os) {
579		builtin.Os.linux,
580		builtin.Os.macosx,
581		builtin.Os.freebsd,
582		builtin.Os.netbsd,
	545	.linux,
	546	.macosx,
	547	.freebsd,
	548	.netbsd,
583	549	=> {
584	550	self.os_data.have_fd = false;
585	551	},
586		builtin.Os.windows => {
	552	.windows => {
587	553	self.os_data.handle = null;
588	554	},
589	555	else => @compileError("Unsupported OS"),
...	...	@@ -592,15 +558,15 @@ pub const CloseOperation = struct {
592	558
593	559	pub fn getHandle(self: *CloseOperation) fd_t {
594	560	switch (builtin.os) {
595		builtin.Os.linux,
596		builtin.Os.macosx,
597		builtin.Os.freebsd,
598		builtin.Os.netbsd,
	561	.linux,
	562	.macosx,
	563	.freebsd,
	564	.netbsd,
599	565	=> {
600	566	assert(self.os_data.have_fd);
601	567	return self.os_data.close_req_node.data.msg.Close.fd;
602	568	},
603		builtin.Os.windows => {
	569	.windows => {
604	570	return self.os_data.handle.?;
605	571	},
606	572	else => @compileError("Unsupported OS"),
...	...	@@ -617,12 +583,12 @@ pub async fn writeFile(loop: *Loop, path: []const u8, contents: []const u8) !voi
617	583	/// contents must remain alive until writeFile completes.
618	584	pub async fn writeFileMode(loop: *Loop, path: []const u8, contents: []const u8, mode: File.Mode) !void {
619	585	switch (builtin.os) {
620		builtin.Os.linux,
621		builtin.Os.macosx,
622		builtin.Os.freebsd,
623		builtin.Os.netbsd,
	586	.linux,
	587	.macosx,
	588	.freebsd,
	589	.netbsd,
624	590	=> return await (async writeFileModeThread(loop, path, contents, mode) catch unreachable),
625		builtin.Os.windows => return await (async writeFileWindows(loop, path, contents) catch unreachable),
	591	.windows => return await (async writeFileWindows(loop, path, contents) catch unreachable),
626	592	else => @compileError("Unsupported OS"),
627	593	}
628	594	}
...	...	@@ -643,11 +609,6 @@ async fn writeFileWindows(loop: *Loop, path: []const u8, contents: []const u8) !
643	609	}
644	610
645	611	async fn writeFileModeThread(loop: *Loop, path: []const u8, contents: []const u8, mode: File.Mode) !void {
646		// workaround for https://github.com/ziglang/zig/issues/1194
647		suspend {
648		resume @handle();
649		}
650
651	612	const path_with_null = try std.cstr.addNullByte(loop.allocator, path);
652	613	defer loop.allocator.free(path_with_null);
653	614
...	...	@@ -667,7 +628,7 @@ async fn writeFileModeThread(loop: *Loop, path: []const u8, contents: []const u8
667	628	.TickNode = Loop.NextTickNode{
668	629	.prev = null,
669	630	.next = null,
670		.data = @handle(),
	631	.data = @frame(),
671	632	},
672	633	},
673	634	},
...	...	@@ -682,7 +643,7 @@ async fn writeFileModeThread(loop: *Loop, path: []const u8, contents: []const u8
682	643	return req_node.data.msg.WriteFile.result;
683	644	}
684	645
685		/// The promise resumes when the last data has been confirmed written, but before the file handle
	646	/// The frame resumes when the last data has been confirmed written, but before the file handle
686	647	/// is closed.
687	648	/// Caller owns returned memory.
688	649	pub async fn readFile(loop: *Loop, file_path: []const u8, max_size: usize) ![]u8 {
...	...	@@ -715,598 +676,583 @@ pub const WatchEventId = enum {
715	676	Delete,
716	677	};
717	678
718		pub const WatchEventError = error{
719		UserResourceLimitReached,
720		SystemResources,
721		AccessDenied,
722		Unexpected, // TODO remove this possibility
723		};
724
725		pub fn Watch(comptime V: type) type {
726		return struct {
727		channel: *event.Channel(Event.Error!Event),
728		os_data: OsData,
729
730		const OsData = switch (builtin.os) {
731		builtin.Os.macosx, builtin.Os.freebsd, builtin.Os.netbsd => struct {
732		file_table: FileTable,
733		table_lock: event.Lock,
734
735		const FileTable = std.AutoHashMap([]const u8, *Put);
736		const Put = struct {
737		putter: promise,
738		value_ptr: *V,
739		};
740		},
741
742		builtin.Os.linux => LinuxOsData,
743		builtin.Os.windows => WindowsOsData,
744
745		else => @compileError("Unsupported OS"),
746		};
747
748		const WindowsOsData = struct {
749		table_lock: event.Lock,
750		dir_table: DirTable,
751		all_putters: std.atomic.Queue(promise),
752		ref_count: std.atomic.Int(usize),
753
754		const DirTable = std.AutoHashMap([]const u8, *Dir);
755		const FileTable = std.AutoHashMap([]const u16, V);
756
757		const Dir = struct {
758		putter: promise,
759		file_table: FileTable,
760		table_lock: event.Lock,
761		};
762		};
763
764		const LinuxOsData = struct {
765		putter: promise,
766		inotify_fd: i32,
767		wd_table: WdTable,
768		table_lock: event.Lock,
769
770		const WdTable = std.AutoHashMap(i32, Dir);
771		const FileTable = std.AutoHashMap([]const u8, V);
772
773		const Dir = struct {
774		dirname: []const u8,
775		file_table: FileTable,
776		};
777		};
778
779		const FileToHandle = std.AutoHashMap([]const u8, promise);
780
781		const Self = @This();
782
783		pub const Event = struct {
784		id: Id,
785		data: V,
786
787		pub const Id = WatchEventId;
788		pub const Error = WatchEventError;
789		};
790
791		pub fn create(loop: Loop, event_buf_count: usize) !Self {
792		const channel = try event.Channel(Self.Event.Error!Self.Event).create(loop, event_buf_count);
793		errdefer channel.destroy();
794
795		switch (builtin.os) {
796		builtin.Os.linux => {
797		const inotify_fd = try os.inotify_init1(os.linux.IN_NONBLOCK \| os.linux.IN_CLOEXEC);
798		errdefer os.close(inotify_fd);
799
800		var result: *Self = undefined;
801		_ = try async<loop.allocator> linuxEventPutter(inotify_fd, channel, &result);
802		return result;
803		},
804
805		builtin.Os.windows => {
806		const self = try loop.allocator.create(Self);
807		errdefer loop.allocator.destroy(self);
808		self.* = Self{
809		.channel = channel,
810		.os_data = OsData{
811		.table_lock = event.Lock.init(loop),
812		.dir_table = OsData.DirTable.init(loop.allocator),
813		.ref_count = std.atomic.Int(usize).init(1),
814		.all_putters = std.atomic.Queue(promise).init(),
815		},
816		};
817		return self;
818		},
819
820		builtin.Os.macosx, builtin.Os.freebsd, builtin.Os.netbsd => {
821		const self = try loop.allocator.create(Self);
822		errdefer loop.allocator.destroy(self);
823
824		self.* = Self{
825		.channel = channel,
826		.os_data = OsData{
827		.table_lock = event.Lock.init(loop),
828		.file_table = OsData.FileTable.init(loop.allocator),
829		},
830		};
831		return self;
832		},
833		else => @compileError("Unsupported OS"),
834		}
835		}
836
837		/// All addFile calls and removeFile calls must have completed.
838		pub fn destroy(self: *Self) void {
839		switch (builtin.os) {
840		builtin.Os.macosx, builtin.Os.freebsd, builtin.Os.netbsd => {
841		// TODO we need to cancel the coroutines before destroying the lock
842		self.os_data.table_lock.deinit();
843		var it = self.os_data.file_table.iterator();
844		while (it.next()) \|entry\| {
845		cancel entry.value.putter;
846		self.channel.loop.allocator.free(entry.key);
847		}
848		self.channel.destroy();
849		},
850		builtin.Os.linux => cancel self.os_data.putter,
851		builtin.Os.windows => {
852		while (self.os_data.all_putters.get()) \|putter_node\| {
853		cancel putter_node.data;
854		}
855		self.deref();
856		},
857		else => @compileError("Unsupported OS"),
858		}
859		}
860
861		fn ref(self: *Self) void {
862		_ = self.os_data.ref_count.incr();
863		}
864
865		fn deref(self: *Self) void {
866		if (self.os_data.ref_count.decr() == 1) {
867		const allocator = self.channel.loop.allocator;
868		self.os_data.table_lock.deinit();
869		var it = self.os_data.dir_table.iterator();
870		while (it.next()) \|entry\| {
871		allocator.free(entry.key);
872		allocator.destroy(entry.value);
873		}
874		self.os_data.dir_table.deinit();
875		self.channel.destroy();
876		allocator.destroy(self);
877		}
878		}
879
880		pub async fn addFile(self: *Self, file_path: []const u8, value: V) !?V {
881		switch (builtin.os) {
882		builtin.Os.macosx, builtin.Os.freebsd, builtin.Os.netbsd => return await (async addFileKEvent(self, file_path, value) catch unreachable),
883		builtin.Os.linux => return await (async addFileLinux(self, file_path, value) catch unreachable),
884		builtin.Os.windows => return await (async addFileWindows(self, file_path, value) catch unreachable),
885		else => @compileError("Unsupported OS"),
886		}
887		}
888
889		async fn addFileKEvent(self: *Self, file_path: []const u8, value: V) !?V {
890		const resolved_path = try std.fs.path.resolve(self.channel.loop.allocator, [_][]const u8{file_path});
891		var resolved_path_consumed = false;
892		defer if (!resolved_path_consumed) self.channel.loop.allocator.free(resolved_path);
893
894		var close_op = try CloseOperation.start(self.channel.loop);
895		var close_op_consumed = false;
896		defer if (!close_op_consumed) close_op.finish();
897
898		const flags = if (os.darwin.is_the_target) os.O_SYMLINK \| os.O_EVTONLY else 0;
899		const mode = 0;
900		const fd = try await (async openPosix(self.channel.loop, resolved_path, flags, mode) catch unreachable);
901		close_op.setHandle(fd);
902
903		var put_data: *OsData.Put = undefined;
904		const putter = try async self.kqPutEvents(close_op, value, &put_data);
905		close_op_consumed = true;
906		errdefer cancel putter;
907
908		const result = blk: {
909		const held = await (async self.os_data.table_lock.acquire() catch unreachable);
910		defer held.release();
911
912		const gop = try self.os_data.file_table.getOrPut(resolved_path);
913		if (gop.found_existing) {
914		const prev_value = gop.kv.value.value_ptr.*;
915		cancel gop.kv.value.putter;
916		gop.kv.value = put_data;
917		break :blk prev_value;
918		} else {
919		resolved_path_consumed = true;
920		gop.kv.value = put_data;
921		break :blk null;
922		}
923		};
924
925		return result;
926		}
927
928		async fn kqPutEvents(self: Self, close_op: CloseOperation, value: V, out_put: **OsData.Put) void {
929		// TODO https://github.com/ziglang/zig/issues/1194
930		suspend {
931		resume @handle();
932		}
933
934		var value_copy = value;
935		var put = OsData.Put{
936		.putter = @handle(),
937		.value_ptr = &value_copy,
938		};
939		out_put.* = &put;
940		self.channel.loop.beginOneEvent();
941
942		defer {
943		close_op.finish();
944		self.channel.loop.finishOneEvent();
945		}
946
947		while (true) {
948		if (await (async self.channel.loop.bsdWaitKev(
949		@intCast(usize, close_op.getHandle()),
950		os.EVFILT_VNODE,
951		os.NOTE_WRITE \| os.NOTE_DELETE,
952		) catch unreachable)) \|kev\| {
953		// TODO handle EV_ERROR
954		if (kev.fflags & os.NOTE_DELETE != 0) {
955		await (async self.channel.put(Self.Event{
956		.id = Event.Id.Delete,
957		.data = value_copy,
958		}) catch unreachable);
959		} else if (kev.fflags & os.NOTE_WRITE != 0) {
960		await (async self.channel.put(Self.Event{
961		.id = Event.Id.CloseWrite,
962		.data = value_copy,
963		}) catch unreachable);
964		}
965		} else \|err\| switch (err) {
966		error.EventNotFound => unreachable,
967		error.ProcessNotFound => unreachable,
968		error.Overflow => unreachable,
969		error.AccessDenied, error.SystemResources => \|casted_err\| {
970		await (async self.channel.put(casted_err) catch unreachable);
971		},
972		}
973		}
974		}
975
976		async fn addFileLinux(self: *Self, file_path: []const u8, value: V) !?V {
977		const value_copy = value;
978
979		const dirname = std.fs.path.dirname(file_path) orelse ".";
980		const dirname_with_null = try std.cstr.addNullByte(self.channel.loop.allocator, dirname);
981		var dirname_with_null_consumed = false;
982		defer if (!dirname_with_null_consumed) self.channel.loop.allocator.free(dirname_with_null);
983
984		const basename = std.fs.path.basename(file_path);
985		const basename_with_null = try std.cstr.addNullByte(self.channel.loop.allocator, basename);
986		var basename_with_null_consumed = false;
987		defer if (!basename_with_null_consumed) self.channel.loop.allocator.free(basename_with_null);
988
989		const wd = try os.inotify_add_watchC(
990		self.os_data.inotify_fd,
991		dirname_with_null.ptr,
992		os.linux.IN_CLOSE_WRITE \| os.linux.IN_ONLYDIR \| os.linux.IN_EXCL_UNLINK,
993		);
994		// wd is either a newly created watch or an existing one.
995
996		const held = await (async self.os_data.table_lock.acquire() catch unreachable);
997		defer held.release();
998
999		const gop = try self.os_data.wd_table.getOrPut(wd);
1000		if (!gop.found_existing) {
1001		gop.kv.value = OsData.Dir{
1002		.dirname = dirname_with_null,
1003		.file_table = OsData.FileTable.init(self.channel.loop.allocator),
1004		};
1005		dirname_with_null_consumed = true;
1006		}
1007		const dir = &gop.kv.value;
1008
1009		const file_table_gop = try dir.file_table.getOrPut(basename_with_null);
1010		if (file_table_gop.found_existing) {
1011		const prev_value = file_table_gop.kv.value;
1012		file_table_gop.kv.value = value_copy;
1013		return prev_value;
1014		} else {
1015		file_table_gop.kv.value = value_copy;
1016		basename_with_null_consumed = true;
1017		return null;
1018		}
1019		}
1020
1021		async fn addFileWindows(self: *Self, file_path: []const u8, value: V) !?V {
1022		const value_copy = value;
1023		// TODO we might need to convert dirname and basename to canonical file paths ("short"?)
1024
1025		const dirname = try std.mem.dupe(self.channel.loop.allocator, u8, std.fs.path.dirname(file_path) orelse ".");
1026		var dirname_consumed = false;
1027		defer if (!dirname_consumed) self.channel.loop.allocator.free(dirname);
1028
1029		const dirname_utf16le = try std.unicode.utf8ToUtf16LeWithNull(self.channel.loop.allocator, dirname);
1030		defer self.channel.loop.allocator.free(dirname_utf16le);
1031
1032		// TODO https://github.com/ziglang/zig/issues/265
1033		const basename = std.fs.path.basename(file_path);
1034		const basename_utf16le_null = try std.unicode.utf8ToUtf16LeWithNull(self.channel.loop.allocator, basename);
1035		var basename_utf16le_null_consumed = false;
1036		defer if (!basename_utf16le_null_consumed) self.channel.loop.allocator.free(basename_utf16le_null);
1037		const basename_utf16le_no_null = basename_utf16le_null[0 .. basename_utf16le_null.len - 1];
1038
1039		const dir_handle = try windows.CreateFileW(
1040		dirname_utf16le.ptr,
1041		windows.FILE_LIST_DIRECTORY,
1042		windows.FILE_SHARE_READ \| windows.FILE_SHARE_DELETE \| windows.FILE_SHARE_WRITE,
1043		null,
1044		windows.OPEN_EXISTING,
1045		windows.FILE_FLAG_BACKUP_SEMANTICS \| windows.FILE_FLAG_OVERLAPPED,
1046		null,
1047		);
1048		var dir_handle_consumed = false;
1049		defer if (!dir_handle_consumed) windows.CloseHandle(dir_handle);
1050
1051		const held = await (async self.os_data.table_lock.acquire() catch unreachable);
1052		defer held.release();
1053
1054		const gop = try self.os_data.dir_table.getOrPut(dirname);
1055		if (gop.found_existing) {
1056		const dir = gop.kv.value;
1057		const held_dir_lock = await (async dir.table_lock.acquire() catch unreachable);
1058		defer held_dir_lock.release();
1059
1060		const file_gop = try dir.file_table.getOrPut(basename_utf16le_no_null);
1061		if (file_gop.found_existing) {
1062		const prev_value = file_gop.kv.value;
1063		file_gop.kv.value = value_copy;
1064		return prev_value;
1065		} else {
1066		file_gop.kv.value = value_copy;
1067		basename_utf16le_null_consumed = true;
1068		return null;
1069		}
1070		} else {
1071		errdefer _ = self.os_data.dir_table.remove(dirname);
1072		const dir = try self.channel.loop.allocator.create(OsData.Dir);
1073		errdefer self.channel.loop.allocator.destroy(dir);
1074
1075		dir.* = OsData.Dir{
1076		.file_table = OsData.FileTable.init(self.channel.loop.allocator),
1077		.table_lock = event.Lock.init(self.channel.loop),
1078		.putter = undefined,
1079		};
1080		gop.kv.value = dir;
1081		assert((try dir.file_table.put(basename_utf16le_no_null, value_copy)) == null);
1082		basename_utf16le_null_consumed = true;
1083
1084		dir.putter = try async self.windowsDirReader(dir_handle, dir);
1085		dir_handle_consumed = true;
1086
1087		dirname_consumed = true;
1088
1089		return null;
1090		}
1091		}
1092
1093		async fn windowsDirReader(self: Self, dir_handle: windows.HANDLE, dir: OsData.Dir) void {
1094		// TODO https://github.com/ziglang/zig/issues/1194
1095		suspend {
1096		resume @handle();
1097		}
1098
1099		self.ref();
1100		defer self.deref();
1101
1102		defer os.close(dir_handle);
1103
1104		var putter_node = std.atomic.Queue(promise).Node{
1105		.data = @handle(),
1106		.prev = null,
1107		.next = null,
1108		};
1109		self.os_data.all_putters.put(&putter_node);
1110		defer _ = self.os_data.all_putters.remove(&putter_node);
1111
1112		var resume_node = Loop.ResumeNode.Basic{
1113		.base = Loop.ResumeNode{
1114		.id = Loop.ResumeNode.Id.Basic,
1115		.handle = @handle(),
1116		.overlapped = windows.OVERLAPPED{
1117		.Internal = 0,
1118		.InternalHigh = 0,
1119		.Offset = 0,
1120		.OffsetHigh = 0,
1121		.hEvent = null,
1122		},
1123		},
1124		};
1125		var event_buf: [4096]u8 align(@alignOf(windows.FILE_NOTIFY_INFORMATION)) = undefined;
1126
1127		// TODO handle this error not in the channel but in the setup
1128		_ = windows.CreateIoCompletionPort(
1129		dir_handle,
1130		self.channel.loop.os_data.io_port,
1131		undefined,
1132		undefined,
1133		) catch \|err\| {
1134		await (async self.channel.put(err) catch unreachable);
1135		return;
1136		};
1137
1138		while (true) {
1139		{
1140		// TODO only 1 beginOneEvent for the whole coroutine
1141		self.channel.loop.beginOneEvent();
1142		errdefer self.channel.loop.finishOneEvent();
1143		errdefer {
1144		_ = windows.kernel32.CancelIoEx(dir_handle, &resume_node.base.overlapped);
1145		}
1146		suspend {
1147		_ = windows.kernel32.ReadDirectoryChangesW(
1148		dir_handle,
1149		&event_buf,
1150		@intCast(windows.DWORD, event_buf.len),
1151		windows.FALSE, // watch subtree
1152		windows.FILE_NOTIFY_CHANGE_FILE_NAME \| windows.FILE_NOTIFY_CHANGE_DIR_NAME \|
1153		windows.FILE_NOTIFY_CHANGE_ATTRIBUTES \| windows.FILE_NOTIFY_CHANGE_SIZE \|
1154		windows.FILE_NOTIFY_CHANGE_LAST_WRITE \| windows.FILE_NOTIFY_CHANGE_LAST_ACCESS \|
1155		windows.FILE_NOTIFY_CHANGE_CREATION \| windows.FILE_NOTIFY_CHANGE_SECURITY,
1156		null, // number of bytes transferred (unused for async)
1157		&resume_node.base.overlapped,
1158		null, // completion routine - unused because we use IOCP
1159		);
1160		}
1161		}
1162		var bytes_transferred: windows.DWORD = undefined;
1163		if (windows.kernel32.GetOverlappedResult(dir_handle, &resume_node.base.overlapped, &bytes_transferred, windows.FALSE) == 0) {
1164		const err = switch (windows.kernel32.GetLastError()) {
1165		else => \|err\| windows.unexpectedError(err),
1166		};
1167		await (async self.channel.put(err) catch unreachable);
1168		} else {
1169		// can't use @bytesToSlice because of the special variable length name field
1170		var ptr = event_buf[0..].ptr;
1171		const end_ptr = ptr + bytes_transferred;
1172		var ev: *windows.FILE_NOTIFY_INFORMATION = undefined;
1173		while (@ptrToInt(ptr) < @ptrToInt(end_ptr)) : (ptr += ev.NextEntryOffset) {
1174		ev = @ptrCast(*windows.FILE_NOTIFY_INFORMATION, ptr);
1175		const emit = switch (ev.Action) {
1176		windows.FILE_ACTION_REMOVED => WatchEventId.Delete,
1177		windows.FILE_ACTION_MODIFIED => WatchEventId.CloseWrite,
1178		else => null,
1179		};
1180		if (emit) \|id\| {
1181		const basename_utf16le = ([*]u16)(&ev.FileName)[0 .. ev.FileNameLength / 2];
1182		const user_value = blk: {
1183		const held = await (async dir.table_lock.acquire() catch unreachable);
1184		defer held.release();
1185
1186		if (dir.file_table.get(basename_utf16le)) \|entry\| {
1187		break :blk entry.value;
1188		} else {
1189		break :blk null;
1190		}
1191		};
1192		if (user_value) \|v\| {
1193		await (async self.channel.put(Event{
1194		.id = id,
1195		.data = v,
1196		}) catch unreachable);
1197		}
1198		}
1199		if (ev.NextEntryOffset == 0) break;
1200		}
1201		}
1202		}
1203		}
1204
1205		pub async fn removeFile(self: *Self, file_path: []const u8) ?V {
1206		@panic("TODO");
1207		}
1208
1209		async fn linuxEventPutter(inotify_fd: i32, channel: event.Channel(Event.Error!Event), out_watch: *Self) void {
1210		// TODO https://github.com/ziglang/zig/issues/1194
1211		suspend {
1212		resume @handle();
1213		}
1214
1215		const loop = channel.loop;
1216
1217		var watch = Self{
1218		.channel = channel,
1219		.os_data = OsData{
1220		.putter = @handle(),
1221		.inotify_fd = inotify_fd,
1222		.wd_table = OsData.WdTable.init(loop.allocator),
1223		.table_lock = event.Lock.init(loop),
1224		},
1225		};
1226		out_watch.* = &watch;
1227
1228		loop.beginOneEvent();
1229
1230		defer {
1231		watch.os_data.table_lock.deinit();
1232		var wd_it = watch.os_data.wd_table.iterator();
1233		while (wd_it.next()) \|wd_entry\| {
1234		var file_it = wd_entry.value.file_table.iterator();
1235		while (file_it.next()) \|file_entry\| {
1236		loop.allocator.free(file_entry.key);
1237		}
1238		loop.allocator.free(wd_entry.value.dirname);
1239		}
1240		loop.finishOneEvent();
1241		os.close(inotify_fd);
1242		channel.destroy();
1243		}
1244
1245		var event_buf: [4096]u8 align(@alignOf(os.linux.inotify_event)) = undefined;
1246
1247		while (true) {
1248		const rc = os.linux.read(inotify_fd, &event_buf, event_buf.len);
1249		const errno = os.linux.getErrno(rc);
1250		switch (errno) {
1251		0 => {
1252		// can't use @bytesToSlice because of the special variable length name field
1253		var ptr = event_buf[0..].ptr;
1254		const end_ptr = ptr + event_buf.len;
1255		var ev: *os.linux.inotify_event = undefined;
1256		while (@ptrToInt(ptr) < @ptrToInt(end_ptr)) : (ptr += @sizeOf(os.linux.inotify_event) + ev.len) {
1257		ev = @ptrCast(*os.linux.inotify_event, ptr);
1258		if (ev.mask & os.linux.IN_CLOSE_WRITE == os.linux.IN_CLOSE_WRITE) {
1259		const basename_ptr = ptr + @sizeOf(os.linux.inotify_event);
1260		const basename_with_null = basename_ptr[0 .. std.mem.len(u8, basename_ptr) + 1];
1261		const user_value = blk: {
1262		const held = await (async watch.os_data.table_lock.acquire() catch unreachable);
1263		defer held.release();
1264
1265		const dir = &watch.os_data.wd_table.get(ev.wd).?.value;
1266		if (dir.file_table.get(basename_with_null)) \|entry\| {
1267		break :blk entry.value;
1268		} else {
1269		break :blk null;
1270		}
1271		};
1272		if (user_value) \|v\| {
1273		await (async channel.put(Event{
1274		.id = WatchEventId.CloseWrite,
1275		.data = v,
1276		}) catch unreachable);
1277		}
1278		}
1279		}
1280		},
1281		os.linux.EINTR => continue,
1282		os.linux.EINVAL => unreachable,
1283		os.linux.EFAULT => unreachable,
1284		os.linux.EAGAIN => {
1285		(await (async loop.linuxWaitFd(
1286		inotify_fd,
1287		os.linux.EPOLLET \| os.linux.EPOLLIN,
1288		) catch unreachable)) catch \|err\| {
1289		const transformed_err = switch (err) {
1290		error.FileDescriptorAlreadyPresentInSet => unreachable,
1291		error.OperationCausesCircularLoop => unreachable,
1292		error.FileDescriptorNotRegistered => unreachable,
1293		error.FileDescriptorIncompatibleWithEpoll => unreachable,
1294		error.Unexpected => unreachable,
1295		else => \|e\| e,
1296		};
1297		await (async channel.put(transformed_err) catch unreachable);
1298		};
1299		},
1300		else => unreachable,
1301		}
1302		}
1303		}
1304		};
1305		}
	679	//pub const WatchEventError = error{
	680	// UserResourceLimitReached,
	681	// SystemResources,
	682	// AccessDenied,
	683	// Unexpected, // TODO remove this possibility
	684	//};
	685	//
	686	//pub fn Watch(comptime V: type) type {
	687	// return struct {
	688	// channel: *event.Channel(Event.Error!Event),
	689	// os_data: OsData,
	690	//
	691	// const OsData = switch (builtin.os) {
	692	// .macosx, .freebsd, .netbsd => struct {
	693	// file_table: FileTable,
	694	// table_lock: event.Lock,
	695	//
	696	// const FileTable = std.AutoHashMap([]const u8, *Put);
	697	// const Put = struct {
	698	// putter: anyframe,
	699	// value_ptr: *V,
	700	// };
	701	// },
	702	//
	703	// .linux => LinuxOsData,
	704	// .windows => WindowsOsData,
	705	//
	706	// else => @compileError("Unsupported OS"),
	707	// };
	708	//
	709	// const WindowsOsData = struct {
	710	// table_lock: event.Lock,
	711	// dir_table: DirTable,
	712	// all_putters: std.atomic.Queue(anyframe),
	713	// ref_count: std.atomic.Int(usize),
	714	//
	715	// const DirTable = std.AutoHashMap([]const u8, *Dir);
	716	// const FileTable = std.AutoHashMap([]const u16, V);
	717	//
	718	// const Dir = struct {
	719	// putter: anyframe,
	720	// file_table: FileTable,
	721	// table_lock: event.Lock,
	722	// };
	723	// };
	724	//
	725	// const LinuxOsData = struct {
	726	// putter: anyframe,
	727	// inotify_fd: i32,
	728	// wd_table: WdTable,
	729	// table_lock: event.Lock,
	730	//
	731	// const WdTable = std.AutoHashMap(i32, Dir);
	732	// const FileTable = std.AutoHashMap([]const u8, V);
	733	//
	734	// const Dir = struct {
	735	// dirname: []const u8,
	736	// file_table: FileTable,
	737	// };
	738	// };
	739	//
	740	// const FileToHandle = std.AutoHashMap([]const u8, anyframe);
	741	//
	742	// const Self = @This();
	743	//
	744	// pub const Event = struct {
	745	// id: Id,
	746	// data: V,
	747	//
	748	// pub const Id = WatchEventId;
	749	// pub const Error = WatchEventError;
	750	// };
	751	//
	752	// pub fn create(loop: Loop, event_buf_count: usize) !Self {
	753	// const channel = try event.Channel(Self.Event.Error!Self.Event).create(loop, event_buf_count);
	754	// errdefer channel.destroy();
	755	//
	756	// switch (builtin.os) {
	757	// .linux => {
	758	// const inotify_fd = try os.inotify_init1(os.linux.IN_NONBLOCK \| os.linux.IN_CLOEXEC);
	759	// errdefer os.close(inotify_fd);
	760	//
	761	// var result: *Self = undefined;
	762	// _ = try async<loop.allocator> linuxEventPutter(inotify_fd, channel, &result);
	763	// return result;
	764	// },
	765	//
	766	// .windows => {
	767	// const self = try loop.allocator.create(Self);
	768	// errdefer loop.allocator.destroy(self);
	769	// self.* = Self{
	770	// .channel = channel,
	771	// .os_data = OsData{
	772	// .table_lock = event.Lock.init(loop),
	773	// .dir_table = OsData.DirTable.init(loop.allocator),
	774	// .ref_count = std.atomic.Int(usize).init(1),
	775	// .all_putters = std.atomic.Queue(anyframe).init(),
	776	// },
	777	// };
	778	// return self;
	779	// },
	780	//
	781	// .macosx, .freebsd, .netbsd => {
	782	// const self = try loop.allocator.create(Self);
	783	// errdefer loop.allocator.destroy(self);
	784	//
	785	// self.* = Self{
	786	// .channel = channel,
	787	// .os_data = OsData{
	788	// .table_lock = event.Lock.init(loop),
	789	// .file_table = OsData.FileTable.init(loop.allocator),
	790	// },
	791	// };
	792	// return self;
	793	// },
	794	// else => @compileError("Unsupported OS"),
	795	// }
	796	// }
	797	//
	798	// /// All addFile calls and removeFile calls must have completed.
	799	// pub fn destroy(self: *Self) void {
	800	// switch (builtin.os) {
	801	// .macosx, .freebsd, .netbsd => {
	802	// // TODO we need to cancel the frames before destroying the lock
	803	// self.os_data.table_lock.deinit();
	804	// var it = self.os_data.file_table.iterator();
	805	// while (it.next()) \|entry\| {
	806	// cancel entry.value.putter;
	807	// self.channel.loop.allocator.free(entry.key);
	808	// }
	809	// self.channel.destroy();
	810	// },
	811	// .linux => cancel self.os_data.putter,
	812	// .windows => {
	813	// while (self.os_data.all_putters.get()) \|putter_node\| {
	814	// cancel putter_node.data;
	815	// }
	816	// self.deref();
	817	// },
	818	// else => @compileError("Unsupported OS"),
	819	// }
	820	// }
	821	//
	822	// fn ref(self: *Self) void {
	823	// _ = self.os_data.ref_count.incr();
	824	// }
	825	//
	826	// fn deref(self: *Self) void {
	827	// if (self.os_data.ref_count.decr() == 1) {
	828	// const allocator = self.channel.loop.allocator;
	829	// self.os_data.table_lock.deinit();
	830	// var it = self.os_data.dir_table.iterator();
	831	// while (it.next()) \|entry\| {
	832	// allocator.free(entry.key);
	833	// allocator.destroy(entry.value);
	834	// }
	835	// self.os_data.dir_table.deinit();
	836	// self.channel.destroy();
	837	// allocator.destroy(self);
	838	// }
	839	// }
	840	//
	841	// pub async fn addFile(self: *Self, file_path: []const u8, value: V) !?V {
	842	// switch (builtin.os) {
	843	// .macosx, .freebsd, .netbsd => return await (async addFileKEvent(self, file_path, value) catch unreachable),
	844	// .linux => return await (async addFileLinux(self, file_path, value) catch unreachable),
	845	// .windows => return await (async addFileWindows(self, file_path, value) catch unreachable),
	846	// else => @compileError("Unsupported OS"),
	847	// }
	848	// }
	849	//
	850	// async fn addFileKEvent(self: *Self, file_path: []const u8, value: V) !?V {
	851	// const resolved_path = try std.fs.path.resolve(self.channel.loop.allocator, [_][]const u8{file_path});
	852	// var resolved_path_consumed = false;
	853	// defer if (!resolved_path_consumed) self.channel.loop.allocator.free(resolved_path);
	854	//
	855	// var close_op = try CloseOperation.start(self.channel.loop);
	856	// var close_op_consumed = false;
	857	// defer if (!close_op_consumed) close_op.finish();
	858	//
	859	// const flags = if (os.darwin.is_the_target) os.O_SYMLINK \| os.O_EVTONLY else 0;
	860	// const mode = 0;
	861	// const fd = try await (async openPosix(self.channel.loop, resolved_path, flags, mode) catch unreachable);
	862	// close_op.setHandle(fd);
	863	//
	864	// var put_data: *OsData.Put = undefined;
	865	// const putter = try async self.kqPutEvents(close_op, value, &put_data);
	866	// close_op_consumed = true;
	867	// errdefer cancel putter;
	868	//
	869	// const result = blk: {
	870	// const held = await (async self.os_data.table_lock.acquire() catch unreachable);
	871	// defer held.release();
	872	//
	873	// const gop = try self.os_data.file_table.getOrPut(resolved_path);
	874	// if (gop.found_existing) {
	875	// const prev_value = gop.kv.value.value_ptr.*;
	876	// cancel gop.kv.value.putter;
	877	// gop.kv.value = put_data;
	878	// break :blk prev_value;
	879	// } else {
	880	// resolved_path_consumed = true;
	881	// gop.kv.value = put_data;
	882	// break :blk null;
	883	// }
	884	// };
	885	//
	886	// return result;
	887	// }
	888	//
	889	// async fn kqPutEvents(self: Self, close_op: CloseOperation, value: V, out_put: **OsData.Put) void {
	890	// var value_copy = value;
	891	// var put = OsData.Put{
	892	// .putter = @frame(),
	893	// .value_ptr = &value_copy,
	894	// };
	895	// out_put.* = &put;
	896	// self.channel.loop.beginOneEvent();
	897	//
	898	// defer {
	899	// close_op.finish();
	900	// self.channel.loop.finishOneEvent();
	901	// }
	902	//
	903	// while (true) {
	904	// if (await (async self.channel.loop.bsdWaitKev(
	905	// @intCast(usize, close_op.getHandle()),
	906	// os.EVFILT_VNODE,
	907	// os.NOTE_WRITE \| os.NOTE_DELETE,
	908	// ) catch unreachable)) \|kev\| {
	909	// // TODO handle EV_ERROR
	910	// if (kev.fflags & os.NOTE_DELETE != 0) {
	911	// await (async self.channel.put(Self.Event{
	912	// .id = Event.Id.Delete,
	913	// .data = value_copy,
	914	// }) catch unreachable);
	915	// } else if (kev.fflags & os.NOTE_WRITE != 0) {
	916	// await (async self.channel.put(Self.Event{
	917	// .id = Event.Id.CloseWrite,
	918	// .data = value_copy,
	919	// }) catch unreachable);
	920	// }
	921	// } else \|err\| switch (err) {
	922	// error.EventNotFound => unreachable,
	923	// error.ProcessNotFound => unreachable,
	924	// error.Overflow => unreachable,
	925	// error.AccessDenied, error.SystemResources => \|casted_err\| {
	926	// await (async self.channel.put(casted_err) catch unreachable);
	927	// },
	928	// }
	929	// }
	930	// }
	931	//
	932	// async fn addFileLinux(self: *Self, file_path: []const u8, value: V) !?V {
	933	// const value_copy = value;
	934	//
	935	// const dirname = std.fs.path.dirname(file_path) orelse ".";
	936	// const dirname_with_null = try std.cstr.addNullByte(self.channel.loop.allocator, dirname);
	937	// var dirname_with_null_consumed = false;
	938	// defer if (!dirname_with_null_consumed) self.channel.loop.allocator.free(dirname_with_null);
	939	//
	940	// const basename = std.fs.path.basename(file_path);
	941	// const basename_with_null = try std.cstr.addNullByte(self.channel.loop.allocator, basename);
	942	// var basename_with_null_consumed = false;
	943	// defer if (!basename_with_null_consumed) self.channel.loop.allocator.free(basename_with_null);
	944	//
	945	// const wd = try os.inotify_add_watchC(
	946	// self.os_data.inotify_fd,
	947	// dirname_with_null.ptr,
	948	// os.linux.IN_CLOSE_WRITE \| os.linux.IN_ONLYDIR \| os.linux.IN_EXCL_UNLINK,
	949	// );
	950	// // wd is either a newly created watch or an existing one.
	951	//
	952	// const held = await (async self.os_data.table_lock.acquire() catch unreachable);
	953	// defer held.release();
	954	//
	955	// const gop = try self.os_data.wd_table.getOrPut(wd);
	956	// if (!gop.found_existing) {
	957	// gop.kv.value = OsData.Dir{
	958	// .dirname = dirname_with_null,
	959	// .file_table = OsData.FileTable.init(self.channel.loop.allocator),
	960	// };
	961	// dirname_with_null_consumed = true;
	962	// }
	963	// const dir = &gop.kv.value;
	964	//
	965	// const file_table_gop = try dir.file_table.getOrPut(basename_with_null);
	966	// if (file_table_gop.found_existing) {
	967	// const prev_value = file_table_gop.kv.value;
	968	// file_table_gop.kv.value = value_copy;
	969	// return prev_value;
	970	// } else {
	971	// file_table_gop.kv.value = value_copy;
	972	// basename_with_null_consumed = true;
	973	// return null;
	974	// }
	975	// }
	976	//
	977	// async fn addFileWindows(self: *Self, file_path: []const u8, value: V) !?V {
	978	// const value_copy = value;
	979	// // TODO we might need to convert dirname and basename to canonical file paths ("short"?)
	980	//
	981	// const dirname = try std.mem.dupe(self.channel.loop.allocator, u8, std.fs.path.dirname(file_path) orelse ".");
	982	// var dirname_consumed = false;
	983	// defer if (!dirname_consumed) self.channel.loop.allocator.free(dirname);
	984	//
	985	// const dirname_utf16le = try std.unicode.utf8ToUtf16LeWithNull(self.channel.loop.allocator, dirname);
	986	// defer self.channel.loop.allocator.free(dirname_utf16le);
	987	//
	988	// // TODO https://github.com/ziglang/zig/issues/265
	989	// const basename = std.fs.path.basename(file_path);
	990	// const basename_utf16le_null = try std.unicode.utf8ToUtf16LeWithNull(self.channel.loop.allocator, basename);
	991	// var basename_utf16le_null_consumed = false;
	992	// defer if (!basename_utf16le_null_consumed) self.channel.loop.allocator.free(basename_utf16le_null);
	993	// const basename_utf16le_no_null = basename_utf16le_null[0 .. basename_utf16le_null.len - 1];
	994	//
	995	// const dir_handle = try windows.CreateFileW(
	996	// dirname_utf16le.ptr,
	997	// windows.FILE_LIST_DIRECTORY,
	998	// windows.FILE_SHARE_READ \| windows.FILE_SHARE_DELETE \| windows.FILE_SHARE_WRITE,
	999	// null,
	1000	// windows.OPEN_EXISTING,
	1001	// windows.FILE_FLAG_BACKUP_SEMANTICS \| windows.FILE_FLAG_OVERLAPPED,
	1002	// null,
	1003	// );
	1004	// var dir_handle_consumed = false;
	1005	// defer if (!dir_handle_consumed) windows.CloseHandle(dir_handle);
	1006	//
	1007	// const held = await (async self.os_data.table_lock.acquire() catch unreachable);
	1008	// defer held.release();
	1009	//
	1010	// const gop = try self.os_data.dir_table.getOrPut(dirname);
	1011	// if (gop.found_existing) {
	1012	// const dir = gop.kv.value;
	1013	// const held_dir_lock = await (async dir.table_lock.acquire() catch unreachable);
	1014	// defer held_dir_lock.release();
	1015	//
	1016	// const file_gop = try dir.file_table.getOrPut(basename_utf16le_no_null);
	1017	// if (file_gop.found_existing) {
	1018	// const prev_value = file_gop.kv.value;
	1019	// file_gop.kv.value = value_copy;
	1020	// return prev_value;
	1021	// } else {
	1022	// file_gop.kv.value = value_copy;
	1023	// basename_utf16le_null_consumed = true;
	1024	// return null;
	1025	// }
	1026	// } else {
	1027	// errdefer _ = self.os_data.dir_table.remove(dirname);
	1028	// const dir = try self.channel.loop.allocator.create(OsData.Dir);
	1029	// errdefer self.channel.loop.allocator.destroy(dir);
	1030	//
	1031	// dir.* = OsData.Dir{
	1032	// .file_table = OsData.FileTable.init(self.channel.loop.allocator),
	1033	// .table_lock = event.Lock.init(self.channel.loop),
	1034	// .putter = undefined,
	1035	// };
	1036	// gop.kv.value = dir;
	1037	// assert((try dir.file_table.put(basename_utf16le_no_null, value_copy)) == null);
	1038	// basename_utf16le_null_consumed = true;
	1039	//
	1040	// dir.putter = try async self.windowsDirReader(dir_handle, dir);
	1041	// dir_handle_consumed = true;
	1042	//
	1043	// dirname_consumed = true;
	1044	//
	1045	// return null;
	1046	// }
	1047	// }
	1048	//
	1049	// async fn windowsDirReader(self: Self, dir_handle: windows.HANDLE, dir: OsData.Dir) void {
	1050	// self.ref();
	1051	// defer self.deref();
	1052	//
	1053	// defer os.close(dir_handle);
	1054	//
	1055	// var putter_node = std.atomic.Queue(anyframe).Node{
	1056	// .data = @frame(),
	1057	// .prev = null,
	1058	// .next = null,
	1059	// };
	1060	// self.os_data.all_putters.put(&putter_node);
	1061	// defer _ = self.os_data.all_putters.remove(&putter_node);
	1062	//
	1063	// var resume_node = Loop.ResumeNode.Basic{
	1064	// .base = Loop.ResumeNode{
	1065	// .id = Loop.ResumeNode.Id.Basic,
	1066	// .handle = @frame(),
	1067	// .overlapped = windows.OVERLAPPED{
	1068	// .Internal = 0,
	1069	// .InternalHigh = 0,
	1070	// .Offset = 0,
	1071	// .OffsetHigh = 0,
	1072	// .hEvent = null,
	1073	// },
	1074	// },
	1075	// };
	1076	// var event_buf: [4096]u8 align(@alignOf(windows.FILE_NOTIFY_INFORMATION)) = undefined;
	1077	//
	1078	// // TODO handle this error not in the channel but in the setup
	1079	// _ = windows.CreateIoCompletionPort(
	1080	// dir_handle,
	1081	// self.channel.loop.os_data.io_port,
	1082	// undefined,
	1083	// undefined,
	1084	// ) catch \|err\| {
	1085	// await (async self.channel.put(err) catch unreachable);
	1086	// return;
	1087	// };
	1088	//
	1089	// while (true) {
	1090	// {
	1091	// // TODO only 1 beginOneEvent for the whole function
	1092	// self.channel.loop.beginOneEvent();
	1093	// errdefer self.channel.loop.finishOneEvent();
	1094	// errdefer {
	1095	// _ = windows.kernel32.CancelIoEx(dir_handle, &resume_node.base.overlapped);
	1096	// }
	1097	// suspend {
	1098	// _ = windows.kernel32.ReadDirectoryChangesW(
	1099	// dir_handle,
	1100	// &event_buf,
	1101	// @intCast(windows.DWORD, event_buf.len),
	1102	// windows.FALSE, // watch subtree
	1103	// windows.FILE_NOTIFY_CHANGE_FILE_NAME \| windows.FILE_NOTIFY_CHANGE_DIR_NAME \|
	1104	// windows.FILE_NOTIFY_CHANGE_ATTRIBUTES \| windows.FILE_NOTIFY_CHANGE_SIZE \|
	1105	// windows.FILE_NOTIFY_CHANGE_LAST_WRITE \| windows.FILE_NOTIFY_CHANGE_LAST_ACCESS \|
	1106	// windows.FILE_NOTIFY_CHANGE_CREATION \| windows.FILE_NOTIFY_CHANGE_SECURITY,
	1107	// null, // number of bytes transferred (unused for async)
	1108	// &resume_node.base.overlapped,
	1109	// null, // completion routine - unused because we use IOCP
	1110	// );
	1111	// }
	1112	// }
	1113	// var bytes_transferred: windows.DWORD = undefined;
	1114	// if (windows.kernel32.GetOverlappedResult(dir_handle, &resume_node.base.overlapped, &bytes_transferred, windows.FALSE) == 0) {
	1115	// const err = switch (windows.kernel32.GetLastError()) {
	1116	// else => \|err\| windows.unexpectedError(err),
	1117	// };
	1118	// await (async self.channel.put(err) catch unreachable);
	1119	// } else {
	1120	// // can't use @bytesToSlice because of the special variable length name field
	1121	// var ptr = event_buf[0..].ptr;
	1122	// const end_ptr = ptr + bytes_transferred;
	1123	// var ev: *windows.FILE_NOTIFY_INFORMATION = undefined;
	1124	// while (@ptrToInt(ptr) < @ptrToInt(end_ptr)) : (ptr += ev.NextEntryOffset) {
	1125	// ev = @ptrCast(*windows.FILE_NOTIFY_INFORMATION, ptr);
	1126	// const emit = switch (ev.Action) {
	1127	// windows.FILE_ACTION_REMOVED => WatchEventId.Delete,
	1128	// windows.FILE_ACTION_MODIFIED => WatchEventId.CloseWrite,
	1129	// else => null,
	1130	// };
	1131	// if (emit) \|id\| {
	1132	// const basename_utf16le = ([*]u16)(&ev.FileName)[0 .. ev.FileNameLength / 2];
	1133	// const user_value = blk: {
	1134	// const held = await (async dir.table_lock.acquire() catch unreachable);
	1135	// defer held.release();
	1136	//
	1137	// if (dir.file_table.get(basename_utf16le)) \|entry\| {
	1138	// break :blk entry.value;
	1139	// } else {
	1140	// break :blk null;
	1141	// }
	1142	// };
	1143	// if (user_value) \|v\| {
	1144	// await (async self.channel.put(Event{
	1145	// .id = id,
	1146	// .data = v,
	1147	// }) catch unreachable);
	1148	// }
	1149	// }
	1150	// if (ev.NextEntryOffset == 0) break;
	1151	// }
	1152	// }
	1153	// }
	1154	// }
	1155	//
	1156	// pub async fn removeFile(self: *Self, file_path: []const u8) ?V {
	1157	// @panic("TODO");
	1158	// }
	1159	//
	1160	// async fn linuxEventPutter(inotify_fd: i32, channel: event.Channel(Event.Error!Event), out_watch: *Self) void {
	1161	// const loop = channel.loop;
	1162	//
	1163	// var watch = Self{
	1164	// .channel = channel,
	1165	// .os_data = OsData{
	1166	// .putter = @frame(),
	1167	// .inotify_fd = inotify_fd,
	1168	// .wd_table = OsData.WdTable.init(loop.allocator),
	1169	// .table_lock = event.Lock.init(loop),
	1170	// },
	1171	// };
	1172	// out_watch.* = &watch;
	1173	//
	1174	// loop.beginOneEvent();
	1175	//
	1176	// defer {
	1177	// watch.os_data.table_lock.deinit();
	1178	// var wd_it = watch.os_data.wd_table.iterator();
	1179	// while (wd_it.next()) \|wd_entry\| {
	1180	// var file_it = wd_entry.value.file_table.iterator();
	1181	// while (file_it.next()) \|file_entry\| {
	1182	// loop.allocator.free(file_entry.key);
	1183	// }
	1184	// loop.allocator.free(wd_entry.value.dirname);
	1185	// }
	1186	// loop.finishOneEvent();
	1187	// os.close(inotify_fd);
	1188	// channel.destroy();
	1189	// }
	1190	//
	1191	// var event_buf: [4096]u8 align(@alignOf(os.linux.inotify_event)) = undefined;
	1192	//
	1193	// while (true) {
	1194	// const rc = os.linux.read(inotify_fd, &event_buf, event_buf.len);
	1195	// const errno = os.linux.getErrno(rc);
	1196	// switch (errno) {
	1197	// 0 => {
	1198	// // can't use @bytesToSlice because of the special variable length name field
	1199	// var ptr = event_buf[0..].ptr;
	1200	// const end_ptr = ptr + event_buf.len;
	1201	// var ev: *os.linux.inotify_event = undefined;
	1202	// while (@ptrToInt(ptr) < @ptrToInt(end_ptr)) : (ptr += @sizeOf(os.linux.inotify_event) + ev.len) {
	1203	// ev = @ptrCast(*os.linux.inotify_event, ptr);
	1204	// if (ev.mask & os.linux.IN_CLOSE_WRITE == os.linux.IN_CLOSE_WRITE) {
	1205	// const basename_ptr = ptr + @sizeOf(os.linux.inotify_event);
	1206	// const basename_with_null = basename_ptr[0 .. std.mem.len(u8, basename_ptr) + 1];
	1207	// const user_value = blk: {
	1208	// const held = await (async watch.os_data.table_lock.acquire() catch unreachable);
	1209	// defer held.release();
	1210	//
	1211	// const dir = &watch.os_data.wd_table.get(ev.wd).?.value;
	1212	// if (dir.file_table.get(basename_with_null)) \|entry\| {
	1213	// break :blk entry.value;
	1214	// } else {
	1215	// break :blk null;
	1216	// }
	1217	// };
	1218	// if (user_value) \|v\| {
	1219	// await (async channel.put(Event{
	1220	// .id = WatchEventId.CloseWrite,
	1221	// .data = v,
	1222	// }) catch unreachable);
	1223	// }
	1224	// }
	1225	// }
	1226	// },
	1227	// os.linux.EINTR => continue,
	1228	// os.linux.EINVAL => unreachable,
	1229	// os.linux.EFAULT => unreachable,
	1230	// os.linux.EAGAIN => {
	1231	// (await (async loop.linuxWaitFd(
	1232	// inotify_fd,
	1233	// os.linux.EPOLLET \| os.linux.EPOLLIN,
	1234	// ) catch unreachable)) catch \|err\| {
	1235	// const transformed_err = switch (err) {
	1236	// error.FileDescriptorAlreadyPresentInSet => unreachable,
	1237	// error.OperationCausesCircularLoop => unreachable,
	1238	// error.FileDescriptorNotRegistered => unreachable,
	1239	// error.FileDescriptorIncompatibleWithEpoll => unreachable,
	1240	// error.Unexpected => unreachable,
	1241	// else => \|e\| e,
	1242	// };
	1243	// await (async channel.put(transformed_err) catch unreachable);
	1244	// };
	1245	// },
	1246	// else => unreachable,
	1247	// }
	1248	// }
	1249	// }
	1250	// };
	1251	//}
1306	1252
1307	1253	const test_tmp_dir = "std_event_fs_test";
1308	1254
1309		// TODO this test is disabled until the coroutine rewrite is finished.
	1255	// TODO this test is disabled until the async function rewrite is finished.
1310	1256	//test "write a file, watch it, write it again" {
1311	1257	// return error.SkipZigTest;
1312	1258	// const allocator = std.heap.direct_allocator;
...	...	@@ -1355,7 +1301,7 @@ async fn testFsWatch(loop: *Loop) !void {
1355	1301
1356	1302	const ev = try async watch.channel.get();
1357	1303	var ev_consumed = false;
1358		defer if (!ev_consumed) cancel ev;
	1304	defer if (!ev_consumed) await ev;
1359	1305
1360	1306	// overwrite line 2
1361	1307	const fd = try await try async openReadWrite(loop, file_path, File.default_mode);
...	...	@@ -1397,11 +1343,11 @@ pub const OutStream = struct {
1397	1343	};
1398	1344	}
1399	1345
1400		async<mem.Allocator> fn writeFn(out_stream: Stream, bytes: []const u8) Error!void {
	1346	fn writeFn(out_stream: *Stream, bytes: []const u8) Error!void {
1401	1347	const self = @fieldParentPtr(OutStream, "stream", out_stream);
1402	1348	const offset = self.offset;
1403	1349	self.offset += bytes.len;
1404		return await (async pwritev(self.loop, self.fd, [][]const u8{bytes}, offset) catch unreachable);
	1350	return pwritev(self.loop, self.fd, [][]const u8{bytes}, offset);
1405	1351	}
1406	1352	};
1407	1353
...	...	@@ -1423,9 +1369,9 @@ pub const InStream = struct {
1423	1369	};
1424	1370	}
1425	1371
1426		async<mem.Allocator> fn readFn(in_stream: Stream, bytes: []u8) Error!usize {
	1372	fn readFn(in_stream: *Stream, bytes: []u8) Error!usize {
1427	1373	const self = @fieldParentPtr(InStream, "stream", in_stream);
1428		const amt = try await (async preadv(self.loop, self.fd, [][]u8{bytes}, self.offset) catch unreachable);
	1374	const amt = try preadv(self.loop, self.fd, [][]u8{bytes}, self.offset);
1429	1375	self.offset += amt;
1430	1376	return amt;
1431	1377	}

std/event/future.zig+21-28

...	...	@@ -2,13 +2,11 @@ const std = @import("../std.zig");
2	2	const assert = std.debug.assert;
3	3	const testing = std.testing;
4	4	const builtin = @import("builtin");
5		const AtomicRmwOp = builtin.AtomicRmwOp;
6		const AtomicOrder = builtin.AtomicOrder;
7	5	const Lock = std.event.Lock;
8	6	const Loop = std.event.Loop;
9	7
10	8	/// This is a value that starts out unavailable, until resolve() is called
11		/// While it is unavailable, coroutines suspend when they try to get() it,
	9	/// While it is unavailable, functions suspend when they try to get() it,
12	10	/// and then are resumed when resolve() is called.
13	11	/// At this point the value remains forever available, and another resolve() is not allowed.
14	12	pub fn Future(comptime T: type) type {
...	...	@@ -23,7 +21,7 @@ pub fn Future(comptime T: type) type {
23	21	available: u8,
24	22
25	23	const Self = @This();
26		const Queue = std.atomic.Queue(promise);
	24	const Queue = std.atomic.Queue(anyframe);
27	25
28	26	pub fn init(loop: *Loop) Self {
29	27	return Self{
...	...	@@ -37,10 +35,10 @@ pub fn Future(comptime T: type) type {
37	35	/// available.
38	36	/// Thread-safe.
39	37	pub async fn get(self: Self) T {
40		if (@atomicLoad(u8, &self.available, AtomicOrder.SeqCst) == 2) {
	38	if (@atomicLoad(u8, &self.available, .SeqCst) == 2) {
41	39	return &self.data;
42	40	}
43		const held = await (async self.lock.acquire() catch unreachable);
	41	const held = self.lock.acquire();
44	42	held.release();
45	43
46	44	return &self.data;
...	...	@@ -49,7 +47,7 @@ pub fn Future(comptime T: type) type {
49	47	/// Gets the data without waiting for it. If it's available, a pointer is
50	48	/// returned. Otherwise, null is returned.
51	49	pub fn getOrNull(self: Self) ?T {
52		if (@atomicLoad(u8, &self.available, AtomicOrder.SeqCst) == 2) {
	50	if (@atomicLoad(u8, &self.available, .SeqCst) == 2) {
53	51	return &self.data;
54	52	} else {
55	53	return null;
...	...	@@ -62,10 +60,10 @@ pub fn Future(comptime T: type) type {
62	60	/// It's not required to call start() before resolve() but it can be useful since
63	61	/// this method is thread-safe.
64	62	pub async fn start(self: Self) ?T {
65		const state = @cmpxchgStrong(u8, &self.available, 0, 1, AtomicOrder.SeqCst, AtomicOrder.SeqCst) orelse return null;
	63	const state = @cmpxchgStrong(u8, &self.available, 0, 1, .SeqCst, .SeqCst) orelse return null;
66	64	switch (state) {
67	65	1 => {
68		const held = await (async self.lock.acquire() catch unreachable);
	66	const held = self.lock.acquire();
69	67	held.release();
70	68	return &self.data;
71	69	},
...	...	@@ -77,7 +75,7 @@ pub fn Future(comptime T: type) type {
77	75	/// Make the data become available. May be called only once.
78	76	/// Before calling this, modify the `data` property.
79	77	pub fn resolve(self: *Self) void {
80		const prev = @atomicRmw(u8, &self.available, AtomicRmwOp.Xchg, 2, AtomicOrder.SeqCst);
	78	const prev = @atomicRmw(u8, &self.available, .Xchg, 2, .SeqCst);
81	79	assert(prev == 0 or prev == 1); // resolve() called twice
82	80	Lock.Held.release(Lock.Held{ .lock = &self.lock });
83	81	}
...	...	@@ -86,7 +84,7 @@ pub fn Future(comptime T: type) type {
86	84
87	85	test "std.event.Future" {
88	86	// https://github.com/ziglang/zig/issues/1908
89		if (builtin.single_threaded or builtin.os != builtin.Os.linux) return error.SkipZigTest;
	87	if (builtin.single_threaded) return error.SkipZigTest;
90	88
91	89	const allocator = std.heap.direct_allocator;
92	90
...	...	@@ -94,38 +92,33 @@ test "std.event.Future" {
94	92	try loop.initMultiThreaded(allocator);
95	93	defer loop.deinit();
96	94
97		const handle = try async<allocator> testFuture(&loop);
98		defer cancel handle;
	95	const handle = async testFuture(&loop);
99	96
100	97	loop.run();
101	98	}
102	99
103	100	async fn testFuture(loop: *Loop) void {
104		suspend {
105		resume @handle();
106		}
107	101	var future = Future(i32).init(loop);
108	102
109		const a = async waitOnFuture(&future) catch @panic("memory");
110		const b = async waitOnFuture(&future) catch @panic("memory");
111		const c = async resolveFuture(&future) catch @panic("memory");
	103	const a = async waitOnFuture(&future);
	104	const b = async waitOnFuture(&future);
	105	const c = async resolveFuture(&future);
112	106
113		const result = (await a) + (await b);
114		cancel c;
	107	// TODO make this work:
	108	//const result = (await a) + (await b);
	109	const a_result = await a;
	110	const b_result = await b;
	111	const result = a_result + b_result;
	112
	113	await c;
115	114	testing.expect(result == 12);
116	115	}
117	116
118	117	async fn waitOnFuture(future: *Future(i32)) i32 {
119		suspend {
120		resume @handle();
121		}
122		return (await (async future.get() catch @panic("memory"))).*;
	118	return future.get().*;
123	119	}
124	120
125	121	async fn resolveFuture(future: *Future(i32)) void {
126		suspend {
127		resume @handle();
128		}
129	122	future.data = 6;
130	123	future.resolve();
131	124	}

std/event/group.zig+29-67

...	...	@@ -2,46 +2,33 @@ const std = @import("../std.zig");
2	2	const builtin = @import("builtin");
3	3	const Lock = std.event.Lock;
4	4	const Loop = std.event.Loop;
5		const AtomicRmwOp = builtin.AtomicRmwOp;
6		const AtomicOrder = builtin.AtomicOrder;
7	5	const testing = std.testing;
8	6
9	7	/// ReturnType must be `void` or `E!void`
10	8	pub fn Group(comptime ReturnType: type) type {
11	9	return struct {
12		coro_stack: Stack,
	10	frame_stack: Stack,
13	11	alloc_stack: Stack,
14	12	lock: Lock,
15	13
16	14	const Self = @This();
17	15
18	16	const Error = switch (@typeInfo(ReturnType)) {
19		builtin.TypeId.ErrorUnion => \|payload\| payload.error_set,
	17	.ErrorUnion => \|payload\| payload.error_set,
20	18	else => void,
21	19	};
22		const Stack = std.atomic.Stack(promise->ReturnType);
	20	const Stack = std.atomic.Stack(anyframe->ReturnType);
23	21
24	22	pub fn init(loop: *Loop) Self {
25	23	return Self{
26		.coro_stack = Stack.init(),
	24	.frame_stack = Stack.init(),
27	25	.alloc_stack = Stack.init(),
28	26	.lock = Lock.init(loop),
29	27	};
30	28	}
31	29
32		/// Cancel all the outstanding promises. Can be called even if wait was already called.
33		pub fn deinit(self: *Self) void {
34		while (self.coro_stack.pop()) \|node\| {
35		cancel node.data;
36		}
37		while (self.alloc_stack.pop()) \|node\| {
38		cancel node.data;
39		self.lock.loop.allocator.destroy(node);
40		}
41		}
42
43		/// Add a promise to the group. Thread-safe.
44		pub fn add(self: *Self, handle: promise->ReturnType) (error{OutOfMemory}!void) {
	30	/// Add a frame to the group. Thread-safe.
	31	pub fn add(self: *Self, handle: anyframe->ReturnType) (error{OutOfMemory}!void) {
45	32	const node = try self.lock.loop.allocator.create(Stack.Node);
46	33	node.* = Stack.Node{
47	34	.next = undefined,
...	...	@@ -51,57 +38,29 @@ pub fn Group(comptime ReturnType: type) type {
51	38	}
52	39
53	40	/// Add a node to the group. Thread-safe. Cannot fail.
54		/// `node.data` should be the promise handle to add to the group.
55		/// The node's memory should be in the coroutine frame of
	41	/// `node.data` should be the frame handle to add to the group.
	42	/// The node's memory should be in the function frame of
56	43	/// the handle that is in the node, or somewhere guaranteed to live
57	44	/// at least as long.
58	45	pub fn addNode(self: Self, node: Stack.Node) void {
59		self.coro_stack.push(node);
60		}
61
62		/// This is equivalent to an async call, but the async function is added to the group, instead
63		/// of returning a promise. func must be async and have return type ReturnType.
64		/// Thread-safe.
65		pub fn call(self: *Self, comptime func: var, args: ...) (error{OutOfMemory}!void) {
66		const S = struct {
67		async fn asyncFunc(node: **Stack.Node, args2: ...) ReturnType {
68		// TODO this is a hack to make the memory following be inside the coro frame
69		suspend {
70		var my_node: Stack.Node = undefined;
71		node.* = &my_node;
72		resume @handle();
73		}
74
75		// TODO this allocation elision should be guaranteed because we await it in
76		// this coro frame
77		return await (async func(args2) catch unreachable);
78		}
79		};
80		var node: *Stack.Node = undefined;
81		const handle = try async<self.lock.loop.allocator> S.asyncFunc(&node, args);
82		node.* = Stack.Node{
83		.next = undefined,
84		.data = handle,
85		};
86		self.coro_stack.push(node);
	46	self.frame_stack.push(node);
87	47	}
88	48
89	49	/// Wait for all the calls and promises of the group to complete.
90	50	/// Thread-safe.
91	51	/// Safe to call any number of times.
92	52	pub async fn wait(self: *Self) ReturnType {
93		// TODO catch unreachable because the allocation can be grouped with
94		// the coro frame allocation
95		const held = await (async self.lock.acquire() catch unreachable);
	53	const held = self.lock.acquire();
96	54	defer held.release();
97	55
98		while (self.coro_stack.pop()) \|node\| {
	56	var result: ReturnType = {};
	57
	58	while (self.frame_stack.pop()) \|node\| {
99	59	if (Error == void) {
100	60	await node.data;
101	61	} else {
102	62	(await node.data) catch \|err\| {
103		self.deinit();
104		return err;
	63	result = err;
105	64	};
106	65	}
107	66	}
...	...	@@ -112,11 +71,11 @@ pub fn Group(comptime ReturnType: type) type {
112	71	await handle;
113	72	} else {
114	73	(await handle) catch \|err\| {
115		self.deinit();
116		return err;
	74	result = err;
117	75	};
118	76	}
119	77	}
	78	return result;
120	79	}
121	80	};
122	81	}
...	...	@@ -131,8 +90,7 @@ test "std.event.Group" {
131	90	try loop.initMultiThreaded(allocator);
132	91	defer loop.deinit();
133	92
134		const handle = try async<allocator> testGroup(&loop);
135		defer cancel handle;
	93	const handle = async testGroup(&loop);
136	94
137	95	loop.run();
138	96	}
...	...	@@ -140,26 +98,30 @@ test "std.event.Group" {
140	98	async fn testGroup(loop: *Loop) void {
141	99	var count: usize = 0;
142	100	var group = Group(void).init(loop);
143		group.add(async sleepALittle(&count) catch @panic("memory")) catch @panic("memory");
144		group.call(increaseByTen, &count) catch @panic("memory");
145		await (async group.wait() catch @panic("memory"));
	101	var sleep_a_little_frame = async sleepALittle(&count);
	102	group.add(&sleep_a_little_frame) catch @panic("memory");
	103	var increase_by_ten_frame = async increaseByTen(&count);
	104	group.add(&increase_by_ten_frame) catch @panic("memory");
	105	group.wait();
146	106	testing.expect(count == 11);
147	107
148	108	var another = Group(anyerror!void).init(loop);
149		another.add(async somethingElse() catch @panic("memory")) catch @panic("memory");
150		another.call(doSomethingThatFails) catch @panic("memory");
151		testing.expectError(error.ItBroke, await (async another.wait() catch @panic("memory")));
	109	var something_else_frame = async somethingElse();
	110	another.add(&something_else_frame) catch @panic("memory");
	111	var something_that_fails_frame = async doSomethingThatFails();
	112	another.add(&something_that_fails_frame) catch @panic("memory");
	113	testing.expectError(error.ItBroke, another.wait());
152	114	}
153	115
154	116	async fn sleepALittle(count: *usize) void {
155	117	std.time.sleep(1 * std.time.millisecond);
156		_ = @atomicRmw(usize, count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	118	_ = @atomicRmw(usize, count, .Add, 1, .SeqCst);
157	119	}
158	120
159	121	async fn increaseByTen(count: *usize) void {
160	122	var i: usize = 0;
161	123	while (i < 10) : (i += 1) {
162		_ = @atomicRmw(usize, count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	124	_ = @atomicRmw(usize, count, .Add, 1, .SeqCst);
163	125	}
164	126	}
165	127

std/event/io.zig+9-10

...	...	@@ -1,6 +1,5 @@
1	1	const std = @import("../std.zig");
2	2	const builtin = @import("builtin");
3		const Allocator = std.mem.Allocator;
4	3	const assert = std.debug.assert;
5	4	const mem = std.mem;
6	5
...	...	@@ -12,13 +11,13 @@ pub fn InStream(comptime ReadError: type) type {
12	11	/// Return the number of bytes read. It may be less than buffer.len.
13	12	/// If the number of bytes read is 0, it means end of stream.
14	13	/// End of stream is not an error condition.
15		readFn: async<Allocator> fn (self: Self, buffer: []u8) Error!usize,
	14	readFn: async fn (self: *Self, buffer: []u8) Error!usize,
16	15
17	16	/// Return the number of bytes read. It may be less than buffer.len.
18	17	/// If the number of bytes read is 0, it means end of stream.
19	18	/// End of stream is not an error condition.
20	19	pub async fn read(self: *Self, buffer: []u8) !usize {
21		return await (async self.readFn(self, buffer) catch unreachable);
	20	return self.readFn(self, buffer);
22	21	}
23	22
24	23	/// Return the number of bytes read. If it is less than buffer.len
...	...	@@ -26,7 +25,7 @@ pub fn InStream(comptime ReadError: type) type {
26	25	pub async fn readFull(self: *Self, buffer: []u8) !usize {
27	26	var index: usize = 0;
28	27	while (index != buf.len) {
29		const amt_read = try await (async self.read(buf[index..]) catch unreachable);
	28	const amt_read = try self.read(buf[index..]);
30	29	if (amt_read == 0) return index;
31	30	index += amt_read;
32	31	}
...	...	@@ -35,25 +34,25 @@ pub fn InStream(comptime ReadError: type) type {
35	34
36	35	/// Same as `readFull` but end of stream returns `error.EndOfStream`.
37	36	pub async fn readNoEof(self: *Self, buf: []u8) !void {
38		const amt_read = try await (async self.readFull(buf[index..]) catch unreachable);
	37	const amt_read = try self.readFull(buf[index..]);
39	38	if (amt_read < buf.len) return error.EndOfStream;
40	39	}
41	40
42	41	pub async fn readIntLittle(self: *Self, comptime T: type) !T {
43	42	var bytes: [@sizeOf(T)]u8 = undefined;
44		try await (async self.readNoEof(bytes[0..]) catch unreachable);
	43	try self.readNoEof(bytes[0..]);
45	44	return mem.readIntLittle(T, &bytes);
46	45	}
47	46
48	47	pub async fn readIntBe(self: *Self, comptime T: type) !T {
49	48	var bytes: [@sizeOf(T)]u8 = undefined;
50		try await (async self.readNoEof(bytes[0..]) catch unreachable);
	49	try self.readNoEof(bytes[0..]);
51	50	return mem.readIntBig(T, &bytes);
52	51	}
53	52
54	53	pub async fn readInt(self: *Self, comptime T: type, endian: builtin.Endian) !T {
55	54	var bytes: [@sizeOf(T)]u8 = undefined;
56		try await (async self.readNoEof(bytes[0..]) catch unreachable);
	55	try self.readNoEof(bytes[0..]);
57	56	return mem.readInt(T, &bytes, endian);
58	57	}
59	58
...	...	@@ -61,7 +60,7 @@ pub fn InStream(comptime ReadError: type) type {
61	60	// Only extern and packed structs have defined in-memory layout.
62	61	comptime assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto);
63	62	var res: [1]T = undefined;
64		try await (async self.readNoEof(@sliceToBytes(res[0..])) catch unreachable);
	63	try self.readNoEof(@sliceToBytes(res[0..]));
65	64	return res[0];
66	65	}
67	66	};
...	...	@@ -72,6 +71,6 @@ pub fn OutStream(comptime WriteError: type) type {
72	71	const Self = @This();
73	72	pub const Error = WriteError;
74	73
75		writeFn: async<Allocator> fn (self: Self, buffer: []u8) Error!void,
	74	writeFn: async fn (self: *Self, buffer: []u8) Error!void,
76	75	};
77	76	}

std/event/lock.zig+23-36

...	...	@@ -3,12 +3,10 @@ const builtin = @import("builtin");
3	3	const assert = std.debug.assert;
4	4	const testing = std.testing;
5	5	const mem = std.mem;
6		const AtomicRmwOp = builtin.AtomicRmwOp;
7		const AtomicOrder = builtin.AtomicOrder;
8	6	const Loop = std.event.Loop;
9	7
10	8	/// Thread-safe async/await lock.
11		/// coroutines which are waiting for the lock are suspended, and
	9	/// Functions which are waiting for the lock are suspended, and
12	10	/// are resumed when the lock is released, in order.
13	11	/// Allows only one actor to hold the lock.
14	12	pub const Lock = struct {
...	...	@@ -17,7 +15,7 @@ pub const Lock = struct {
17	15	queue: Queue,
18	16	queue_empty_bit: u8, // TODO make this a bool
19	17
20		const Queue = std.atomic.Queue(promise);
	18	const Queue = std.atomic.Queue(anyframe);
21	19
22	20	pub const Held = struct {
23	21	lock: *Lock,
...	...	@@ -30,19 +28,19 @@ pub const Lock = struct {
30	28	}
31	29
32	30	// We need to release the lock.
33		_ = @atomicRmw(u8, &self.lock.queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
34		_ = @atomicRmw(u8, &self.lock.shared_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	31	_ = @atomicRmw(u8, &self.lock.queue_empty_bit, .Xchg, 1, .SeqCst);
	32	_ = @atomicRmw(u8, &self.lock.shared_bit, .Xchg, 0, .SeqCst);
35	33
36	34	// There might be a queue item. If we know the queue is empty, we can be done,
37	35	// because the other actor will try to obtain the lock.
38	36	// But if there's a queue item, we are the actor which must loop and attempt
39	37	// to grab the lock again.
40		if (@atomicLoad(u8, &self.lock.queue_empty_bit, AtomicOrder.SeqCst) == 1) {
	38	if (@atomicLoad(u8, &self.lock.queue_empty_bit, .SeqCst) == 1) {
41	39	return;
42	40	}
43	41
44	42	while (true) {
45		const old_bit = @atomicRmw(u8, &self.lock.shared_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
	43	const old_bit = @atomicRmw(u8, &self.lock.shared_bit, .Xchg, 1, .SeqCst);
46	44	if (old_bit != 0) {
47	45	// We did not obtain the lock. Great, the queue is someone else's problem.
48	46	return;
...	...	@@ -55,11 +53,11 @@ pub const Lock = struct {
55	53	}
56	54
57	55	// Release the lock again.
58		_ = @atomicRmw(u8, &self.lock.queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
59		_ = @atomicRmw(u8, &self.lock.shared_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	56	_ = @atomicRmw(u8, &self.lock.queue_empty_bit, .Xchg, 1, .SeqCst);
	57	_ = @atomicRmw(u8, &self.lock.shared_bit, .Xchg, 0, .SeqCst);
60	58
61	59	// Find out if we can be done.
62		if (@atomicLoad(u8, &self.lock.queue_empty_bit, AtomicOrder.SeqCst) == 1) {
	60	if (@atomicLoad(u8, &self.lock.queue_empty_bit, .SeqCst) == 1) {
63	61	return;
64	62	}
65	63	}
...	...	@@ -88,28 +86,23 @@ pub const Lock = struct {
88	86	/// All calls to acquire() and release() must complete before calling deinit().
89	87	pub fn deinit(self: *Lock) void {
90	88	assert(self.shared_bit == 0);
91		while (self.queue.get()) \|node\| cancel node.data;
	89	while (self.queue.get()) \|node\| resume node.data;
92	90	}
93	91
94	92	pub async fn acquire(self: *Lock) Held {
95		// TODO explicitly put this memory in the coroutine frame #1194
96		suspend {
97		resume @handle();
98		}
99		var my_tick_node = Loop.NextTickNode.init(@handle());
	93	var my_tick_node = Loop.NextTickNode.init(@frame());
100	94
101	95	errdefer _ = self.queue.remove(&my_tick_node); // TODO test canceling an acquire
102	96	suspend {
103	97	self.queue.put(&my_tick_node);
104	98
105		// At this point, we are in the queue, so we might have already been resumed and this coroutine
106		// frame might be destroyed. For the rest of the suspend block we cannot access the coroutine frame.
	99	// At this point, we are in the queue, so we might have already been resumed.
107	100
108	101	// We set this bit so that later we can rely on the fact, that if queue_empty_bit is 1, some actor
109	102	// will attempt to grab the lock.
110		_ = @atomicRmw(u8, &self.queue_empty_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	103	_ = @atomicRmw(u8, &self.queue_empty_bit, .Xchg, 0, .SeqCst);
111	104
112		const old_bit = @atomicRmw(u8, &self.shared_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
	105	const old_bit = @atomicRmw(u8, &self.shared_bit, .Xchg, 1, .SeqCst);
113	106	if (old_bit == 0) {
114	107	if (self.queue.get()) \|node\| {
115	108	// Whether this node is us or someone else, we tail resume it.
...	...	@@ -123,8 +116,7 @@ pub const Lock = struct {
123	116	};
124	117
125	118	test "std.event.Lock" {
126		// TODO https://github.com/ziglang/zig/issues/2377
127		if (true) return error.SkipZigTest;
	119	// TODO https://github.com/ziglang/zig/issues/1908
128	120	if (builtin.single_threaded) return error.SkipZigTest;
129	121
130	122	const allocator = std.heap.direct_allocator;
...	...	@@ -136,39 +128,34 @@ test "std.event.Lock" {
136	128	var lock = Lock.init(&loop);
137	129	defer lock.deinit();
138	130
139		const handle = try async<allocator> testLock(&loop, &lock);
140		defer cancel handle;
	131	_ = async testLock(&loop, &lock);
141	132	loop.run();
142	133
143	134	testing.expectEqualSlices(i32, [1]i32{3 * @intCast(i32, shared_test_data.len)} ** shared_test_data.len, shared_test_data);
144	135	}
145	136
146	137	async fn testLock(loop: Loop, lock: Lock) void {
147		// TODO explicitly put next tick node memory in the coroutine frame #1194
148		suspend {
149		resume @handle();
150		}
151		const handle1 = async lockRunner(lock) catch @panic("out of memory");
	138	const handle1 = async lockRunner(lock);
152	139	var tick_node1 = Loop.NextTickNode{
153	140	.prev = undefined,
154	141	.next = undefined,
155		.data = handle1,
	142	.data = &handle1,
156	143	};
157	144	loop.onNextTick(&tick_node1);
158	145
159		const handle2 = async lockRunner(lock) catch @panic("out of memory");
	146	const handle2 = async lockRunner(lock);
160	147	var tick_node2 = Loop.NextTickNode{
161	148	.prev = undefined,
162	149	.next = undefined,
163		.data = handle2,
	150	.data = &handle2,
164	151	};
165	152	loop.onNextTick(&tick_node2);
166	153
167		const handle3 = async lockRunner(lock) catch @panic("out of memory");
	154	const handle3 = async lockRunner(lock);
168	155	var tick_node3 = Loop.NextTickNode{
169	156	.prev = undefined,
170	157	.next = undefined,
171		.data = handle3,
	158	.data = &handle3,
172	159	};
173	160	loop.onNextTick(&tick_node3);
174	161
...	...	@@ -185,7 +172,7 @@ async fn lockRunner(lock: *Lock) void {
185	172
186	173	var i: usize = 0;
187	174	while (i < shared_test_data.len) : (i += 1) {
188		const lock_promise = async lock.acquire() catch @panic("out of memory");
	175	const lock_promise = async lock.acquire();
189	176	const handle = await lock_promise;
190	177	defer handle.release();
191	178

std/event/locked.zig+1-1

...	...	@@ -3,7 +3,7 @@ const Lock = std.event.Lock;
3	3	const Loop = std.event.Loop;
4	4
5	5	/// Thread-safe async/await lock that protects one piece of data.
6		/// coroutines which are waiting for the lock are suspended, and
	6	/// Functions which are waiting for the lock are suspended, and
7	7	/// are resumed when the lock is released, in order.
8	8	pub fn Locked(comptime T: type) type {
9	9	return struct {

std/event/loop.zig+83-69

...	...	@@ -1,5 +1,6 @@
1	1	const std = @import("../std.zig");
2	2	const builtin = @import("builtin");
	3	const root = @import("root");
3	4	const assert = std.debug.assert;
4	5	const testing = std.testing;
5	6	const mem = std.mem;
...	...	@@ -13,7 +14,7 @@ const Thread = std.Thread;
13	14
14	15	pub const Loop = struct {
15	16	allocator: *mem.Allocator,
16		next_tick_queue: std.atomic.Queue(promise),
	17	next_tick_queue: std.atomic.Queue(anyframe),
17	18	os_data: OsData,
18	19	final_resume_node: ResumeNode,
19	20	pending_event_count: usize,
...	...	@@ -24,11 +25,11 @@ pub const Loop = struct {
24	25	available_eventfd_resume_nodes: std.atomic.Stack(ResumeNode.EventFd),
25	26	eventfd_resume_nodes: []std.atomic.Stack(ResumeNode.EventFd).Node,
26	27
27		pub const NextTickNode = std.atomic.Queue(promise).Node;
	28	pub const NextTickNode = std.atomic.Queue(anyframe).Node;
28	29
29	30	pub const ResumeNode = struct {
30	31	id: Id,
31		handle: promise,
	32	handle: anyframe,
32	33	overlapped: Overlapped,
33	34
34	35	pub const overlapped_init = switch (builtin.os) {
...	...	@@ -85,18 +86,43 @@ pub const Loop = struct {
85	86	};
86	87	};
87	88
	89	pub const IoMode = enum {
	90	blocking,
	91	evented,
	92	};
	93	pub const io_mode: IoMode = if (@hasDecl(root, "io_mode")) root.io_mode else IoMode.blocking;
	94	var global_instance_state: Loop = undefined;
	95	const default_instance: ?*Loop = switch (io_mode) {
	96	.blocking => null,
	97	.evented => &global_instance_state,
	98	};
	99	pub const instance: ?*Loop = if (@hasDecl(root, "event_loop")) root.event_loop else default_instance;
	100
	101	/// TODO copy elision / named return values so that the threads referencing *Loop
	102	/// have the correct pointer value.
	103	/// https://github.com/ziglang/zig/issues/2761 and https://github.com/ziglang/zig/issues/2765
	104	pub fn init(self: Loop, allocator: mem.Allocator) !void {
	105	if (builtin.single_threaded) {
	106	return self.initSingleThreaded(allocator);
	107	} else {
	108	return self.initMultiThreaded(allocator);
	109	}
	110	}
	111
88	112	/// After initialization, call run().
89	113	/// TODO copy elision / named return values so that the threads referencing *Loop
90	114	/// have the correct pointer value.
	115	/// https://github.com/ziglang/zig/issues/2761 and https://github.com/ziglang/zig/issues/2765
91	116	pub fn initSingleThreaded(self: Loop, allocator: mem.Allocator) !void {
92	117	return self.initInternal(allocator, 1);
93	118	}
94	119
95	120	/// The allocator must be thread-safe because we use it for multiplexing
96		/// coroutines onto kernel threads.
	121	/// async functions onto kernel threads.
97	122	/// After initialization, call run().
98	123	/// TODO copy elision / named return values so that the threads referencing *Loop
99	124	/// have the correct pointer value.
	125	/// https://github.com/ziglang/zig/issues/2761 and https://github.com/ziglang/zig/issues/2765
100	126	pub fn initMultiThreaded(self: Loop, allocator: mem.Allocator) !void {
101	127	if (builtin.single_threaded) @compileError("initMultiThreaded unavailable when building in single-threaded mode");
102	128	const core_count = try Thread.cpuCount();
...	...	@@ -110,7 +136,7 @@ pub const Loop = struct {
110	136	.pending_event_count = 1,
111	137	.allocator = allocator,
112	138	.os_data = undefined,
113		.next_tick_queue = std.atomic.Queue(promise).init(),
	139	.next_tick_queue = std.atomic.Queue(anyframe).init(),
114	140	.extra_threads = undefined,
115	141	.available_eventfd_resume_nodes = std.atomic.Stack(ResumeNode.EventFd).init(),
116	142	.eventfd_resume_nodes = undefined,
...	...	@@ -397,7 +423,7 @@ pub const Loop = struct {
397	423	}
398	424	}
399	425
400		/// resume_node must live longer than the promise that it holds a reference to.
	426	/// resume_node must live longer than the anyframe that it holds a reference to.
401	427	/// flags must contain EPOLLET
402	428	pub fn linuxAddFd(self: Loop, fd: i32, resume_node: ResumeNode, flags: u32) !void {
403	429	assert(flags & os.EPOLLET == os.EPOLLET);
...	...	@@ -428,11 +454,10 @@ pub const Loop = struct {
428	454	pub async fn linuxWaitFd(self: *Loop, fd: i32, flags: u32) !void {
429	455	defer self.linuxRemoveFd(fd);
430	456	suspend {
431		// TODO explicitly put this memory in the coroutine frame #1194
432	457	var resume_node = ResumeNode.Basic{
433	458	.base = ResumeNode{
434	459	.id = ResumeNode.Id.Basic,
435		.handle = @handle(),
	460	.handle = @frame(),
436	461	.overlapped = ResumeNode.overlapped_init,
437	462	},
438	463	};
...	...	@@ -441,14 +466,10 @@ pub const Loop = struct {
441	466	}
442	467
443	468	pub async fn bsdWaitKev(self: *Loop, ident: usize, filter: i16, fflags: u32) !os.Kevent {
444		// TODO #1194
445		suspend {
446		resume @handle();
447		}
448	469	var resume_node = ResumeNode.Basic{
449	470	.base = ResumeNode{
450	471	.id = ResumeNode.Id.Basic,
451		.handle = @handle(),
	472	.handle = @frame(),
452	473	.overlapped = ResumeNode.overlapped_init,
453	474	},
454	475	.kev = undefined,
...	...	@@ -460,7 +481,7 @@ pub const Loop = struct {
460	481	return resume_node.kev;
461	482	}
462	483
463		/// resume_node must live longer than the promise that it holds a reference to.
	484	/// resume_node must live longer than the anyframe that it holds a reference to.
464	485	pub fn bsdAddKev(self: Loop, resume_node: ResumeNode.Basic, ident: usize, filter: i16, fflags: u32) !void {
465	486	self.beginOneEvent();
466	487	errdefer self.finishOneEvent();
...	...	@@ -561,10 +582,10 @@ pub const Loop = struct {
561	582	self.workerRun();
562	583
563	584	switch (builtin.os) {
564		builtin.Os.linux,
565		builtin.Os.macosx,
566		builtin.Os.freebsd,
567		builtin.Os.netbsd,
	585	.linux,
	586	.macosx,
	587	.freebsd,
	588	.netbsd,
568	589	=> self.os_data.fs_thread.wait(),
569	590	else => {},
570	591	}
...	...	@@ -574,45 +595,39 @@ pub const Loop = struct {
574	595	}
575	596	}
576	597
577		/// This is equivalent to an async call, except instead of beginning execution of the async function,
578		/// it immediately returns to the caller, and the async function is queued in the event loop. It still
579		/// returns a promise to be awaited.
580		pub fn call(self: *Loop, comptime func: var, args: ...) !(promise->@typeOf(func).ReturnType) {
581		const S = struct {
582		async fn asyncFunc(loop: Loop, handle: promise->@typeOf(func).ReturnType, args2: ...) @typeOf(func).ReturnType {
583		suspend {
584		handle.* = @handle();
585		var my_tick_node = Loop.NextTickNode{
586		.prev = undefined,
587		.next = undefined,
588		.data = @handle(),
589		};
590		loop.onNextTick(&my_tick_node);
591		}
592		// TODO guaranteed allocation elision for await in same func as async
593		return await (async func(args2) catch unreachable);
594		}
595		};
596		var handle: promise->@typeOf(func).ReturnType = undefined;
597		return async<self.allocator> S.asyncFunc(self, &handle, args);
	598	/// This is equivalent to function call, except it calls `startCpuBoundOperation` first.
	599	pub fn call(comptime func: var, args: ...) @typeOf(func).ReturnType {
	600	startCpuBoundOperation();
	601	return func(args);
598	602	}
599	603
600		/// Awaiting a yield lets the event loop run, starting any unstarted async operations.
	604	/// Yielding lets the event loop run, starting any unstarted async operations.
601	605	/// Note that async operations automatically start when a function yields for any other reason,
602	606	/// for example, when async I/O is performed. This function is intended to be used only when
603	607	/// CPU bound tasks would be waiting in the event loop but never get started because no async I/O
604	608	/// is performed.
605		pub async fn yield(self: *Loop) void {
	609	pub fn yield(self: *Loop) void {
606	610	suspend {
607		var my_tick_node = Loop.NextTickNode{
	611	var my_tick_node = NextTickNode{
608	612	.prev = undefined,
609	613	.next = undefined,
610		.data = @handle(),
	614	.data = @frame(),
611	615	};
612	616	self.onNextTick(&my_tick_node);
613	617	}
614	618	}
615	619
	620	/// If the build is multi-threaded and there is an event loop, then it calls `yield`. Otherwise,
	621	/// does nothing.
	622	pub fn startCpuBoundOperation() void {
	623	if (builtin.single_threaded) {
	624	return;
	625	} else if (instance) \|event_loop\| {
	626	event_loop.yield();
	627	}
	628	}
	629
	630
616	631	/// call finishOneEvent when done
617	632	pub fn beginOneEvent(self: *Loop) void {
618	633	_ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
...	...	@@ -672,9 +687,9 @@ pub const Loop = struct {
672	687	const handle = resume_node.handle;
673	688	const resume_node_id = resume_node.id;
674	689	switch (resume_node_id) {
675		ResumeNode.Id.Basic => {},
676		ResumeNode.Id.Stop => return,
677		ResumeNode.Id.EventFd => {
	690	.Basic => {},
	691	.Stop => return,
	692	.EventFd => {
678	693	const event_fd_node = @fieldParentPtr(ResumeNode.EventFd, "base", resume_node);
679	694	event_fd_node.epoll_op = os.EPOLL_CTL_MOD;
680	695	const stack_node = @fieldParentPtr(std.atomic.Stack(ResumeNode.EventFd).Node, "data", event_fd_node);
...	...	@@ -696,12 +711,12 @@ pub const Loop = struct {
696	711	const handle = resume_node.handle;
697	712	const resume_node_id = resume_node.id;
698	713	switch (resume_node_id) {
699		ResumeNode.Id.Basic => {
	714	.Basic => {
700	715	const basic_node = @fieldParentPtr(ResumeNode.Basic, "base", resume_node);
701	716	basic_node.kev = ev;
702	717	},
703		ResumeNode.Id.Stop => return,
704		ResumeNode.Id.EventFd => {
	718	.Stop => return,
	719	.EventFd => {
705	720	const event_fd_node = @fieldParentPtr(ResumeNode.EventFd, "base", resume_node);
706	721	const stack_node = @fieldParentPtr(std.atomic.Stack(ResumeNode.EventFd).Node, "data", event_fd_node);
707	722	self.available_eventfd_resume_nodes.push(stack_node);
...	...	@@ -730,9 +745,9 @@ pub const Loop = struct {
730	745	const handle = resume_node.handle;
731	746	const resume_node_id = resume_node.id;
732	747	switch (resume_node_id) {
733		ResumeNode.Id.Basic => {},
734		ResumeNode.Id.Stop => return,
735		ResumeNode.Id.EventFd => {
	748	.Basic => {},
	749	.Stop => return,
	750	.EventFd => {
736	751	const event_fd_node = @fieldParentPtr(ResumeNode.EventFd, "base", resume_node);
737	752	const stack_node = @fieldParentPtr(std.atomic.Stack(ResumeNode.EventFd).Node, "data", event_fd_node);
738	753	self.available_eventfd_resume_nodes.push(stack_node);
...	...	@@ -750,12 +765,12 @@ pub const Loop = struct {
750	765	self.beginOneEvent(); // finished in posixFsRun after processing the msg
751	766	self.os_data.fs_queue.put(request_node);
752	767	switch (builtin.os) {
753		builtin.Os.macosx, builtin.Os.freebsd, builtin.Os.netbsd => {
	768	.macosx, .freebsd, .netbsd => {
754	769	const fs_kevs = (*const [1]os.Kevent)(&self.os_data.fs_kevent_wake);
755	770	const empty_kevs = ([*]os.Kevent)(undefined)[0..0];
756	771	_ = os.kevent(self.os_data.fs_kqfd, fs_kevs, empty_kevs, null) catch unreachable;
757	772	},
758		builtin.Os.linux => {
	773	.linux => {
759	774	_ = @atomicRmw(i32, &self.os_data.fs_queue_item, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
760	775	const rc = os.linux.futex_wake(&self.os_data.fs_queue_item, os.linux.FUTEX_WAKE, 1);
761	776	switch (os.linux.getErrno(rc)) {
...	...	@@ -781,18 +796,18 @@ pub const Loop = struct {
781	796	}
782	797	while (self.os_data.fs_queue.get()) \|node\| {
783	798	switch (node.data.msg) {
784		@TagType(fs.Request.Msg).End => return,
785		@TagType(fs.Request.Msg).PWriteV => \|*msg\| {
	799	.End => return,
	800	.PWriteV => \|*msg\| {
786	801	msg.result = os.pwritev(msg.fd, msg.iov, msg.offset);
787	802	},
788		@TagType(fs.Request.Msg).PReadV => \|*msg\| {
	803	.PReadV => \|*msg\| {
789	804	msg.result = os.preadv(msg.fd, msg.iov, msg.offset);
790	805	},
791		@TagType(fs.Request.Msg).Open => \|*msg\| {
	806	.Open => \|*msg\| {
792	807	msg.result = os.openC(msg.path.ptr, msg.flags, msg.mode);
793	808	},
794		@TagType(fs.Request.Msg).Close => \|*msg\| os.close(msg.fd),
795		@TagType(fs.Request.Msg).WriteFile => \|*msg\| blk: {
	809	.Close => \|*msg\| os.close(msg.fd),
	810	.WriteFile => \|*msg\| blk: {
796	811	const flags = os.O_LARGEFILE \| os.O_WRONLY \| os.O_CREAT \|
797	812	os.O_CLOEXEC \| os.O_TRUNC;
798	813	const fd = os.openC(msg.path.ptr, flags, msg.mode) catch \|err\| {
...	...	@@ -804,11 +819,11 @@ pub const Loop = struct {
804	819	},
805	820	}
806	821	switch (node.data.finish) {
807		@TagType(fs.Request.Finish).TickNode => \|*tick_node\| self.onNextTick(tick_node),
808		@TagType(fs.Request.Finish).DeallocCloseOperation => \|close_op\| {
	822	.TickNode => \|*tick_node\| self.onNextTick(tick_node),
	823	.DeallocCloseOperation => \|close_op\| {
809	824	self.allocator.destroy(close_op);
810	825	},
811		@TagType(fs.Request.Finish).NoAction => {},
	826	.NoAction => {},
812	827	}
813	828	self.finishOneEvent();
814	829	}
...	...	@@ -864,7 +879,7 @@ pub const Loop = struct {
864	879
865	880	test "std.event.Loop - basic" {
866	881	// https://github.com/ziglang/zig/issues/1908
867		if (builtin.single_threaded or builtin.os != builtin.Os.linux) return error.SkipZigTest;
	882	if (builtin.single_threaded) return error.SkipZigTest;
868	883
869	884	const allocator = std.heap.direct_allocator;
870	885
...	...	@@ -877,7 +892,7 @@ test "std.event.Loop - basic" {
877	892
878	893	test "std.event.Loop - call" {
879	894	// https://github.com/ziglang/zig/issues/1908
880		if (builtin.single_threaded or builtin.os != builtin.Os.linux) return error.SkipZigTest;
	895	if (builtin.single_threaded) return error.SkipZigTest;
881	896
882	897	const allocator = std.heap.direct_allocator;
883	898
...	...	@@ -886,9 +901,8 @@ test "std.event.Loop - call" {
886	901	defer loop.deinit();
887	902
888	903	var did_it = false;
889		const handle = try loop.call(testEventLoop);
890		const handle2 = try loop.call(testEventLoop2, handle, &did_it);
891		defer cancel handle2;
	904	const handle = async Loop.call(testEventLoop);
	905	const handle2 = async Loop.call(testEventLoop2, &handle, &did_it);
892	906
893	907	loop.run();
894	908
...	...	@@ -899,7 +913,7 @@ async fn testEventLoop() i32 {
899	913	return 1234;
900	914	}
901	915
902		async fn testEventLoop2(h: promise->i32, did_it: *bool) void {
	916	async fn testEventLoop2(h: anyframe->i32, did_it: *bool) void {
903	917	const value = await h;
904	918	testing.expect(value == 1234);
905	919	did_it.* = true;

std/event/net.zig+27-38

...	...	@@ -9,24 +9,24 @@ const File = std.fs.File;
9	9	const fd_t = os.fd_t;
10	10
11	11	pub const Server = struct {
12		handleRequestFn: async<mem.Allocator> fn (Server, *const std.net.Address, File) void,
	12	handleRequestFn: async fn (Server, const std.net.Address, File) void,
13	13
14	14	loop: *Loop,
15	15	sockfd: ?i32,
16		accept_coro: ?promise,
	16	accept_frame: ?anyframe,
17	17	listen_address: std.net.Address,
18	18
19	19	waiting_for_emfile_node: PromiseNode,
20	20	listen_resume_node: event.Loop.ResumeNode,
21	21
22		const PromiseNode = std.TailQueue(promise).Node;
	22	const PromiseNode = std.TailQueue(anyframe).Node;
23	23
24	24	pub fn init(loop: *Loop) Server {
25		// TODO can't initialize handler coroutine here because we need well defined copy elision
	25	// TODO can't initialize handler here because we need well defined copy elision
26	26	return Server{
27	27	.loop = loop,
28	28	.sockfd = null,
29		.accept_coro = null,
	29	.accept_frame = null,
30	30	.handleRequestFn = undefined,
31	31	.waiting_for_emfile_node = undefined,
32	32	.listen_address = undefined,
...	...	@@ -41,7 +41,7 @@ pub const Server = struct {
41	41	pub fn listen(
42	42	self: *Server,
43	43	address: *const std.net.Address,
44		handleRequestFn: async<mem.Allocator> fn (Server, *const std.net.Address, File) void,
	44	handleRequestFn: async fn (Server, const std.net.Address, File) void,
45	45	) !void {
46	46	self.handleRequestFn = handleRequestFn;
47	47
...	...	@@ -53,10 +53,10 @@ pub const Server = struct {
53	53	try os.listen(sockfd, os.SOMAXCONN);
54	54	self.listen_address = std.net.Address.initPosix(try os.getsockname(sockfd));
55	55
56		self.accept_coro = try async<self.loop.allocator> Server.handler(self);
57		errdefer cancel self.accept_coro.?;
	56	self.accept_frame = async Server.handler(self);
	57	errdefer await self.accept_frame.?;
58	58
59		self.listen_resume_node.handle = self.accept_coro.?;
	59	self.listen_resume_node.handle = self.accept_frame.?;
60	60	try self.loop.linuxAddFd(sockfd, &self.listen_resume_node, os.EPOLLIN \| os.EPOLLOUT \| os.EPOLLET);
61	61	errdefer self.loop.removeFd(sockfd);
62	62	}
...	...	@@ -71,7 +71,7 @@ pub const Server = struct {
71	71	}
72	72
73	73	pub fn deinit(self: *Server) void {
74		if (self.accept_coro) \|accept_coro\| cancel accept_coro;
	74	if (self.accept_frame) \|accept_frame\| await accept_frame;
75	75	if (self.sockfd) \|sockfd\| os.close(sockfd);
76	76	}
77	77
...	...	@@ -86,12 +86,7 @@ pub const Server = struct {
86	86	continue;
87	87	}
88	88	var socket = File.openHandle(accepted_fd);
89		_ = async<self.loop.allocator> self.handleRequestFn(self, &accepted_addr, socket) catch \|err\| switch (err) {
90		error.OutOfMemory => {
91		socket.close();
92		continue;
93		},
94		};
	89	self.handleRequestFn(self, &accepted_addr, socket);
95	90	} else \|err\| switch (err) {
96	91	error.ProcessFdQuotaExceeded => @panic("TODO handle this error"),
97	92	error.ConnectionAborted => continue,
...	...	@@ -124,7 +119,7 @@ pub async fn connectUnixSocket(loop: *Loop, path: []const u8) !i32 {
124	119	mem.copy(u8, sock_addr.path[0..], path);
125	120	const size = @intCast(u32, @sizeOf(os.sa_family_t) + path.len);
126	121	try os.connect_async(sockfd, &sock_addr, size);
127		try await try async loop.linuxWaitFd(sockfd, os.EPOLLIN \| os.EPOLLOUT \| os.EPOLLET);
	122	try loop.linuxWaitFd(sockfd, os.EPOLLIN \| os.EPOLLOUT \| os.EPOLLET);
128	123	try os.getsockoptError(sockfd);
129	124
130	125	return sockfd;
...	...	@@ -149,7 +144,7 @@ pub async fn read(loop: *std.event.Loop, fd: fd_t, buffer: []u8) ReadError!usize
149	144	.iov_len = buffer.len,
150	145	};
151	146	const iovs: *const [1]os.iovec = &iov;
152		return await (async readvPosix(loop, fd, iovs, 1) catch unreachable);
	147	return readvPosix(loop, fd, iovs, 1);
153	148	}
154	149
155	150	pub const WriteError = error{};
...	...	@@ -160,7 +155,7 @@ pub async fn write(loop: *std.event.Loop, fd: fd_t, buffer: []const u8) WriteErr
160	155	.iov_len = buffer.len,
161	156	};
162	157	const iovs: *const [1]os.iovec_const = &iov;
163		return await (async writevPosix(loop, fd, iovs, 1) catch unreachable);
	158	return writevPosix(loop, fd, iovs, 1);
164	159	}
165	160
166	161	pub async fn writevPosix(loop: Loop, fd: i32, iov: []const os.iovec_const, count: usize) !void {
...	...	@@ -174,7 +169,7 @@ pub async fn writevPosix(loop: Loop, fd: i32, iov: []const os.iovec_const, cou
174	169	os.EINVAL => unreachable,
175	170	os.EFAULT => unreachable,
176	171	os.EAGAIN => {
177		try await (async loop.linuxWaitFd(fd, os.EPOLLET \| os.EPOLLOUT) catch unreachable);
	172	try loop.linuxWaitFd(fd, os.EPOLLET \| os.EPOLLOUT);
178	173	continue;
179	174	},
180	175	os.EBADF => unreachable, // always a race condition
...	...	@@ -205,7 +200,7 @@ pub async fn readvPosix(loop: std.event.Loop, fd: i32, iov: []os.iovec, count:
205	200	os.EINVAL => unreachable,
206	201	os.EFAULT => unreachable,
207	202	os.EAGAIN => {
208		try await (async loop.linuxWaitFd(fd, os.EPOLLET \| os.EPOLLIN) catch unreachable);
	203	try loop.linuxWaitFd(fd, os.EPOLLET \| os.EPOLLIN);
209	204	continue;
210	205	},
211	206	os.EBADF => unreachable, // always a race condition
...	...	@@ -232,7 +227,7 @@ pub async fn writev(loop: *Loop, fd: fd_t, data: []const []const u8) !void {
232	227	};
233	228	}
234	229
235		return await (async writevPosix(loop, fd, iovecs.ptr, data.len) catch unreachable);
	230	return writevPosix(loop, fd, iovecs.ptr, data.len);
236	231	}
237	232
238	233	pub async fn readv(loop: *Loop, fd: fd_t, data: []const []u8) !usize {
...	...	@@ -246,7 +241,7 @@ pub async fn readv(loop: *Loop, fd: fd_t, data: []const []u8) !usize {
246	241	};
247	242	}
248	243
249		return await (async readvPosix(loop, fd, iovecs.ptr, data.len) catch unreachable);
	244	return readvPosix(loop, fd, iovecs.ptr, data.len);
250	245	}
251	246
252	247	pub async fn connect(loop: Loop, _address: const std.net.Address) !File {
...	...	@@ -256,7 +251,7 @@ pub async fn connect(loop: Loop, _address: const std.net.Address) !File {
256	251	errdefer os.close(sockfd);
257	252
258	253	try os.connect_async(sockfd, &address.os_addr, @sizeOf(os.sockaddr_in));
259		try await try async loop.linuxWaitFd(sockfd, os.EPOLLIN \| os.EPOLLOUT \| os.EPOLLET);
	254	try loop.linuxWaitFd(sockfd, os.EPOLLIN \| os.EPOLLOUT \| os.EPOLLET);
260	255	try os.getsockoptError(sockfd);
261	256
262	257	return File.openHandle(sockfd);
...	...	@@ -275,18 +270,13 @@ test "listen on a port, send bytes, receive bytes" {
275	270	tcp_server: Server,
276	271
277	272	const Self = @This();
278		async<mem.Allocator> fn handler(tcp_server: Server, _addr: *const std.net.Address, _socket: File) void {
	273	async fn handler(tcp_server: Server, _addr: const std.net.Address, _socket: File) void {
279	274	const self = @fieldParentPtr(Self, "tcp_server", tcp_server);
280	275	var socket = _socket; // TODO https://github.com/ziglang/zig/issues/1592
281	276	defer socket.close();
282		// TODO guarantee elision of this allocation
283		const next_handler = async errorableHandler(self, _addr, socket) catch unreachable;
284		(await next_handler) catch \|err\| {
	277	const next_handler = errorableHandler(self, _addr, socket) catch \|err\| {
285	278	std.debug.panic("unable to handle connection: {}\n", err);
286	279	};
287		suspend {
288		cancel @handle();
289		}
290	280	}
291	281	async fn errorableHandler(self: Self, _addr: const std.net.Address, _socket: File) !void {
292	282	const addr = _addr.*; // TODO https://github.com/ziglang/zig/issues/1592
...	...	@@ -306,15 +296,14 @@ test "listen on a port, send bytes, receive bytes" {
306	296	defer server.tcp_server.deinit();
307	297	try server.tcp_server.listen(&addr, MyServer.handler);
308	298
309		const p = try async<std.debug.global_allocator> doAsyncTest(&loop, &server.tcp_server.listen_address, &server.tcp_server);
310		defer cancel p;
	299	_ = async doAsyncTest(&loop, &server.tcp_server.listen_address, &server.tcp_server);
311	300	loop.run();
312	301	}
313	302
314	303	async fn doAsyncTest(loop: Loop, address: const std.net.Address, server: *Server) void {
315	304	errdefer @panic("test failure");
316	305
317		var socket_file = try await try async connect(loop, address);
	306	var socket_file = try connect(loop, address);
318	307	defer socket_file.close();
319	308
320	309	var buf: [512]u8 = undefined;
...	...	@@ -340,9 +329,9 @@ pub const OutStream = struct {
340	329	};
341	330	}
342	331
343		async<mem.Allocator> fn writeFn(out_stream: Stream, bytes: []const u8) Error!void {
	332	async fn writeFn(out_stream: *Stream, bytes: []const u8) Error!void {
344	333	const self = @fieldParentPtr(OutStream, "stream", out_stream);
345		return await (async write(self.loop, self.fd, bytes) catch unreachable);
	334	return write(self.loop, self.fd, bytes);
346	335	}
347	336	};
348	337
...	...	@@ -362,8 +351,8 @@ pub const InStream = struct {
362	351	};
363	352	}
364	353
365		async<mem.Allocator> fn readFn(in_stream: Stream, bytes: []u8) Error!usize {
	354	async fn readFn(in_stream: *Stream, bytes: []u8) Error!usize {
366	355	const self = @fieldParentPtr(InStream, "stream", in_stream);
367		return await (async read(self.loop, self.fd, bytes) catch unreachable);
	356	return read(self.loop, self.fd, bytes);
368	357	}
369	358	};

std/event/rwlock.zig+46-47

...	...	@@ -3,12 +3,10 @@ const builtin = @import("builtin");
3	3	const assert = std.debug.assert;
4	4	const testing = std.testing;
5	5	const mem = std.mem;
6		const AtomicRmwOp = builtin.AtomicRmwOp;
7		const AtomicOrder = builtin.AtomicOrder;
8	6	const Loop = std.event.Loop;
9	7
10	8	/// Thread-safe async/await lock.
11		/// coroutines which are waiting for the lock are suspended, and
	9	/// Functions which are waiting for the lock are suspended, and
12	10	/// are resumed when the lock is released, in order.
13	11	/// Many readers can hold the lock at the same time; however locking for writing is exclusive.
14	12	/// When a read lock is held, it will not be released until the reader queue is empty.
...	...	@@ -28,19 +26,19 @@ pub const RwLock = struct {
28	26	const ReadLock = 2;
29	27	};
30	28
31		const Queue = std.atomic.Queue(promise);
	29	const Queue = std.atomic.Queue(anyframe);
32	30
33	31	pub const HeldRead = struct {
34	32	lock: *RwLock,
35	33
36	34	pub fn release(self: HeldRead) void {
37	35	// If other readers still hold the lock, we're done.
38		if (@atomicRmw(usize, &self.lock.reader_lock_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst) != 1) {
	36	if (@atomicRmw(usize, &self.lock.reader_lock_count, .Sub, 1, .SeqCst) != 1) {
39	37	return;
40	38	}
41	39
42		_ = @atomicRmw(u8, &self.lock.reader_queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
43		if (@cmpxchgStrong(u8, &self.lock.shared_state, State.ReadLock, State.Unlocked, AtomicOrder.SeqCst, AtomicOrder.SeqCst) != null) {
	40	_ = @atomicRmw(u8, &self.lock.reader_queue_empty_bit, .Xchg, 1, .SeqCst);
	41	if (@cmpxchgStrong(u8, &self.lock.shared_state, State.ReadLock, State.Unlocked, .SeqCst, .SeqCst) != null) {
44	42	// Didn't unlock. Someone else's problem.
45	43	return;
46	44	}
...	...	@@ -61,17 +59,17 @@ pub const RwLock = struct {
61	59	}
62	60
63	61	// We need to release the write lock. Check if any readers are waiting to grab the lock.
64		if (@atomicLoad(u8, &self.lock.reader_queue_empty_bit, AtomicOrder.SeqCst) == 0) {
	62	if (@atomicLoad(u8, &self.lock.reader_queue_empty_bit, .SeqCst) == 0) {
65	63	// Switch to a read lock.
66		_ = @atomicRmw(u8, &self.lock.shared_state, AtomicRmwOp.Xchg, State.ReadLock, AtomicOrder.SeqCst);
	64	_ = @atomicRmw(u8, &self.lock.shared_state, .Xchg, State.ReadLock, .SeqCst);
67	65	while (self.lock.reader_queue.get()) \|node\| {
68	66	self.lock.loop.onNextTick(node);
69	67	}
70	68	return;
71	69	}
72	70
73		_ = @atomicRmw(u8, &self.lock.writer_queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
74		_ = @atomicRmw(u8, &self.lock.shared_state, AtomicRmwOp.Xchg, State.Unlocked, AtomicOrder.SeqCst);
	71	_ = @atomicRmw(u8, &self.lock.writer_queue_empty_bit, .Xchg, 1, .SeqCst);
	72	_ = @atomicRmw(u8, &self.lock.shared_state, .Xchg, State.Unlocked, .SeqCst);
75	73
76	74	self.lock.commonPostUnlock();
77	75	}
...	...	@@ -93,32 +91,30 @@ pub const RwLock = struct {
93	91	/// All calls to acquire() and release() must complete before calling deinit().
94	92	pub fn deinit(self: *RwLock) void {
95	93	assert(self.shared_state == State.Unlocked);
96		while (self.writer_queue.get()) \|node\| cancel node.data;
97		while (self.reader_queue.get()) \|node\| cancel node.data;
	94	while (self.writer_queue.get()) \|node\| resume node.data;
	95	while (self.reader_queue.get()) \|node\| resume node.data;
98	96	}
99	97
100	98	pub async fn acquireRead(self: *RwLock) HeldRead {
101		_ = @atomicRmw(usize, &self.reader_lock_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	99	_ = @atomicRmw(usize, &self.reader_lock_count, .Add, 1, .SeqCst);
102	100
103	101	suspend {
104		// TODO explicitly put this memory in the coroutine frame #1194
105	102	var my_tick_node = Loop.NextTickNode{
106		.data = @handle(),
	103	.data = @frame(),
107	104	.prev = undefined,
108	105	.next = undefined,
109	106	};
110	107
111	108	self.reader_queue.put(&my_tick_node);
112	109
113		// At this point, we are in the reader_queue, so we might have already been resumed and this coroutine
114		// frame might be destroyed. For the rest of the suspend block we cannot access the coroutine frame.
	110	// At this point, we are in the reader_queue, so we might have already been resumed.
115	111
116	112	// We set this bit so that later we can rely on the fact, that if reader_queue_empty_bit is 1,
117	113	// some actor will attempt to grab the lock.
118		_ = @atomicRmw(u8, &self.reader_queue_empty_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	114	_ = @atomicRmw(u8, &self.reader_queue_empty_bit, .Xchg, 0, .SeqCst);
119	115
120	116	// Here we don't care if we are the one to do the locking or if it was already locked for reading.
121		const have_read_lock = if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.ReadLock, AtomicOrder.SeqCst, AtomicOrder.SeqCst)) \|old_state\| old_state == State.ReadLock else true;
	117	const have_read_lock = if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.ReadLock, .SeqCst, .SeqCst)) \|old_state\| old_state == State.ReadLock else true;
122	118	if (have_read_lock) {
123	119	// Give out all the read locks.
124	120	if (self.reader_queue.get()) \|first_node\| {
...	...	@@ -134,24 +130,22 @@ pub const RwLock = struct {
134	130
135	131	pub async fn acquireWrite(self: *RwLock) HeldWrite {
136	132	suspend {
137		// TODO explicitly put this memory in the coroutine frame #1194
138	133	var my_tick_node = Loop.NextTickNode{
139		.data = @handle(),
	134	.data = @frame(),
140	135	.prev = undefined,
141	136	.next = undefined,
142	137	};
143	138
144	139	self.writer_queue.put(&my_tick_node);
145	140
146		// At this point, we are in the writer_queue, so we might have already been resumed and this coroutine
147		// frame might be destroyed. For the rest of the suspend block we cannot access the coroutine frame.
	141	// At this point, we are in the writer_queue, so we might have already been resumed.
148	142
149	143	// We set this bit so that later we can rely on the fact, that if writer_queue_empty_bit is 1,
150	144	// some actor will attempt to grab the lock.
151		_ = @atomicRmw(u8, &self.writer_queue_empty_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst);
	145	_ = @atomicRmw(u8, &self.writer_queue_empty_bit, .Xchg, 0, .SeqCst);
152	146
153	147	// Here we must be the one to acquire the write lock. It cannot already be locked.
154		if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.WriteLock, AtomicOrder.SeqCst, AtomicOrder.SeqCst) == null) {
	148	if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.WriteLock, .SeqCst, .SeqCst) == null) {
155	149	// We now have a write lock.
156	150	if (self.writer_queue.get()) \|node\| {
157	151	// Whether this node is us or someone else, we tail resume it.
...	...	@@ -169,8 +163,8 @@ pub const RwLock = struct {
169	163	// obtain the lock.
170	164	// But if there's a writer_queue item or a reader_queue item,
171	165	// we are the actor which must loop and attempt to grab the lock again.
172		if (@atomicLoad(u8, &self.writer_queue_empty_bit, AtomicOrder.SeqCst) == 0) {
173		if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.WriteLock, AtomicOrder.SeqCst, AtomicOrder.SeqCst) != null) {
	166	if (@atomicLoad(u8, &self.writer_queue_empty_bit, .SeqCst) == 0) {
	167	if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.WriteLock, .SeqCst, .SeqCst) != null) {
174	168	// We did not obtain the lock. Great, the queues are someone else's problem.
175	169	return;
176	170	}
...	...	@@ -180,13 +174,13 @@ pub const RwLock = struct {
180	174	return;
181	175	}
182	176	// Release the lock again.
183		_ = @atomicRmw(u8, &self.writer_queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
184		_ = @atomicRmw(u8, &self.shared_state, AtomicRmwOp.Xchg, State.Unlocked, AtomicOrder.SeqCst);
	177	_ = @atomicRmw(u8, &self.writer_queue_empty_bit, .Xchg, 1, .SeqCst);
	178	_ = @atomicRmw(u8, &self.shared_state, .Xchg, State.Unlocked, .SeqCst);
185	179	continue;
186	180	}
187	181
188		if (@atomicLoad(u8, &self.reader_queue_empty_bit, AtomicOrder.SeqCst) == 0) {
189		if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.ReadLock, AtomicOrder.SeqCst, AtomicOrder.SeqCst) != null) {
	182	if (@atomicLoad(u8, &self.reader_queue_empty_bit, .SeqCst) == 0) {
	183	if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.ReadLock, .SeqCst, .SeqCst) != null) {
190	184	// We did not obtain the lock. Great, the queues are someone else's problem.
191	185	return;
192	186	}
...	...	@@ -199,8 +193,8 @@ pub const RwLock = struct {
199	193	return;
200	194	}
201	195	// Release the lock again.
202		_ = @atomicRmw(u8, &self.reader_queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst);
203		if (@cmpxchgStrong(u8, &self.shared_state, State.ReadLock, State.Unlocked, AtomicOrder.SeqCst, AtomicOrder.SeqCst) != null) {
	196	_ = @atomicRmw(u8, &self.reader_queue_empty_bit, .Xchg, 1, .SeqCst);
	197	if (@cmpxchgStrong(u8, &self.shared_state, State.ReadLock, State.Unlocked, .SeqCst, .SeqCst) != null) {
204	198	// Didn't unlock. Someone else's problem.
205	199	return;
206	200	}
...	...	@@ -215,6 +209,9 @@ test "std.event.RwLock" {
215	209	// https://github.com/ziglang/zig/issues/2377
216	210	if (true) return error.SkipZigTest;
217	211
	212	// https://github.com/ziglang/zig/issues/1908
	213	if (builtin.single_threaded) return error.SkipZigTest;
	214
218	215	const allocator = std.heap.direct_allocator;
219	216
220	217	var loop: Loop = undefined;
...	...	@@ -224,8 +221,7 @@ test "std.event.RwLock" {
224	221	var lock = RwLock.init(&loop);
225	222	defer lock.deinit();
226	223
227		const handle = try async<allocator> testLock(&loop, &lock);
228		defer cancel handle;
	224	const handle = testLock(&loop, &lock);
229	225	loop.run();
230	226
231	227	const expected_result = [1]i32{shared_it_count * @intCast(i32, shared_test_data.len)} ** shared_test_data.len;
...	...	@@ -233,28 +229,31 @@ test "std.event.RwLock" {
233	229	}
234	230
235	231	async fn testLock(loop: Loop, lock: RwLock) void {
236		// TODO explicitly put next tick node memory in the coroutine frame #1194
237		suspend {
238		resume @handle();
239		}
240
241	232	var read_nodes: [100]Loop.NextTickNode = undefined;
242	233	for (read_nodes) \|*read_node\| {
243		read_node.data = async readRunner(lock) catch @panic("out of memory");
	234	const frame = loop.allocator.create(@Frame(readRunner)) catch @panic("memory");
	235	read_node.data = frame;
	236	frame.* = async readRunner(lock);
244	237	loop.onNextTick(read_node);
245	238	}
246	239
247	240	var write_nodes: [shared_it_count]Loop.NextTickNode = undefined;
248	241	for (write_nodes) \|*write_node\| {
249		write_node.data = async writeRunner(lock) catch @panic("out of memory");
	242	const frame = loop.allocator.create(@Frame(writeRunner)) catch @panic("memory");
	243	write_node.data = frame;
	244	frame.* = async writeRunner(lock);
250	245	loop.onNextTick(write_node);
251	246	}
252	247
253	248	for (write_nodes) \|*write_node\| {
254		await @ptrCast(promise->void, write_node.data);
	249	const casted = @ptrCast(*const @Frame(writeRunner), write_node.data);
	250	await casted;
	251	loop.allocator.destroy(casted);
255	252	}
256	253	for (read_nodes) \|*read_node\| {
257		await @ptrCast(promise->void, read_node.data);
	254	const casted = @ptrCast(*const @Frame(readRunner), read_node.data);
	255	await casted;
	256	loop.allocator.destroy(casted);
258	257	}
259	258	}
260	259
...	...	@@ -269,7 +268,7 @@ async fn writeRunner(lock: *RwLock) void {
269	268	var i: usize = 0;
270	269	while (i < shared_test_data.len) : (i += 1) {
271	270	std.time.sleep(100 * std.time.microsecond);
272		const lock_promise = async lock.acquireWrite() catch @panic("out of memory");
	271	const lock_promise = async lock.acquireWrite();
273	272	const handle = await lock_promise;
274	273	defer handle.release();
275	274
...	...	@@ -287,7 +286,7 @@ async fn readRunner(lock: *RwLock) void {
287	286
288	287	var i: usize = 0;
289	288	while (i < shared_test_data.len) : (i += 1) {
290		const lock_promise = async lock.acquireRead() catch @panic("out of memory");
	289	const lock_promise = async lock.acquireRead();
291	290	const handle = await lock_promise;
292	291	defer handle.release();
293	292

std/event/rwlocked.zig+1-1

...	...	@@ -3,7 +3,7 @@ const RwLock = std.event.RwLock;
3	3	const Loop = std.event.Loop;
4	4
5	5	/// Thread-safe async/await RW lock that protects one piece of data.
6		/// coroutines which are waiting for the lock are suspended, and
	6	/// Functions which are waiting for the lock are suspended, and
7	7	/// are resumed when the lock is released, in order.
8	8	pub fn RwLocked(comptime T: type) type {
9	9	return struct {

std/fmt.zig-3

...	...	@@ -328,9 +328,6 @@ pub fn formatType(
328	328	try output(context, "error.");
329	329	return output(context, @errorName(value));
330	330	},
331		.Promise => {
332		return format(context, Errors, output, "promise@{x}", @ptrToInt(value));
333		},
334	331	.Enum => {
335	332	if (comptime std.meta.trait.hasFn("format")(T)) {
336	333	return value.format(fmt, options, context, Errors, output);

std/hash/auto_hash.zig+25-24

...	...	@@ -8,31 +8,32 @@ const meta = std.meta;
8	8	pub fn autoHash(hasher: var, key: var) void {
9	9	const Key = @typeOf(key);
10	10	switch (@typeInfo(Key)) {
11		builtin.TypeId.NoReturn,
12		builtin.TypeId.Opaque,
13		builtin.TypeId.Undefined,
14		builtin.TypeId.ArgTuple,
15		builtin.TypeId.Void,
16		builtin.TypeId.Null,
17		builtin.TypeId.BoundFn,
18		builtin.TypeId.ComptimeFloat,
19		builtin.TypeId.ComptimeInt,
20		builtin.TypeId.Type,
21		builtin.TypeId.EnumLiteral,
	11	.NoReturn,
	12	.Opaque,
	13	.Undefined,
	14	.ArgTuple,
	15	.Void,
	16	.Null,
	17	.BoundFn,
	18	.ComptimeFloat,
	19	.ComptimeInt,
	20	.Type,
	21	.EnumLiteral,
	22	.Frame,
22	23	=> @compileError("cannot hash this type"),
23	24
24	25	// Help the optimizer see that hashing an int is easy by inlining!
25	26	// TODO Check if the situation is better after #561 is resolved.
26		builtin.TypeId.Int => @inlineCall(hasher.update, std.mem.asBytes(&key)),
	27	.Int => @inlineCall(hasher.update, std.mem.asBytes(&key)),
27	28
28		builtin.TypeId.Float => \|info\| autoHash(hasher, @bitCast(@IntType(false, info.bits), key)),
	29	.Float => \|info\| autoHash(hasher, @bitCast(@IntType(false, info.bits), key)),
29	30
30		builtin.TypeId.Bool => autoHash(hasher, @boolToInt(key)),
31		builtin.TypeId.Enum => autoHash(hasher, @enumToInt(key)),
32		builtin.TypeId.ErrorSet => autoHash(hasher, @errorToInt(key)),
33		builtin.TypeId.Promise, builtin.TypeId.Fn => autoHash(hasher, @ptrToInt(key)),
	31	.Bool => autoHash(hasher, @boolToInt(key)),
	32	.Enum => autoHash(hasher, @enumToInt(key)),
	33	.ErrorSet => autoHash(hasher, @errorToInt(key)),
	34	.AnyFrame, .Fn => autoHash(hasher, @ptrToInt(key)),
34	35
35		builtin.TypeId.Pointer => \|info\| switch (info.size) {
	36	.Pointer => \|info\| switch (info.size) {
36	37	builtin.TypeInfo.Pointer.Size.One,
37	38	builtin.TypeInfo.Pointer.Size.Many,
38	39	builtin.TypeInfo.Pointer.Size.C,
...	...	@@ -44,9 +45,9 @@ pub fn autoHash(hasher: var, key: var) void {
44	45	},
45	46	},
46	47
47		builtin.TypeId.Optional => if (key) \|k\| autoHash(hasher, k),
	48	.Optional => if (key) \|k\| autoHash(hasher, k),
48	49
49		builtin.TypeId.Array => {
	50	.Array => {
50	51	// TODO detect via a trait when Key has no padding bits to
51	52	// hash it as an array of bytes.
52	53	// Otherwise, hash every element.
...	...	@@ -55,7 +56,7 @@ pub fn autoHash(hasher: var, key: var) void {
55	56	}
56	57	},
57	58
58		builtin.TypeId.Vector => \|info\| {
	59	.Vector => \|info\| {
59	60	if (info.child.bit_count % 8 == 0) {
60	61	// If there's no unused bits in the child type, we can just hash
61	62	// this as an array of bytes.
...	...	@@ -71,7 +72,7 @@ pub fn autoHash(hasher: var, key: var) void {
71	72	}
72	73	},
73	74
74		builtin.TypeId.Struct => \|info\| {
	75	.Struct => \|info\| {
75	76	// TODO detect via a trait when Key has no padding bits to
76	77	// hash it as an array of bytes.
77	78	// Otherwise, hash every field.
...	...	@@ -82,7 +83,7 @@ pub fn autoHash(hasher: var, key: var) void {
82	83	}
83	84	},
84	85
85		builtin.TypeId.Union => \|info\| blk: {
	86	.Union => \|info\| blk: {
86	87	if (info.tag_type) \|tag_type\| {
87	88	const tag = meta.activeTag(key);
88	89	const s = autoHash(hasher, tag);
...	...	@@ -99,7 +100,7 @@ pub fn autoHash(hasher: var, key: var) void {
99	100	} else @compileError("cannot hash untagged union type: " ++ @typeName(Key) ++ ", provide your own hash function");
100	101	},
101	102
102		builtin.TypeId.ErrorUnion => blk: {
	103	.ErrorUnion => blk: {
103	104	const payload = key catch \|err\| {
104	105	autoHash(hasher, err);
105	106	break :blk;

std/meta.zig+1-3

...	...	@@ -104,8 +104,7 @@ pub fn Child(comptime T: type) type {
104	104	TypeId.Array => \|info\| info.child,
105	105	TypeId.Pointer => \|info\| info.child,
106	106	TypeId.Optional => \|info\| info.child,
107		TypeId.Promise => \|info\| if (info.child) \|child\| child else null,
108		else => @compileError("Expected promise, pointer, optional, or array type, " ++ "found '" ++ @typeName(T) ++ "'"),
	107	else => @compileError("Expected pointer, optional, or array type, " ++ "found '" ++ @typeName(T) ++ "'"),
109	108	};
110	109	}
111	110
...	...	@@ -114,7 +113,6 @@ test "std.meta.Child" {
114	113	testing.expect(Child(*u8) == u8);
115	114	testing.expect(Child([]u8) == u8);
116	115	testing.expect(Child(?u8) == u8);
117		testing.expect(Child(promise->u8) == u8);
118	116	}
119	117
120	118	pub fn containerLayout(comptime T: type) TypeInfo.ContainerLayout {

std/testing.zig+26-25

...	...	@@ -25,36 +25,37 @@ pub fn expectError(expected_error: anyerror, actual_error_union: var) void {
25	25	/// The types must match exactly.
26	26	pub fn expectEqual(expected: var, actual: @typeOf(expected)) void {
27	27	switch (@typeInfo(@typeOf(actual))) {
28		TypeId.NoReturn,
29		TypeId.BoundFn,
30		TypeId.ArgTuple,
31		TypeId.Opaque,
	28	.NoReturn,
	29	.BoundFn,
	30	.ArgTuple,
	31	.Opaque,
	32	.Frame,
	33	.AnyFrame,
32	34	=> @compileError("value of type " ++ @typeName(@typeOf(actual)) ++ " encountered"),
33	35
34		TypeId.Undefined,
35		TypeId.Null,
36		TypeId.Void,
	36	.Undefined,
	37	.Null,
	38	.Void,
37	39	=> return,
38	40
39		TypeId.Type,
40		TypeId.Bool,
41		TypeId.Int,
42		TypeId.Float,
43		TypeId.ComptimeFloat,
44		TypeId.ComptimeInt,
45		TypeId.EnumLiteral,
46		TypeId.Enum,
47		TypeId.Fn,
48		TypeId.Promise,
49		TypeId.Vector,
50		TypeId.ErrorSet,
	41	.Type,
	42	.Bool,
	43	.Int,
	44	.Float,
	45	.ComptimeFloat,
	46	.ComptimeInt,
	47	.EnumLiteral,
	48	.Enum,
	49	.Fn,
	50	.Vector,
	51	.ErrorSet,
51	52	=> {
52	53	if (actual != expected) {
53	54	std.debug.panic("expected {}, found {}", expected, actual);
54	55	}
55	56	},
56	57
57		TypeId.Pointer => \|pointer\| {
	58	.Pointer => \|pointer\| {
58	59	switch (pointer.size) {
59	60	builtin.TypeInfo.Pointer.Size.One,
60	61	builtin.TypeInfo.Pointer.Size.Many,
...	...	@@ -76,22 +77,22 @@ pub fn expectEqual(expected: var, actual: @typeOf(expected)) void {
76	77	}
77	78	},
78	79
79		TypeId.Array => \|array\| expectEqualSlices(array.child, &expected, &actual),
	80	.Array => \|array\| expectEqualSlices(array.child, &expected, &actual),
80	81
81		TypeId.Struct => \|structType\| {
	82	.Struct => \|structType\| {
82	83	inline for (structType.fields) \|field\| {
83	84	expectEqual(@field(expected, field.name), @field(actual, field.name));
84	85	}
85	86	},
86	87
87		TypeId.Union => \|union_info\| {
	88	.Union => \|union_info\| {
88	89	if (union_info.tag_type == null) {
89	90	@compileError("Unable to compare untagged union values");
90	91	}
91	92	@compileError("TODO implement testing.expectEqual for tagged unions");
92	93	},
93	94
94		TypeId.Optional => {
	95	.Optional => {
95	96	if (expected) \|expected_payload\| {
96	97	if (actual) \|actual_payload\| {
97	98	expectEqual(expected_payload, actual_payload);
...	...	@@ -105,7 +106,7 @@ pub fn expectEqual(expected: var, actual: @typeOf(expected)) void {
105	106	}
106	107	},
107	108
108		TypeId.ErrorUnion => {
	109	.ErrorUnion => {
109	110	if (expected) \|expected_payload\| {
110	111	if (actual) \|actual_payload\| {
111	112	expectEqual(expected_payload, actual_payload);

std/zig/ast.zig+8-8

...	...	@@ -400,7 +400,7 @@ pub const Node = struct {
400	400	VarType,
401	401	ErrorType,
402	402	FnProto,
403		PromiseType,
	403	AnyFrameType,
404	404
405	405	// Primary expressions
406	406	IntegerLiteral,
...	...	@@ -952,9 +952,9 @@ pub const Node = struct {
952	952	}
953	953	};
954	954
955		pub const PromiseType = struct {
	955	pub const AnyFrameType = struct {
956	956	base: Node,
957		promise_token: TokenIndex,
	957	anyframe_token: TokenIndex,
958	958	result: ?Result,
959	959
960	960	pub const Result = struct {
...	...	@@ -962,7 +962,7 @@ pub const Node = struct {
962	962	return_type: *Node,
963	963	};
964	964
965		pub fn iterate(self: PromiseType, index: usize) ?Node {
	965	pub fn iterate(self: AnyFrameType, index: usize) ?Node {
966	966	var i = index;
967	967
968	968	if (self.result) \|result\| {
...	...	@@ -973,13 +973,13 @@ pub const Node = struct {
973	973	return null;
974	974	}
975	975
976		pub fn firstToken(self: *const PromiseType) TokenIndex {
977		return self.promise_token;
	976	pub fn firstToken(self: *const AnyFrameType) TokenIndex {
	977	return self.anyframe_token;
978	978	}
979	979
980		pub fn lastToken(self: *const PromiseType) TokenIndex {
	980	pub fn lastToken(self: *const AnyFrameType) TokenIndex {
981	981	if (self.result) \|result\| return result.return_type.lastToken();
982		return self.promise_token;
	982	return self.anyframe_token;
983	983	}
984	984	};
985	985

std/zig/parse.zig+20-35

...	...	@@ -814,7 +814,6 @@ fn parsePrefixExpr(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {
814	814	/// <- AsmExpr
815	815	/// / IfExpr
816	816	/// / KEYWORD_break BreakLabel? Expr?
817		/// / KEYWORD_cancel Expr
818	817	/// / KEYWORD_comptime Expr
819	818	/// / KEYWORD_continue BreakLabel?
820	819	/// / KEYWORD_resume Expr
...	...	@@ -839,20 +838,6 @@ fn parsePrimaryExpr(arena: Allocator, it: TokenIterator, tree: Tree) !?Node
839	838	return &node.base;
840	839	}
841	840
842		if (eatToken(it, .Keyword_cancel)) \|token\| {
843		const expr_node = try expectNode(arena, it, tree, parseExpr, AstError{
844		.ExpectedExpr = AstError.ExpectedExpr{ .token = it.index },
845		});
846		const node = try arena.create(Node.PrefixOp);
847		node.* = Node.PrefixOp{
848		.base = Node{ .id = .PrefixOp },
849		.op_token = token,
850		.op = Node.PrefixOp.Op.Cancel,
851		.rhs = expr_node,
852		};
853		return &node.base;
854		}
855
856	841	if (eatToken(it, .Keyword_comptime)) \|token\| {
857	842	const expr_node = try expectNode(arena, it, tree, parseExpr, AstError{
858	843	.ExpectedExpr = AstError.ExpectedExpr{ .token = it.index },
...	...	@@ -1201,7 +1186,7 @@ fn parseSuffixExpr(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {
1201	1186	/// / KEYWORD_error DOT IDENTIFIER
1202	1187	/// / KEYWORD_false
1203	1188	/// / KEYWORD_null
1204		/// / KEYWORD_promise
	1189	/// / KEYWORD_anyframe
1205	1190	/// / KEYWORD_true
1206	1191	/// / KEYWORD_undefined
1207	1192	/// / KEYWORD_unreachable
...	...	@@ -1256,11 +1241,11 @@ fn parsePrimaryTypeExpr(arena: Allocator, it: TokenIterator, tree: Tree) !?N
1256	1241	}
1257	1242	if (eatToken(it, .Keyword_false)) \|token\| return createLiteral(arena, Node.BoolLiteral, token);
1258	1243	if (eatToken(it, .Keyword_null)) \|token\| return createLiteral(arena, Node.NullLiteral, token);
1259		if (eatToken(it, .Keyword_promise)) \|token\| {
1260		const node = try arena.create(Node.PromiseType);
1261		node.* = Node.PromiseType{
1262		.base = Node{ .id = .PromiseType },
1263		.promise_token = token,
	1244	if (eatToken(it, .Keyword_anyframe)) \|token\| {
	1245	const node = try arena.create(Node.AnyFrameType);
	1246	node.* = Node.AnyFrameType{
	1247	.base = Node{ .id = .AnyFrameType },
	1248	.anyframe_token = token,
1264	1249	.result = null,
1265	1250	};
1266	1251	return &node.base;
...	...	@@ -2194,7 +2179,7 @@ fn parsePrefixOp(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {
2194	2179
2195	2180	/// PrefixTypeOp
2196	2181	/// <- QUESTIONMARK
2197		/// / KEYWORD_promise MINUSRARROW
	2182	/// / KEYWORD_anyframe MINUSRARROW
2198	2183	/// / ArrayTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
2199	2184	/// / PtrTypeStart (KEYWORD_align LPAREN Expr (COLON INTEGER COLON INTEGER)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
2200	2185	fn parsePrefixTypeOp(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {
...	...	@@ -2209,20 +2194,20 @@ fn parsePrefixTypeOp(arena: Allocator, it: TokenIterator, tree: Tree) !?Node
2209	2194	return &node.base;
2210	2195	}
2211	2196
2212		// TODO: Returning a PromiseType instead of PrefixOp makes casting and setting .rhs or
	2197	// TODO: Returning a AnyFrameType instead of PrefixOp makes casting and setting .rhs or
2213	2198	// .return_type more difficult for the caller (see parsePrefixOpExpr helper).
2214		// Consider making the PromiseType a member of PrefixOp and add a
2215		// PrefixOp.PromiseType variant?
2216		if (eatToken(it, .Keyword_promise)) \|token\| {
	2199	// Consider making the AnyFrameType a member of PrefixOp and add a
	2200	// PrefixOp.AnyFrameType variant?
	2201	if (eatToken(it, .Keyword_anyframe)) \|token\| {
2217	2202	const arrow = eatToken(it, .Arrow) orelse {
2218	2203	putBackToken(it, token);
2219	2204	return null;
2220	2205	};
2221		const node = try arena.create(Node.PromiseType);
2222		node.* = Node.PromiseType{
2223		.base = Node{ .id = .PromiseType },
2224		.promise_token = token,
2225		.result = Node.PromiseType.Result{
	2206	const node = try arena.create(Node.AnyFrameType);
	2207	node.* = Node.AnyFrameType{
	2208	.base = Node{ .id = .AnyFrameType },
	2209	.anyframe_token = token,
	2210	.result = Node.AnyFrameType.Result{
2226	2211	.arrow_token = arrow,
2227	2212	.return_type = undefined, // set by caller
2228	2213	},
...	...	@@ -2903,8 +2888,8 @@ fn parsePrefixOpExpr(
2903	2888	rightmost_op = rhs;
2904	2889	} else break;
2905	2890	},
2906		.PromiseType => {
2907		const prom = rightmost_op.cast(Node.PromiseType).?;
	2891	.AnyFrameType => {
	2892	const prom = rightmost_op.cast(Node.AnyFrameType).?;
2908	2893	if (try opParseFn(arena, it, tree)) \|rhs\| {
2909	2894	prom.result.?.return_type = rhs;
2910	2895	rightmost_op = rhs;
...	...	@@ -2922,8 +2907,8 @@ fn parsePrefixOpExpr(
2922	2907	.InvalidToken = AstError.InvalidToken{ .token = it.index },
2923	2908	});
2924	2909	},
2925		.PromiseType => {
2926		const prom = rightmost_op.cast(Node.PromiseType).?;
	2910	.AnyFrameType => {
	2911	const prom = rightmost_op.cast(Node.AnyFrameType).?;
2927	2912	prom.result.?.return_type = try expectNode(arena, it, tree, childParseFn, AstError{
2928	2913	.InvalidToken = AstError.InvalidToken{ .token = it.index },
2929	2914	});

std/zig/parser_test.zig+6-6

...	...	@@ -1183,7 +1183,7 @@ test "zig fmt: resume from suspend block" {
1183	1183	try testCanonical(
1184	1184	\\fn foo() void {
1185	1185	\\ suspend {
1186		\\ resume @handle();
	1186	\\ resume @frame();
1187	1187	\\ }
1188	1188	\\}
1189	1189	\\
...	...	@@ -2103,7 +2103,7 @@ test "zig fmt: inline asm" {
2103	2103	);
2104	2104	}
2105	2105
2106		test "zig fmt: coroutines" {
	2106	test "zig fmt: async functions" {
2107	2107	try testCanonical(
2108	2108	\\async fn simpleAsyncFn() void {
2109	2109	\\ const a = async a.b();
...	...	@@ -2111,14 +2111,14 @@ test "zig fmt: coroutines" {
2111	2111	\\ suspend;
2112	2112	\\ x += 1;
2113	2113	\\ suspend;
2114		\\ const p: promise->void = async simpleAsyncFn() catch unreachable;
	2114	\\ const p: anyframe->void = async simpleAsyncFn() catch unreachable;
2115	2115	\\ await p;
2116	2116	\\}
2117	2117	\\
2118		\\test "coroutine suspend, resume, cancel" {
2119		\\ const p: promise = try async<std.debug.global_allocator> testAsyncSeq();
	2118	\\test "suspend, resume, await" {
	2119	\\ const p: anyframe = async testAsyncSeq();
2120	2120	\\ resume p;
2121		\\ cancel p;
	2121	\\ await p;
2122	2122	\\}
2123	2123	\\
2124	2124	);

std/zig/render.zig+5-5

...	...	@@ -1205,15 +1205,15 @@ fn renderExpression(
1205	1205	}
1206	1206	},
1207	1207
1208		ast.Node.Id.PromiseType => {
1209		const promise_type = @fieldParentPtr(ast.Node.PromiseType, "base", base);
	1208	ast.Node.Id.AnyFrameType => {
	1209	const anyframe_type = @fieldParentPtr(ast.Node.AnyFrameType, "base", base);
1210	1210
1211		if (promise_type.result) \|result\| {
1212		try renderToken(tree, stream, promise_type.promise_token, indent, start_col, Space.None); // promise
	1211	if (anyframe_type.result) \|result\| {
	1212	try renderToken(tree, stream, anyframe_type.anyframe_token, indent, start_col, Space.None); // anyframe
1213	1213	try renderToken(tree, stream, result.arrow_token, indent, start_col, Space.None); // ->
1214	1214	return renderExpression(allocator, stream, tree, indent, start_col, result.return_type, space);
1215	1215	} else {
1216		return renderToken(tree, stream, promise_type.promise_token, indent, start_col, space); // promise
	1216	return renderToken(tree, stream, anyframe_type.anyframe_token, indent, start_col, space); // anyframe
1217	1217	}
1218	1218	},
1219	1219

std/zig/tokenizer.zig+2-4

...	...	@@ -15,12 +15,12 @@ pub const Token = struct {
15	15	Keyword{ .bytes = "align", .id = Id.Keyword_align },
16	16	Keyword{ .bytes = "allowzero", .id = Id.Keyword_allowzero },
17	17	Keyword{ .bytes = "and", .id = Id.Keyword_and },
	18	Keyword{ .bytes = "anyframe", .id = Id.Keyword_anyframe },
18	19	Keyword{ .bytes = "asm", .id = Id.Keyword_asm },
19	20	Keyword{ .bytes = "async", .id = Id.Keyword_async },
20	21	Keyword{ .bytes = "await", .id = Id.Keyword_await },
21	22	Keyword{ .bytes = "break", .id = Id.Keyword_break },
22	23	Keyword{ .bytes = "catch", .id = Id.Keyword_catch },
23		Keyword{ .bytes = "cancel", .id = Id.Keyword_cancel },
24	24	Keyword{ .bytes = "comptime", .id = Id.Keyword_comptime },
25	25	Keyword{ .bytes = "const", .id = Id.Keyword_const },
26	26	Keyword{ .bytes = "continue", .id = Id.Keyword_continue },
...	...	@@ -42,7 +42,6 @@ pub const Token = struct {
42	42	Keyword{ .bytes = "or", .id = Id.Keyword_or },
43	43	Keyword{ .bytes = "orelse", .id = Id.Keyword_orelse },
44	44	Keyword{ .bytes = "packed", .id = Id.Keyword_packed },
45		Keyword{ .bytes = "promise", .id = Id.Keyword_promise },
46	45	Keyword{ .bytes = "pub", .id = Id.Keyword_pub },
47	46	Keyword{ .bytes = "resume", .id = Id.Keyword_resume },
48	47	Keyword{ .bytes = "return", .id = Id.Keyword_return },
...	...	@@ -151,7 +150,6 @@ pub const Token = struct {
151	150	Keyword_async,
152	151	Keyword_await,
153	152	Keyword_break,
154		Keyword_cancel,
155	153	Keyword_catch,
156	154	Keyword_comptime,
157	155	Keyword_const,
...	...	@@ -174,7 +172,7 @@ pub const Token = struct {
174	172	Keyword_or,
175	173	Keyword_orelse,
176	174	Keyword_packed,
177		Keyword_promise,
	175	Keyword_anyframe,
178	176	Keyword_pub,
179	177	Keyword_resume,
180	178	Keyword_return,

test/compile_errors.zig+122-27

...	...	@@ -2,6 +2,118 @@ const tests = @import("tests.zig");
2	2	const builtin = @import("builtin");
3	3
4	4	pub fn addCases(cases: *tests.CompileErrorContext) void {
	5	cases.add(
	6	"@frame() causes function to be async",
	7	\\export fn entry() void {
	8	\\ func();
	9	\\}
	10	\\fn func() void {
	11	\\ _ = @frame();
	12	\\}
	13	,
	14	"tmp.zig:1:1: error: function with calling convention 'ccc' cannot be async",
	15	"tmp.zig:5:9: note: @frame() causes function to be async",
	16	);
	17	cases.add(
	18	"invalid suspend in exported function",
	19	\\export fn entry() void {
	20	\\ var frame = async func();
	21	\\ var result = await frame;
	22	\\}
	23	\\fn func() void {
	24	\\ suspend;
	25	\\}
	26	,
	27	"tmp.zig:1:1: error: function with calling convention 'ccc' cannot be async",
	28	"tmp.zig:3:18: note: await is a suspend point",
	29	);
	30
	31	cases.add(
	32	"async function indirectly depends on its own frame",
	33	\\export fn entry() void {
	34	\\ _ = async amain();
	35	\\}
	36	\\async fn amain() void {
	37	\\ other();
	38	\\}
	39	\\fn other() void {
	40	\\ var x: [@sizeOf(@Frame(amain))]u8 = undefined;
	41	\\}
	42	,
	43	"tmp.zig:4:1: error: unable to determine async function frame of 'amain'",
	44	"tmp.zig:5:10: note: analysis of function 'other' depends on the frame",
	45	"tmp.zig:8:13: note: depends on the frame here",
	46	);
	47
	48	cases.add(
	49	"async function depends on its own frame",
	50	\\export fn entry() void {
	51	\\ _ = async amain();
	52	\\}
	53	\\async fn amain() void {
	54	\\ var x: [@sizeOf(@Frame(amain))]u8 = undefined;
	55	\\}
	56	,
	57	"tmp.zig:4:1: error: cannot resolve '@Frame(amain)': function not fully analyzed yet",
	58	"tmp.zig:5:13: note: depends on its own frame here",
	59	);
	60
	61	cases.add(
	62	"non async function pointer passed to @asyncCall",
	63	\\export fn entry() void {
	64	\\ var ptr = afunc;
	65	\\ var bytes: [100]u8 = undefined;
	66	\\ _ = @asyncCall(&bytes, {}, ptr);
	67	\\}
	68	\\fn afunc() void { }
	69	,
	70	"tmp.zig:4:32: error: expected async function, found 'fn() void'",
	71	);
	72
	73	cases.add(
	74	"runtime-known async function called",
	75	\\export fn entry() void {
	76	\\ _ = async amain();
	77	\\}
	78	\\fn amain() void {
	79	\\ var ptr = afunc;
	80	\\ _ = ptr();
	81	\\}
	82	\\async fn afunc() void {}
	83	,
	84	"tmp.zig:6:12: error: function is not comptime-known; @asyncCall required",
	85	);
	86
	87	cases.add(
	88	"runtime-known function called with async keyword",
	89	\\export fn entry() void {
	90	\\ var ptr = afunc;
	91	\\ _ = async ptr();
	92	\\}
	93	\\
	94	\\async fn afunc() void { }
	95	,
	96	"tmp.zig:3:15: error: function is not comptime-known; @asyncCall required",
	97	);
	98
	99	cases.add(
	100	"function with ccc indirectly calling async function",
	101	\\export fn entry() void {
	102	\\ foo();
	103	\\}
	104	\\fn foo() void {
	105	\\ bar();
	106	\\}
	107	\\fn bar() void {
	108	\\ suspend;
	109	\\}
	110	,
	111	"tmp.zig:1:1: error: function with calling convention 'ccc' cannot be async",
	112	"tmp.zig:2:8: note: async function call here",
	113	"tmp.zig:5:8: note: async function call here",
	114	"tmp.zig:8:5: note: suspends here",
	115	);
	116
5	117	cases.add(
6	118	"capture group on switch prong with incompatible payload types",
7	119	\\const Union = union(enum) {
...	...	@@ -1319,24 +1431,14 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
1319	1431	);
1320	1432
1321	1433	cases.add(
1322		"@handle() called outside of function definition",
1323		\\var handle_undef: promise = undefined;
1324		\\var handle_dummy: promise = @handle();
	1434	"@frame() called outside of function definition",
	1435	\\var handle_undef: anyframe = undefined;
	1436	\\var handle_dummy: anyframe = @frame();
1325	1437	\\export fn entry() bool {
1326	1438	\\ return handle_undef == handle_dummy;
1327	1439	\\}
1328	1440	,
1329		"tmp.zig:2:29: error: @handle() called outside of function definition",
1330		);
1331
1332		cases.add(
1333		"@handle() in non-async function",
1334		\\export fn entry() bool {
1335		\\ var handle_undef: promise = undefined;
1336		\\ return handle_undef == @handle();
1337		\\}
1338		,
1339		"tmp.zig:3:28: error: @handle() in non-async function",
	1441	"tmp.zig:2:30: error: @frame() called outside of function definition",
1340	1442	);
1341	1443
1342	1444	cases.add(
...	...	@@ -1712,15 +1814,9 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
1712	1814
1713	1815	cases.add(
1714	1816	"suspend inside suspend block",
1715		\\const std = @import("std",);
1716		\\
1717	1817	\\export fn entry() void {
1718		\\ var buf: [500]u8 = undefined;
1719		\\ var a = &std.heap.FixedBufferAllocator.init(buf[0..]).allocator;
1720		\\ const p = (async<a> foo()) catch unreachable;
1721		\\ cancel p;
	1818	\\ _ = async foo();
1722	1819	\\}
1723		\\
1724	1820	\\async fn foo() void {
1725	1821	\\ suspend {
1726	1822	\\ suspend {
...	...	@@ -1728,8 +1824,8 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
1728	1824	\\ }
1729	1825	\\}
1730	1826	,
1731		"tmp.zig:12:9: error: cannot suspend inside suspend block",
1732		"tmp.zig:11:5: note: other suspend block here",
	1827	"tmp.zig:6:9: error: cannot suspend inside suspend block",
	1828	"tmp.zig:5:5: note: other suspend block here",
1733	1829	);
1734	1830
1735	1831	cases.add(
...	...	@@ -1770,15 +1866,14 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
1770	1866
1771	1867	cases.add(
1772	1868	"returning error from void async function",
1773		\\const std = @import("std",);
1774	1869	\\export fn entry() void {
1775		\\ const p = async<std.debug.global_allocator> amain() catch unreachable;
	1870	\\ _ = async amain();
1776	1871	\\}
1777	1872	\\async fn amain() void {
1778	1873	\\ return error.ShouldBeCompileError;
1779	1874	\\}
1780	1875	,
1781		"tmp.zig:6:17: error: expected type 'void', found 'error{ShouldBeCompileError}'",
	1876	"tmp.zig:5:17: error: expected type 'void', found 'error{ShouldBeCompileError}'",
1782	1877	);
1783	1878
1784	1879	cases.add(
...	...	@@ -3307,7 +3402,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
3307	3402	\\
3308	3403	\\export fn entry() usize { return @sizeOf(@typeOf(Foo)); }
3309	3404	,
3310		"tmp.zig:5:18: error: unable to evaluate constant expression",
	3405	"tmp.zig:5:25: error: unable to evaluate constant expression",
3311	3406	"tmp.zig:2:12: note: called from here",
3312	3407	"tmp.zig:2:8: note: called from here",
3313	3408	);

test/runtime_safety.zig+96-5

...	...	@@ -1,6 +1,91 @@
1	1	const tests = @import("tests.zig");
2	2
3	3	pub fn addCases(cases: *tests.CompareOutputContext) void {
	4	cases.addRuntimeSafety("awaiting twice",
	5	\\pub fn panic(message: []const u8, stack_trace: ?*@import("builtin").StackTrace) noreturn {
	6	\\ @import("std").os.exit(126);
	7	\\}
	8	\\var frame: anyframe = undefined;
	9	\\
	10	\\pub fn main() void {
	11	\\ _ = async amain();
	12	\\ resume frame;
	13	\\}
	14	\\
	15	\\fn amain() void {
	16	\\ var f = async func();
	17	\\ await f;
	18	\\ await f;
	19	\\}
	20	\\
	21	\\fn func() void {
	22	\\ suspend {
	23	\\ frame = @frame();
	24	\\ }
	25	\\}
	26	);
	27
	28	cases.addRuntimeSafety("@asyncCall with too small a frame",
	29	\\pub fn panic(message: []const u8, stack_trace: ?*@import("builtin").StackTrace) noreturn {
	30	\\ @import("std").os.exit(126);
	31	\\}
	32	\\pub fn main() void {
	33	\\ var bytes: [1]u8 = undefined;
	34	\\ var ptr = other;
	35	\\ var frame = @asyncCall(&bytes, {}, ptr);
	36	\\}
	37	\\async fn other() void {
	38	\\ suspend;
	39	\\}
	40	);
	41
	42	cases.addRuntimeSafety("resuming a function which is awaiting a frame",
	43	\\pub fn panic(message: []const u8, stack_trace: ?*@import("builtin").StackTrace) noreturn {
	44	\\ @import("std").os.exit(126);
	45	\\}
	46	\\pub fn main() void {
	47	\\ var frame = async first();
	48	\\ resume frame;
	49	\\}
	50	\\fn first() void {
	51	\\ var frame = async other();
	52	\\ await frame;
	53	\\}
	54	\\fn other() void {
	55	\\ suspend;
	56	\\}
	57	);
	58
	59	cases.addRuntimeSafety("resuming a function which is awaiting a call",
	60	\\pub fn panic(message: []const u8, stack_trace: ?*@import("builtin").StackTrace) noreturn {
	61	\\ @import("std").os.exit(126);
	62	\\}
	63	\\pub fn main() void {
	64	\\ var frame = async first();
	65	\\ resume frame;
	66	\\}
	67	\\fn first() void {
	68	\\ other();
	69	\\}
	70	\\fn other() void {
	71	\\ suspend;
	72	\\}
	73	);
	74
	75	cases.addRuntimeSafety("invalid resume of async function",
	76	\\pub fn panic(message: []const u8, stack_trace: ?*@import("builtin").StackTrace) noreturn {
	77	\\ @import("std").os.exit(126);
	78	\\}
	79	\\pub fn main() void {
	80	\\ var p = async suspendOnce();
	81	\\ resume p; //ok
	82	\\ resume p; //bad
	83	\\}
	84	\\fn suspendOnce() void {
	85	\\ suspend;
	86	\\}
	87	);
	88
4	89	cases.addRuntimeSafety(".? operator on null pointer",
5	90	\\pub fn panic(message: []const u8, stack_trace: ?*@import("builtin").StackTrace) noreturn {
6	91	\\ @import("std").os.exit(126);
...	...	@@ -483,23 +568,29 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
483	568	\\ std.os.exit(126);
484	569	\\}
485	570	\\
	571	\\var failing_frame: @Frame(failing) = undefined;
	572	\\
486	573	\\pub fn main() void {
487	574	\\ const p = nonFailing();
488	575	\\ resume p;
489		\\ const p2 = async<std.debug.global_allocator> printTrace(p) catch unreachable;
490		\\ cancel p2;
	576	\\ const p2 = async printTrace(p);
491	577	\\}
492	578	\\
493		\\fn nonFailing() promise->anyerror!void {
494		\\ return async<std.debug.global_allocator> failing() catch unreachable;
	579	\\fn nonFailing() anyframe->anyerror!void {
	580	\\ failing_frame = async failing();
	581	\\ return &failing_frame;
495	582	\\}
496	583	\\
497	584	\\async fn failing() anyerror!void {
498	585	\\ suspend;
	586	\\ return second();
	587	\\}
	588	\\
	589	\\async fn second() anyerror!void {
499	590	\\ return error.Fail;
500	591	\\}
501	592	\\
502		\\async fn printTrace(p: promise->anyerror!void) void {
	593	\\async fn printTrace(p: anyframe->anyerror!void) void {
503	594	\\ (await p) catch unreachable;
504	595	\\}
505	596	);

test/stage1/behavior.zig+6-7

...	...	@@ -3,12 +3,13 @@ comptime {
3	3	_ = @import("behavior/alignof.zig");
4	4	_ = @import("behavior/array.zig");
5	5	_ = @import("behavior/asm.zig");
	6	_ = @import("behavior/async_fn.zig");
6	7	_ = @import("behavior/atomics.zig");
	8	_ = @import("behavior/await_struct.zig");
7	9	_ = @import("behavior/bit_shifting.zig");
8	10	_ = @import("behavior/bitcast.zig");
9	11	_ = @import("behavior/bitreverse.zig");
10	12	_ = @import("behavior/bool.zig");
11		_ = @import("behavior/byteswap.zig");
12	13	_ = @import("behavior/bugs/1025.zig");
13	14	_ = @import("behavior/bugs/1076.zig");
14	15	_ = @import("behavior/bugs/1111.zig");
...	...	@@ -38,23 +39,23 @@ comptime {
38	39	_ = @import("behavior/bugs/726.zig");
39	40	_ = @import("behavior/bugs/828.zig");
40	41	_ = @import("behavior/bugs/920.zig");
	42	_ = @import("behavior/byteswap.zig");
41	43	_ = @import("behavior/byval_arg_var.zig");
42		_ = @import("behavior/cancel.zig");
43	44	_ = @import("behavior/cast.zig");
44	45	_ = @import("behavior/const_slice_child.zig");
45		_ = @import("behavior/coroutine_await_struct.zig");
46		_ = @import("behavior/coroutines.zig");
47	46	_ = @import("behavior/defer.zig");
48	47	_ = @import("behavior/enum.zig");
49	48	_ = @import("behavior/enum_with_members.zig");
50	49	_ = @import("behavior/error.zig");
51	50	_ = @import("behavior/eval.zig");
52	51	_ = @import("behavior/field_parent_ptr.zig");
	52	_ = @import("behavior/floatop.zig");
53	53	_ = @import("behavior/fn.zig");
54	54	_ = @import("behavior/fn_in_struct_in_comptime.zig");
55	55	_ = @import("behavior/for.zig");
56	56	_ = @import("behavior/generics.zig");
57	57	_ = @import("behavior/hasdecl.zig");
	58	_ = @import("behavior/hasfield.zig");
58	59	_ = @import("behavior/if.zig");
59	60	_ = @import("behavior/import.zig");
60	61	_ = @import("behavior/incomplete_struct_param_tld.zig");
...	...	@@ -63,14 +64,13 @@ comptime {
63	64	_ = @import("behavior/math.zig");
64	65	_ = @import("behavior/merge_error_sets.zig");
65	66	_ = @import("behavior/misc.zig");
	67	_ = @import("behavior/muladd.zig");
66	68	_ = @import("behavior/namespace_depends_on_compile_var.zig");
67	69	_ = @import("behavior/new_stack_call.zig");
68	70	_ = @import("behavior/null.zig");
69	71	_ = @import("behavior/optional.zig");
70	72	_ = @import("behavior/pointers.zig");
71	73	_ = @import("behavior/popcount.zig");
72		_ = @import("behavior/muladd.zig");
73		_ = @import("behavior/floatop.zig");
74	74	_ = @import("behavior/ptrcast.zig");
75	75	_ = @import("behavior/pub_enum.zig");
76	76	_ = @import("behavior/ref_var_in_if_after_if_2nd_switch_prong.zig");
...	...	@@ -99,5 +99,4 @@ comptime {
99	99	_ = @import("behavior/void.zig");
100	100	_ = @import("behavior/while.zig");
101	101	_ = @import("behavior/widening.zig");
102		_ = @import("behavior/hasfield.zig");
103	102	}

test/stage1/behavior/align.zig+62

...	...	@@ -228,3 +228,65 @@ test "alignment of extern() void" {
228	228	}
229	229
230	230	extern fn nothing() void {}
	231
	232	test "return error union with 128-bit integer" {
	233	expect(3 == try give());
	234	}
	235	fn give() anyerror!u128 {
	236	return 3;
	237	}
	238
	239	test "alignment of >= 128-bit integer type" {
	240	expect(@alignOf(u128) == 16);
	241	expect(@alignOf(u129) == 16);
	242	}
	243
	244	test "alignment of struct with 128-bit field" {
	245	expect(@alignOf(struct {
	246	x: u128,
	247	}) == 16);
	248
	249	comptime {
	250	expect(@alignOf(struct {
	251	x: u128,
	252	}) == 16);
	253	}
	254	}
	255
	256	test "size of extern struct with 128-bit field" {
	257	expect(@sizeOf(extern struct {
	258	x: u128,
	259	y: u8,
	260	}) == 32);
	261
	262	comptime {
	263	expect(@sizeOf(extern struct {
	264	x: u128,
	265	y: u8,
	266	}) == 32);
	267	}
	268	}
	269
	270	const DefaultAligned = struct {
	271	nevermind: u32,
	272	badguy: i128,
	273	};
	274
	275	test "read 128-bit field from default aligned struct in stack memory" {
	276	var default_aligned = DefaultAligned{
	277	.nevermind = 1,
	278	.badguy = 12,
	279	};
	280	expect((@ptrToInt(&default_aligned.badguy) % 16) == 0);
	281	expect(12 == default_aligned.badguy);
	282	}
	283
	284	var default_aligned_global = DefaultAligned{
	285	.nevermind = 1,
	286	.badguy = 12,
	287	};
	288
	289	test "read 128-bit field from default aligned struct in global memory" {
	290	expect((@ptrToInt(&default_aligned_global.badguy) % 16) == 0);
	291	expect(12 == default_aligned_global.badguy);
	292	}

test/stage1/behavior/async_fn.zig created+736

...	...	@@ -0,0 +1,736 @@
	1	const std = @import("std");
	2	const builtin = @import("builtin");
	3	const expect = std.testing.expect;
	4	const expectEqual = std.testing.expectEqual;
	5
	6	var global_x: i32 = 1;
	7
	8	test "simple coroutine suspend and resume" {
	9	const frame = async simpleAsyncFn();
	10	expect(global_x == 2);
	11	resume frame;
	12	expect(global_x == 3);
	13	const af: anyframe->void = &frame;
	14	resume frame;
	15	expect(global_x == 4);
	16	}
	17	fn simpleAsyncFn() void {
	18	global_x += 1;
	19	suspend;
	20	global_x += 1;
	21	suspend;
	22	global_x += 1;
	23	}
	24
	25	var global_y: i32 = 1;
	26
	27	test "pass parameter to coroutine" {
	28	const p = async simpleAsyncFnWithArg(2);
	29	expect(global_y == 3);
	30	resume p;
	31	expect(global_y == 5);
	32	}
	33	fn simpleAsyncFnWithArg(delta: i32) void {
	34	global_y += delta;
	35	suspend;
	36	global_y += delta;
	37	}
	38
	39	test "suspend at end of function" {
	40	const S = struct {
	41	var x: i32 = 1;
	42
	43	fn doTheTest() void {
	44	expect(x == 1);
	45	const p = async suspendAtEnd();
	46	expect(x == 2);
	47	}
	48
	49	fn suspendAtEnd() void {
	50	x += 1;
	51	suspend;
	52	}
	53	};
	54	S.doTheTest();
	55	}
	56
	57	test "local variable in async function" {
	58	const S = struct {
	59	var x: i32 = 0;
	60
	61	fn doTheTest() void {
	62	expect(x == 0);
	63	const p = async add(1, 2);
	64	expect(x == 0);
	65	resume p;
	66	expect(x == 0);
	67	resume p;
	68	expect(x == 0);
	69	resume p;
	70	expect(x == 3);
	71	}
	72
	73	fn add(a: i32, b: i32) void {
	74	var accum: i32 = 0;
	75	suspend;
	76	accum += a;
	77	suspend;
	78	accum += b;
	79	suspend;
	80	x = accum;
	81	}
	82	};
	83	S.doTheTest();
	84	}
	85
	86	test "calling an inferred async function" {
	87	const S = struct {
	88	var x: i32 = 1;
	89	var other_frame: *@Frame(other) = undefined;
	90
	91	fn doTheTest() void {
	92	_ = async first();
	93	expect(x == 1);
	94	resume other_frame.*;
	95	expect(x == 2);
	96	}
	97
	98	fn first() void {
	99	other();
	100	}
	101	fn other() void {
	102	other_frame = @frame();
	103	suspend;
	104	x += 1;
	105	}
	106	};
	107	S.doTheTest();
	108	}
	109
	110	test "@frameSize" {
	111	const S = struct {
	112	fn doTheTest() void {
	113	{
	114	var ptr = @ptrCast(async fn(i32) void, other);
	115	const size = @frameSize(ptr);
	116	expect(size == @sizeOf(@Frame(other)));
	117	}
	118	{
	119	var ptr = @ptrCast(async fn() void, first);
	120	const size = @frameSize(ptr);
	121	expect(size == @sizeOf(@Frame(first)));
	122	}
	123	}
	124
	125	fn first() void {
	126	other(1);
	127	}
	128	fn other(param: i32) void {
	129	var local: i32 = undefined;
	130	suspend;
	131	}
	132	};
	133	S.doTheTest();
	134	}
	135
	136	test "coroutine suspend, resume" {
	137	const S = struct {
	138	var frame: anyframe = undefined;
	139
	140	fn doTheTest() void {
	141	_ = async amain();
	142	seq('d');
	143	resume frame;
	144	seq('h');
	145
	146	expect(std.mem.eql(u8, points, "abcdefgh"));
	147	}
	148
	149	fn amain() void {
	150	seq('a');
	151	var f = async testAsyncSeq();
	152	seq('c');
	153	await f;
	154	seq('g');
	155	}
	156
	157	fn testAsyncSeq() void {
	158	defer seq('f');
	159
	160	seq('b');
	161	suspend {
	162	frame = @frame();
	163	}
	164	seq('e');
	165	}
	166	var points = [_]u8{'x'} ** "abcdefgh".len;
	167	var index: usize = 0;
	168
	169	fn seq(c: u8) void {
	170	points[index] = c;
	171	index += 1;
	172	}
	173	};
	174	S.doTheTest();
	175	}
	176
	177	test "coroutine suspend with block" {
	178	const p = async testSuspendBlock();
	179	expect(!global_result);
	180	resume a_promise;
	181	expect(global_result);
	182	}
	183
	184	var a_promise: anyframe = undefined;
	185	var global_result = false;
	186	async fn testSuspendBlock() void {
	187	suspend {
	188	comptime expect(@typeOf(@frame()) == *@Frame(testSuspendBlock));
	189	a_promise = @frame();
	190	}
	191
	192	// Test to make sure that @frame() works as advertised (issue #1296)
	193	// var our_handle: anyframe = @frame();
	194	expect(a_promise == anyframe(@frame()));
	195
	196	global_result = true;
	197	}
	198
	199	var await_a_promise: anyframe = undefined;
	200	var await_final_result: i32 = 0;
	201
	202	test "coroutine await" {
	203	await_seq('a');
	204	const p = async await_amain();
	205	await_seq('f');
	206	resume await_a_promise;
	207	await_seq('i');
	208	expect(await_final_result == 1234);
	209	expect(std.mem.eql(u8, await_points, "abcdefghi"));
	210	}
	211	async fn await_amain() void {
	212	await_seq('b');
	213	const p = async await_another();
	214	await_seq('e');
	215	await_final_result = await p;
	216	await_seq('h');
	217	}
	218	async fn await_another() i32 {
	219	await_seq('c');
	220	suspend {
	221	await_seq('d');
	222	await_a_promise = @frame();
	223	}
	224	await_seq('g');
	225	return 1234;
	226	}
	227
	228	var await_points = [_]u8{0} ** "abcdefghi".len;
	229	var await_seq_index: usize = 0;
	230
	231	fn await_seq(c: u8) void {
	232	await_points[await_seq_index] = c;
	233	await_seq_index += 1;
	234	}
	235
	236	var early_final_result: i32 = 0;
	237
	238	test "coroutine await early return" {
	239	early_seq('a');
	240	const p = async early_amain();
	241	early_seq('f');
	242	expect(early_final_result == 1234);
	243	expect(std.mem.eql(u8, early_points, "abcdef"));
	244	}
	245	async fn early_amain() void {
	246	early_seq('b');
	247	const p = async early_another();
	248	early_seq('d');
	249	early_final_result = await p;
	250	early_seq('e');
	251	}
	252	async fn early_another() i32 {
	253	early_seq('c');
	254	return 1234;
	255	}
	256
	257	var early_points = [_]u8{0} ** "abcdef".len;
	258	var early_seq_index: usize = 0;
	259
	260	fn early_seq(c: u8) void {
	261	early_points[early_seq_index] = c;
	262	early_seq_index += 1;
	263	}
	264
	265	test "async function with dot syntax" {
	266	const S = struct {
	267	var y: i32 = 1;
	268	async fn foo() void {
	269	y += 1;
	270	suspend;
	271	}
	272	};
	273	const p = async S.foo();
	274	expect(S.y == 2);
	275	}
	276
	277	test "async fn pointer in a struct field" {
	278	var data: i32 = 1;
	279	const Foo = struct {
	280	bar: async fn (*i32) void,
	281	};
	282	var foo = Foo{ .bar = simpleAsyncFn2 };
	283	var bytes: [64]u8 = undefined;
	284	const f = @asyncCall(&bytes, {}, foo.bar, &data);
	285	comptime expect(@typeOf(f) == anyframe->void);
	286	expect(data == 2);
	287	resume f;
	288	expect(data == 4);
	289	_ = async doTheAwait(f);
	290	expect(data == 4);
	291	}
	292
	293	fn doTheAwait(f: anyframe->void) void {
	294	await f;
	295	}
	296
	297	async fn simpleAsyncFn2(y: *i32) void {
	298	defer y.* += 2;
	299	y.* += 1;
	300	suspend;
	301	}
	302
	303	test "@asyncCall with return type" {
	304	const Foo = struct {
	305	bar: async fn () i32,
	306
	307	var global_frame: anyframe = undefined;
	308
	309	async fn middle() i32 {
	310	return afunc();
	311	}
	312
	313	fn afunc() i32 {
	314	global_frame = @frame();
	315	suspend;
	316	return 1234;
	317	}
	318	};
	319	var foo = Foo{ .bar = Foo.middle };
	320	var bytes: [150]u8 = undefined;
	321	var aresult: i32 = 0;
	322	_ = @asyncCall(&bytes, &aresult, foo.bar);
	323	expect(aresult == 0);
	324	resume Foo.global_frame;
	325	expect(aresult == 1234);
	326	}
	327
	328	test "async fn with inferred error set" {
	329	const S = struct {
	330	var global_frame: anyframe = undefined;
	331
	332	fn doTheTest() void {
	333	var frame: [1]@Frame(middle) = undefined;
	334	var result: anyerror!void = undefined;
	335	_ = @asyncCall(@sliceToBytes(frame[0..]), &result, middle);
	336	resume global_frame;
	337	std.testing.expectError(error.Fail, result);
	338	}
	339
	340	async fn middle() !void {
	341	var f = async middle2();
	342	return await f;
	343	}
	344
	345	fn middle2() !void {
	346	return failing();
	347	}
	348
	349	fn failing() !void {
	350	global_frame = @frame();
	351	suspend;
	352	return error.Fail;
	353	}
	354	};
	355	S.doTheTest();
	356	}
	357
	358	test "error return trace across suspend points - early return" {
	359	const p = nonFailing();
	360	resume p;
	361	const p2 = async printTrace(p);
	362	}
	363
	364	test "error return trace across suspend points - async return" {
	365	const p = nonFailing();
	366	const p2 = async printTrace(p);
	367	resume p;
	368	}
	369
	370	fn nonFailing() (anyframe->anyerror!void) {
	371	const Static = struct {
	372	var frame: @Frame(suspendThenFail) = undefined;
	373	};
	374	Static.frame = async suspendThenFail();
	375	return &Static.frame;
	376	}
	377	async fn suspendThenFail() anyerror!void {
	378	suspend;
	379	return error.Fail;
	380	}
	381	async fn printTrace(p: anyframe->(anyerror!void)) void {
	382	(await p) catch \|e\| {
	383	std.testing.expect(e == error.Fail);
	384	if (@errorReturnTrace()) \|trace\| {
	385	expect(trace.index == 1);
	386	} else switch (builtin.mode) {
	387	.Debug, .ReleaseSafe => @panic("expected return trace"),
	388	.ReleaseFast, .ReleaseSmall => {},
	389	}
	390	};
	391	}
	392
	393	test "break from suspend" {
	394	var my_result: i32 = 1;
	395	const p = async testBreakFromSuspend(&my_result);
	396	std.testing.expect(my_result == 2);
	397	}
	398	async fn testBreakFromSuspend(my_result: *i32) void {
	399	suspend {
	400	resume @frame();
	401	}
	402	my_result.* += 1;
	403	suspend;
	404	my_result.* += 1;
	405	}
	406
	407	test "heap allocated async function frame" {
	408	const S = struct {
	409	var x: i32 = 42;
	410
	411	fn doTheTest() !void {
	412	const frame = try std.heap.direct_allocator.create(@Frame(someFunc));
	413	defer std.heap.direct_allocator.destroy(frame);
	414
	415	expect(x == 42);
	416	frame.* = async someFunc();
	417	expect(x == 43);
	418	resume frame;
	419	expect(x == 44);
	420	}
	421
	422	fn someFunc() void {
	423	x += 1;
	424	suspend;
	425	x += 1;
	426	}
	427	};
	428	try S.doTheTest();
	429	}
	430
	431	test "async function call return value" {
	432	const S = struct {
	433	var frame: anyframe = undefined;
	434	var pt = Point{.x = 10, .y = 11 };
	435
	436	fn doTheTest() void {
	437	expectEqual(pt.x, 10);
	438	expectEqual(pt.y, 11);
	439	_ = async first();
	440	expectEqual(pt.x, 10);
	441	expectEqual(pt.y, 11);
	442	resume frame;
	443	expectEqual(pt.x, 1);
	444	expectEqual(pt.y, 2);
	445	}
	446
	447	fn first() void {
	448	pt = second(1, 2);
	449	}
	450
	451	fn second(x: i32, y: i32) Point {
	452	return other(x, y);
	453	}
	454
	455	fn other(x: i32, y: i32) Point {
	456	frame = @frame();
	457	suspend;
	458	return Point{
	459	.x = x,
	460	.y = y,
	461	};
	462	}
	463
	464	const Point = struct {
	465	x: i32,
	466	y: i32,
	467	};
	468	};
	469	S.doTheTest();
	470	}
	471
	472	test "suspension points inside branching control flow" {
	473	const S = struct {
	474	var result: i32 = 10;
	475
	476	fn doTheTest() void {
	477	expect(10 == result);
	478	var frame = async func(true);
	479	expect(10 == result);
	480	resume frame;
	481	expect(11 == result);
	482	resume frame;
	483	expect(12 == result);
	484	resume frame;
	485	expect(13 == result);
	486	}
	487
	488	fn func(b: bool) void {
	489	while (b) {
	490	suspend;
	491	result += 1;
	492	}
	493	}
	494	};
	495	S.doTheTest();
	496	}
	497
	498	test "call async function which has struct return type" {
	499	const S = struct {
	500	var frame: anyframe = undefined;
	501
	502	fn doTheTest() void {
	503	_ = async atest();
	504	resume frame;
	505	}
	506
	507	fn atest() void {
	508	const result = func();
	509	expect(result.x == 5);
	510	expect(result.y == 6);
	511	}
	512
	513	const Point = struct {
	514	x: usize,
	515	y: usize,
	516	};
	517
	518	fn func() Point {
	519	suspend {
	520	frame = @frame();
	521	}
	522	return Point{
	523	.x = 5,
	524	.y = 6,
	525	};
	526	}
	527	};
	528	S.doTheTest();
	529	}
	530
	531	test "pass string literal to async function" {
	532	const S = struct {
	533	var frame: anyframe = undefined;
	534	var ok: bool = false;
	535
	536	fn doTheTest() void {
	537	_ = async hello("hello");
	538	resume frame;
	539	expect(ok);
	540	}
	541
	542	fn hello(msg: []const u8) void {
	543	frame = @frame();
	544	suspend;
	545	expectEqual(([]const u8)("hello"), msg);
	546	ok = true;
	547	}
	548	};
	549	S.doTheTest();
	550	}
	551
	552	test "await inside an errdefer" {
	553	const S = struct {
	554	var frame: anyframe = undefined;
	555
	556	fn doTheTest() void {
	557	_ = async amainWrap();
	558	resume frame;
	559	}
	560
	561	fn amainWrap() !void {
	562	var foo = async func();
	563	errdefer await foo;
	564	return error.Bad;
	565	}
	566
	567	fn func() void {
	568	frame = @frame();
	569	suspend;
	570	}
	571
	572	};
	573	S.doTheTest();
	574	}
	575
	576	test "try in an async function with error union and non-zero-bit payload" {
	577	const S = struct {
	578	var frame: anyframe = undefined;
	579	var ok = false;
	580
	581	fn doTheTest() void {
	582	_ = async amain();
	583	resume frame;
	584	expect(ok);
	585	}
	586
	587	fn amain() void {
	588	std.testing.expectError(error.Bad, theProblem());
	589	ok = true;
	590	}
	591
	592	fn theProblem() ![]u8 {
	593	frame = @frame();
	594	suspend;
	595	const result = try other();
	596	return result;
	597	}
	598
	599	fn other() ![]u8 {
	600	return error.Bad;
	601	}
	602	};
	603	S.doTheTest();
	604	}
	605
	606	test "returning a const error from async function" {
	607	const S = struct {
	608	var frame: anyframe = undefined;
	609	var ok = false;
	610
	611	fn doTheTest() void {
	612	_ = async amain();
	613	resume frame;
	614	expect(ok);
	615	}
	616
	617	fn amain() !void {
	618	var download_frame = async fetchUrl(10, "a string");
	619	const download_text = try await download_frame;
	620
	621	@panic("should not get here");
	622	}
	623
	624	fn fetchUrl(unused: i32, url: []const u8) ![]u8 {
	625	frame = @frame();
	626	suspend;
	627	ok = true;
	628	return error.OutOfMemory;
	629	}
	630	};
	631	S.doTheTest();
	632	}
	633
	634	test "async/await typical usage" {
	635	inline for ([_]bool{false, true}) \|b1\| {
	636	inline for ([_]bool{false, true}) \|b2\| {
	637	inline for ([_]bool{false, true}) \|b3\| {
	638	inline for ([_]bool{false, true}) \|b4\| {
	639	testAsyncAwaitTypicalUsage(b1, b2, b3, b4).doTheTest();
	640	}
	641	}
	642	}
	643	}
	644	}
	645
	646	fn testAsyncAwaitTypicalUsage(
	647	comptime simulate_fail_download: bool,
	648	comptime simulate_fail_file: bool,
	649	comptime suspend_download: bool,
	650	comptime suspend_file: bool) type
	651	{
	652	return struct {
	653	fn doTheTest() void {
	654	_ = async amainWrap();
	655	if (suspend_file) {
	656	resume global_file_frame;
	657	}
	658	if (suspend_download) {
	659	resume global_download_frame;
	660	}
	661	}
	662	fn amainWrap() void {
	663	if (amain()) \|_\| {
	664	expect(!simulate_fail_download);
	665	expect(!simulate_fail_file);
	666	} else \|e\| switch (e) {
	667	error.NoResponse => expect(simulate_fail_download),
	668	error.FileNotFound => expect(simulate_fail_file),
	669	else => @panic("test failure"),
	670	}
	671	}
	672
	673	fn amain() !void {
	674	const allocator = std.heap.direct_allocator; // TODO once we have the debug allocator, use that, so that this can detect leaks
	675	var download_frame = async fetchUrl(allocator, "https://example.com/");
	676	var download_awaited = false;
	677	errdefer if (!download_awaited) {
	678	if (await download_frame) \|x\| allocator.free(x) else \|_\| {}
	679	};
	680
	681	var file_frame = async readFile(allocator, "something.txt");
	682	var file_awaited = false;
	683	errdefer if (!file_awaited) {
	684	if (await file_frame) \|x\| allocator.free(x) else \|_\| {}
	685	};
	686
	687	download_awaited = true;
	688	const download_text = try await download_frame;
	689	defer allocator.free(download_text);
	690
	691	file_awaited = true;
	692	const file_text = try await file_frame;
	693	defer allocator.free(file_text);
	694
	695	expect(std.mem.eql(u8, "expected download text", download_text));
	696	expect(std.mem.eql(u8, "expected file text", file_text));
	697	}
	698
	699	var global_download_frame: anyframe = undefined;
	700	fn fetchUrl(allocator: *std.mem.Allocator, url: []const u8) anyerror![]u8 {
	701	const result = try std.mem.dupe(allocator, u8, "expected download text");
	702	errdefer allocator.free(result);
	703	if (suspend_download) {
	704	suspend {
	705	global_download_frame = @frame();
	706	}
	707	}
	708	if (simulate_fail_download) return error.NoResponse;
	709	return result;
	710	}
	711
	712	var global_file_frame: anyframe = undefined;
	713	fn readFile(allocator: *std.mem.Allocator, filename: []const u8) anyerror![]u8 {
	714	const result = try std.mem.dupe(allocator, u8, "expected file text");
	715	errdefer allocator.free(result);
	716	if (suspend_file) {
	717	suspend {
	718	global_file_frame = @frame();
	719	}
	720	}
	721	if (simulate_fail_file) return error.FileNotFound;
	722	return result;
	723	}
	724	};
	725	}
	726
	727	test "alignment of local variables in async functions" {
	728	const S = struct {
	729	fn doTheTest() void {
	730	var y: u8 = 123;
	731	var x: u8 align(128) = 1;
	732	expect(@ptrToInt(&x) % 128 == 0);
	733	}
	734	};
	735	S.doTheTest();
	736	}

test/stage1/behavior/await_struct.zig created+44

...	...	@@ -0,0 +1,44 @@
	1	const std = @import("std");
	2	const builtin = @import("builtin");
	3	const expect = std.testing.expect;
	4
	5	const Foo = struct {
	6	x: i32,
	7	};
	8
	9	var await_a_promise: anyframe = undefined;
	10	var await_final_result = Foo{ .x = 0 };
	11
	12	test "coroutine await struct" {
	13	await_seq('a');
	14	const p = async await_amain();
	15	await_seq('f');
	16	resume await_a_promise;
	17	await_seq('i');
	18	expect(await_final_result.x == 1234);
	19	expect(std.mem.eql(u8, await_points, "abcdefghi"));
	20	}
	21	async fn await_amain() void {
	22	await_seq('b');
	23	const p = async await_another();
	24	await_seq('e');
	25	await_final_result = await p;
	26	await_seq('h');
	27	}
	28	async fn await_another() Foo {
	29	await_seq('c');
	30	suspend {
	31	await_seq('d');
	32	await_a_promise = @frame();
	33	}
	34	await_seq('g');
	35	return Foo{ .x = 1234 };
	36	}
	37
	38	var await_points = [_]u8{0} ** "abcdefghi".len;
	39	var await_seq_index: usize = 0;
	40
	41	fn await_seq(c: u8) void {
	42	await_points[await_seq_index] = c;
	43	await_seq_index += 1;
	44	}

test/stage1/behavior/cancel.zig deleted-86

...	...	@@ -1,86 +0,0 @@
1		const std = @import("std");
2
3		var defer_f1: bool = false;
4		var defer_f2: bool = false;
5		var defer_f3: bool = false;
6
7		test "cancel forwards" {
8		const p = async<std.heap.direct_allocator> f1() catch unreachable;
9		cancel p;
10		std.testing.expect(defer_f1);
11		std.testing.expect(defer_f2);
12		std.testing.expect(defer_f3);
13		}
14
15		async fn f1() void {
16		defer {
17		defer_f1 = true;
18		}
19		await (async f2() catch unreachable);
20		}
21
22		async fn f2() void {
23		defer {
24		defer_f2 = true;
25		}
26		await (async f3() catch unreachable);
27		}
28
29		async fn f3() void {
30		defer {
31		defer_f3 = true;
32		}
33		suspend;
34		}
35
36		var defer_b1: bool = false;
37		var defer_b2: bool = false;
38		var defer_b3: bool = false;
39		var defer_b4: bool = false;
40
41		test "cancel backwards" {
42		const p = async<std.heap.direct_allocator> b1() catch unreachable;
43		cancel p;
44		std.testing.expect(defer_b1);
45		std.testing.expect(defer_b2);
46		std.testing.expect(defer_b3);
47		std.testing.expect(defer_b4);
48		}
49
50		async fn b1() void {
51		defer {
52		defer_b1 = true;
53		}
54		await (async b2() catch unreachable);
55		}
56
57		var b4_handle: promise = undefined;
58
59		async fn b2() void {
60		const b3_handle = async b3() catch unreachable;
61		resume b4_handle;
62		cancel b4_handle;
63		defer {
64		defer_b2 = true;
65		}
66		const value = await b3_handle;
67		@panic("unreachable");
68		}
69
70		async fn b3() i32 {
71		defer {
72		defer_b3 = true;
73		}
74		await (async b4() catch unreachable);
75		return 1234;
76		}
77
78		async fn b4() void {
79		defer {
80		defer_b4 = true;
81		}
82		suspend {
83		b4_handle = @handle();
84		}
85		suspend;
86		}

test/stage1/behavior/coroutine_await_struct.zig deleted-44

...	...	@@ -1,44 +0,0 @@
1		const std = @import("std");
2		const builtin = @import("builtin");
3		const expect = std.testing.expect;
4
5		const Foo = struct {
6		x: i32,
7		};
8
9		var await_a_promise: promise = undefined;
10		var await_final_result = Foo{ .x = 0 };
11
12		test "coroutine await struct" {
13		await_seq('a');
14		const p = async<std.heap.direct_allocator> await_amain() catch unreachable;
15		await_seq('f');
16		resume await_a_promise;
17		await_seq('i');
18		expect(await_final_result.x == 1234);
19		expect(std.mem.eql(u8, await_points, "abcdefghi"));
20		}
21		async fn await_amain() void {
22		await_seq('b');
23		const p = async await_another() catch unreachable;
24		await_seq('e');
25		await_final_result = await p;
26		await_seq('h');
27		}
28		async fn await_another() Foo {
29		await_seq('c');
30		suspend {
31		await_seq('d');
32		await_a_promise = @handle();
33		}
34		await_seq('g');
35		return Foo{ .x = 1234 };
36		}
37
38		var await_points = [_]u8{0} ** "abcdefghi".len;
39		var await_seq_index: usize = 0;
40
41		fn await_seq(c: u8) void {
42		await_points[await_seq_index] = c;
43		await_seq_index += 1;
44		}

test/stage1/behavior/coroutines.zig deleted-236

...	...	@@ -1,236 +0,0 @@
1		const std = @import("std");
2		const builtin = @import("builtin");
3		const expect = std.testing.expect;
4		const allocator = std.heap.direct_allocator;
5
6		var x: i32 = 1;
7
8		test "create a coroutine and cancel it" {
9		const p = try async<allocator> simpleAsyncFn();
10		comptime expect(@typeOf(p) == promise->void);
11		cancel p;
12		expect(x == 2);
13		}
14		async fn simpleAsyncFn() void {
15		x += 1;
16		suspend;
17		x += 1;
18		}
19
20		test "coroutine suspend, resume, cancel" {
21		seq('a');
22		const p = try async<allocator> testAsyncSeq();
23		seq('c');
24		resume p;
25		seq('f');
26		cancel p;
27		seq('g');
28
29		expect(std.mem.eql(u8, points, "abcdefg"));
30		}
31		async fn testAsyncSeq() void {
32		defer seq('e');
33
34		seq('b');
35		suspend;
36		seq('d');
37		}
38		var points = [_]u8{0} ** "abcdefg".len;
39		var index: usize = 0;
40
41		fn seq(c: u8) void {
42		points[index] = c;
43		index += 1;
44		}
45
46		test "coroutine suspend with block" {
47		const p = try async<allocator> testSuspendBlock();
48		std.testing.expect(!result);
49		resume a_promise;
50		std.testing.expect(result);
51		cancel p;
52		}
53
54		var a_promise: promise = undefined;
55		var result = false;
56		async fn testSuspendBlock() void {
57		suspend {
58		comptime expect(@typeOf(@handle()) == promise->void);
59		a_promise = @handle();
60		}
61
62		//Test to make sure that @handle() works as advertised (issue #1296)
63		//var our_handle: promise = @handle();
64		expect(a_promise == @handle());
65
66		result = true;
67		}
68
69		var await_a_promise: promise = undefined;
70		var await_final_result: i32 = 0;
71
72		test "coroutine await" {
73		await_seq('a');
74		const p = async<allocator> await_amain() catch unreachable;
75		await_seq('f');
76		resume await_a_promise;
77		await_seq('i');
78		expect(await_final_result == 1234);
79		expect(std.mem.eql(u8, await_points, "abcdefghi"));
80		}
81		async fn await_amain() void {
82		await_seq('b');
83		const p = async await_another() catch unreachable;
84		await_seq('e');
85		await_final_result = await p;
86		await_seq('h');
87		}
88		async fn await_another() i32 {
89		await_seq('c');
90		suspend {
91		await_seq('d');
92		await_a_promise = @handle();
93		}
94		await_seq('g');
95		return 1234;
96		}
97
98		var await_points = [_]u8{0} ** "abcdefghi".len;
99		var await_seq_index: usize = 0;
100
101		fn await_seq(c: u8) void {
102		await_points[await_seq_index] = c;
103		await_seq_index += 1;
104		}
105
106		var early_final_result: i32 = 0;
107
108		test "coroutine await early return" {
109		early_seq('a');
110		const p = async<allocator> early_amain() catch @panic("out of memory");
111		early_seq('f');
112		expect(early_final_result == 1234);
113		expect(std.mem.eql(u8, early_points, "abcdef"));
114		}
115		async fn early_amain() void {
116		early_seq('b');
117		const p = async early_another() catch @panic("out of memory");
118		early_seq('d');
119		early_final_result = await p;
120		early_seq('e');
121		}
122		async fn early_another() i32 {
123		early_seq('c');
124		return 1234;
125		}
126
127		var early_points = [_]u8{0} ** "abcdef".len;
128		var early_seq_index: usize = 0;
129
130		fn early_seq(c: u8) void {
131		early_points[early_seq_index] = c;
132		early_seq_index += 1;
133		}
134
135		test "coro allocation failure" {
136		var failing_allocator = std.debug.FailingAllocator.init(std.debug.global_allocator, 0);
137		if (async<&failing_allocator.allocator> asyncFuncThatNeverGetsRun()) {
138		@panic("expected allocation failure");
139		} else \|err\| switch (err) {
140		error.OutOfMemory => {},
141		}
142		}
143		async fn asyncFuncThatNeverGetsRun() void {
144		@panic("coro frame allocation should fail");
145		}
146
147		test "async function with dot syntax" {
148		const S = struct {
149		var y: i32 = 1;
150		async fn foo() void {
151		y += 1;
152		suspend;
153		}
154		};
155		const p = try async<allocator> S.foo();
156		cancel p;
157		expect(S.y == 2);
158		}
159
160		test "async fn pointer in a struct field" {
161		var data: i32 = 1;
162		const Foo = struct {
163		bar: async<std.mem.Allocator> fn (i32) void,
164		};
165		var foo = Foo{ .bar = simpleAsyncFn2 };
166		const p = (async<allocator> foo.bar(&data)) catch unreachable;
167		expect(data == 2);
168		cancel p;
169		expect(data == 4);
170		}
171		async<std.mem.Allocator> fn simpleAsyncFn2(y: i32) void {
172		defer y.* += 2;
173		y.* += 1;
174		suspend;
175		}
176
177		test "async fn with inferred error set" {
178		const p = (async<allocator> failing()) catch unreachable;
179		resume p;
180		cancel p;
181		}
182
183		async fn failing() !void {
184		suspend;
185		return error.Fail;
186		}
187
188		test "error return trace across suspend points - early return" {
189		const p = nonFailing();
190		resume p;
191		const p2 = try async<allocator> printTrace(p);
192		cancel p2;
193		}
194
195		test "error return trace across suspend points - async return" {
196		const p = nonFailing();
197		const p2 = try async<std.debug.global_allocator> printTrace(p);
198		resume p;
199		cancel p2;
200		}
201
202		fn nonFailing() (promise->anyerror!void) {
203		return async<std.debug.global_allocator> suspendThenFail() catch unreachable;
204		}
205		async fn suspendThenFail() anyerror!void {
206		suspend;
207		return error.Fail;
208		}
209		async fn printTrace(p: promise->(anyerror!void)) void {
210		(await p) catch \|e\| {
211		std.testing.expect(e == error.Fail);
212		if (@errorReturnTrace()) \|trace\| {
213		expect(trace.index == 1);
214		} else switch (builtin.mode) {
215		builtin.Mode.Debug, builtin.Mode.ReleaseSafe => @panic("expected return trace"),
216		builtin.Mode.ReleaseFast, builtin.Mode.ReleaseSmall => {},
217		}
218		};
219		}
220
221		test "break from suspend" {
222		var buf: [500]u8 = undefined;
223		var a = &std.heap.FixedBufferAllocator.init(buf[0..]).allocator;
224		var my_result: i32 = 1;
225		const p = try async<a> testBreakFromSuspend(&my_result);
226		cancel p;
227		std.testing.expect(my_result == 2);
228		}
229		async fn testBreakFromSuspend(my_result: *i32) void {
230		suspend {
231		resume @handle();
232		}
233		my_result.* += 1;
234		suspend;
235		my_result.* += 1;
236		}

test/stage1/behavior/type_info.zig+20-17

...	...	@@ -116,21 +116,6 @@ fn testOptional() void {
116	116	expect(null_info.Optional.child == void);
117	117	}
118	118
119		test "type info: promise info" {
120		testPromise();
121		comptime testPromise();
122		}
123
124		fn testPromise() void {
125		const null_promise_info = @typeInfo(promise);
126		expect(TypeId(null_promise_info) == TypeId.Promise);
127		expect(null_promise_info.Promise.child == null);
128
129		const promise_info = @typeInfo(promise->usize);
130		expect(TypeId(promise_info) == TypeId.Promise);
131		expect(promise_info.Promise.child.? == usize);
132		}
133
134	119	test "type info: error set, error union info" {
135	120	testErrorSet();
136	121	comptime testErrorSet();
...	...	@@ -192,7 +177,7 @@ fn testUnion() void {
192	177	expect(TypeId(typeinfo_info) == TypeId.Union);
193	178	expect(typeinfo_info.Union.layout == TypeInfo.ContainerLayout.Auto);
194	179	expect(typeinfo_info.Union.tag_type.? == TypeId);
195		expect(typeinfo_info.Union.fields.len == 25);
	180	expect(typeinfo_info.Union.fields.len == 26);
196	181	expect(typeinfo_info.Union.fields[4].enum_field != null);
197	182	expect(typeinfo_info.Union.fields[4].enum_field.?.value == 4);
198	183	expect(typeinfo_info.Union.fields[4].field_type == @typeOf(@typeInfo(u8).Int));
...	...	@@ -265,7 +250,6 @@ fn testFunction() void {
265	250	expect(fn_info.Fn.args.len == 2);
266	251	expect(fn_info.Fn.is_var_args);
267	252	expect(fn_info.Fn.return_type == null);
268		expect(fn_info.Fn.async_allocator_type == null);
269	253
270	254	const test_instance: TestStruct = undefined;
271	255	const bound_fn_info = @typeInfo(@typeOf(test_instance.foo));
...	...	@@ -296,6 +280,25 @@ fn testVector() void {
296	280	expect(vec_info.Vector.child == i32);
297	281	}
298	282
	283	test "type info: anyframe and anyframe->T" {
	284	testAnyFrame();
	285	comptime testAnyFrame();
	286	}
	287
	288	fn testAnyFrame() void {
	289	{
	290	const anyframe_info = @typeInfo(anyframe->i32);
	291	expect(TypeId(anyframe_info) == .AnyFrame);
	292	expect(anyframe_info.AnyFrame.child.? == i32);
	293	}
	294
	295	{
	296	const anyframe_info = @typeInfo(anyframe);
	297	expect(TypeId(anyframe_info) == .AnyFrame);
	298	expect(anyframe_info.AnyFrame.child == null);
	299	}
	300	}
	301
299	302	test "type info: optional field unwrapping" {
300	303	const Struct = struct {
301	304	cdOffset: u32,