Commit 2a990d69669c2a2cd16134e8ebbd2750060f8071 | zig

author	Andrew Kelley <andrew@ziglang.org> 2021-05-27 16:32:35-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2021-05-28 12:58:40-07:00
log	2a990d69669c2a2cd16134e8ebbd2750060f8071
tree	b60796aa1852a533cd23f3a0750957d4dd1ea71a
parent	673ae5b457479760ff8e08b3e06917a4b2651436

stage1: rework tokenizer to match stage2

 * Extracts AstGen logic from ir.cpp into astgen.cpp. Reduces the
   largest file of stage1 from 33,551 lines to 25,510.
 * tokenizer: rework it completely to match the stage2 tokenizer logic.
   They can now be maintained together; when one is changed, the other
   can be changed in the same way.
   - Each token now takes up 13 bytes instead of 64 bytes. The tokenizer
     does not parse char literals, string literals, integer literals,
     etc into meaningful data. Instead, that happens during parsing or
     astgen.
   - no longer store line offsets. Error messages scan source
     files to find the line/column as needed (same as stage2).
   - main loop: instead of checking the loop, handle a null byte
     explicitly in the switch statements. This is a nice improvement
     that we may want to backport to stage2.
   - delete some dead tokens, artifacts of past syntax that no longer
     exists.
 * Parser: fix a TODO by parsing builtin functions as tokens rather than
   `@` as a separate token. This is how stage2 does it.
 * Remove some debugging infrastructure. These will need to be redone,
   if at all, as the code migrates to match stage2.
   - remove the ast_render code.
   - remove the IR debugging stuff
   - remove teh token printing code

25 files changed, 11377 insertions(+), 12359 deletions(-)

CMakeLists.txt+2-2

...	...	@@ -292,7 +292,7 @@ set(ZIG0_SOURCES
292	292
293	293	set(STAGE1_SOURCES
294	294	"${CMAKE_SOURCE_DIR}/src/stage1/analyze.cpp"
295		"${CMAKE_SOURCE_DIR}/src/stage1/ast_render.cpp"
	295	"${CMAKE_SOURCE_DIR}/src/stage1/astgen.cpp"
296	296	"${CMAKE_SOURCE_DIR}/src/stage1/bigfloat.cpp"
297	297	"${CMAKE_SOURCE_DIR}/src/stage1/bigint.cpp"
298	298	"${CMAKE_SOURCE_DIR}/src/stage1/buffer.cpp"
...	...	@@ -307,11 +307,11 @@ set(STAGE1_SOURCES
307	307	"${CMAKE_SOURCE_DIR}/src/stage1/os.cpp"
308	308	"${CMAKE_SOURCE_DIR}/src/stage1/parser.cpp"
309	309	"${CMAKE_SOURCE_DIR}/src/stage1/range_set.cpp"
	310	"${CMAKE_SOURCE_DIR}/src/stage1/softfloat_ext.cpp"
310	311	"${CMAKE_SOURCE_DIR}/src/stage1/stage1.cpp"
311	312	"${CMAKE_SOURCE_DIR}/src/stage1/target.cpp"
312	313	"${CMAKE_SOURCE_DIR}/src/stage1/tokenizer.cpp"
313	314	"${CMAKE_SOURCE_DIR}/src/stage1/util.cpp"
314		"${CMAKE_SOURCE_DIR}/src/stage1/softfloat_ext.cpp"
315	315	)
316	316	set(OPTIMIZED_C_SOURCES
317	317	"${CMAKE_SOURCE_DIR}/src/stage1/parse_f128.c"

lib/std/zig/string_literal.zig+1

...	...	@@ -107,6 +107,7 @@ pub fn parseAppend(buf: *std.ArrayList(u8), bytes: []const u8) error{OutOfMemory
107	107	const hex_str = slice[index + 2 .. index_end];
108	108	if (std.fmt.parseUnsigned(u32, hex_str, 16)) \|uint\| {
109	109	if (uint <= 0x10ffff) {
	110	// TODO this incorrectly depends on endianness
110	111	try buf.appendSlice(std.mem.toBytes(uint)[0..]);
111	112	state = State.Start;
112	113	index = index_end; // loop-header increments

src/stage1.zig+2

...	...	@@ -252,6 +252,8 @@ const Error = extern enum {
252	252	ZigIsTheCCompiler,
253	253	FileBusy,
254	254	Locked,
	255	InvalidCharacter,
	256	UnicodePointTooLarge,
255	257	};
256	258
257	259	// ABI warning

src/stage1/all_types.hpp+25-77

...	...	@@ -670,7 +670,7 @@ enum NodeType {
670	670	NodeTypeIntLiteral,
671	671	NodeTypeStringLiteral,
672	672	NodeTypeCharLiteral,
673		NodeTypeSymbol,
	673	NodeTypeIdentifier,
674	674	NodeTypePrefixOpExpr,
675	675	NodeTypePointerType,
676	676	NodeTypeFnCallExpr,
...	...	@@ -710,6 +710,7 @@ enum NodeType {
710	710	NodeTypeAwaitExpr,
711	711	NodeTypeSuspend,
712	712	NodeTypeAnyFrameType,
	713	// main_token points to the identifier.
713	714	NodeTypeEnumLiteral,
714	715	NodeTypeAnyTypeField,
715	716	};
...	...	@@ -733,7 +734,8 @@ struct AstNodeFnProto {
733	734	AstNode *section_expr;
734	735	// populated if the "callconv(S)" is present
735	736	AstNode *callconv_expr;
736		Buf doc_comments;
	737
	738	TokenIndex doc_comments;
737	739
738	740	// This is set based only on the existence of a noinline or inline keyword.
739	741	// This is then resolved to an is_noinline bool and (potentially .Inline)
...	...	@@ -755,8 +757,8 @@ struct AstNodeFnDef {
755	757	struct AstNodeParamDecl {
756	758	Buf *name;
757	759	AstNode *type;
758		Token *anytype_token;
759		Buf doc_comments;
	760	TokenIndex doc_comments;
	761	TokenIndex anytype_token;
760	762	bool is_noalias;
761	763	bool is_comptime;
762	764	bool is_var_args;
...	...	@@ -799,9 +801,9 @@ struct AstNodeVariableDeclaration {
799	801	AstNode *align_expr;
800	802	// populated if the "section(S)" is present
801	803	AstNode *section_expr;
802		Token *threadlocal_tok;
803		Buf doc_comments;
	804	TokenIndex doc_comments;
804	805
	806	TokenIndex threadlocal_tok;
805	807	VisibMod visib_mod;
806	808	bool is_const;
807	809	bool is_comptime;
...	...	@@ -935,13 +937,14 @@ struct AstNodePrefixOpExpr {
935	937	};
936	938
937	939	struct AstNodePointerType {
938		Token *star_token;
	940	TokenIndex star_token;
	941	TokenIndex allow_zero_token;
	942	TokenIndex bit_offset_start;
	943	TokenIndex host_int_bytes;
	944
939	945	AstNode *sentinel;
940	946	AstNode *align_expr;
941		BigInt *bit_offset_start;
942		BigInt *host_int_bytes;
943	947	AstNode *op_expr;
944		Token *allow_zero_token;
945	948	bool is_const;
946	949	bool is_volatile;
947	950	};
...	...	@@ -956,7 +959,7 @@ struct AstNodeArrayType {
956	959	AstNode *sentinel;
957	960	AstNode *child_type;
958	961	AstNode *align_expr;
959		Token *allow_zero_token;
	962	TokenIndex allow_zero_token;
960	963	bool is_const;
961	964	bool is_volatile;
962	965	};
...	...	@@ -974,11 +977,11 @@ struct AstNodeIfBoolExpr {
974	977
975	978	struct AstNodeTryExpr {
976	979	Buf *var_symbol;
977		bool var_is_ptr;
978	980	AstNode *target_node;
979	981	AstNode *then_node;
980	982	AstNode *else_node;
981	983	Buf *err_symbol;
	984	bool var_is_ptr;
982	985	};
983	986
984	987	struct AstNodeTestExpr {
...	...	@@ -993,12 +996,12 @@ struct AstNodeWhileExpr {
993	996	Buf *name;
994	997	AstNode *condition;
995	998	Buf *var_symbol;
996		bool var_is_ptr;
997	999	AstNode *continue_expr;
998	1000	AstNode *body;
999	1001	AstNode *else_node;
1000	1002	Buf *err_symbol;
1001	1003	bool is_inline;
	1004	bool var_is_ptr;
1002	1005	};
1003	1006
1004	1007	struct AstNodeForExpr {
...	...	@@ -1070,7 +1073,7 @@ struct AsmToken {
1070	1073	};
1071	1074
1072	1075	struct AstNodeAsmExpr {
1073		Token *volatile_token;
	1076	TokenIndex volatile_token;
1074	1077	AstNode *asm_template;
1075	1078	ZigList<AsmOutput*> output_list;
1076	1079	ZigList<AsmInput*> input_list;
...	...	@@ -1094,7 +1097,7 @@ struct AstNodeContainerDecl {
1094	1097	AstNode *init_arg_expr; // enum(T), struct(endianness), or union(T), or union(enum(T))
1095	1098	ZigList<AstNode *> fields;
1096	1099	ZigList<AstNode *> decls;
1097		Buf doc_comments;
	1100	TokenIndex doc_comments;
1098	1101
1099	1102	ContainerKind kind;
1100	1103	ContainerLayout layout;
...	...	@@ -1103,7 +1106,7 @@ struct AstNodeContainerDecl {
1103	1106	};
1104	1107
1105	1108	struct AstNodeErrorSetField {
1106		Buf doc_comments;
	1109	TokenIndex doc_comments;
1107	1110	AstNode *field_name;
1108	1111	};
1109	1112
...	...	@@ -1118,28 +1121,8 @@ struct AstNodeStructField {
1118	1121	AstNode *value;
1119	1122	// populated if the "align(A)" is present
1120	1123	AstNode *align_expr;
1121		Buf doc_comments;
1122		Token *comptime_token;
1123		};
1124
1125		struct AstNodeStringLiteral {
1126		Buf *buf;
1127		};
1128
1129		struct AstNodeCharLiteral {
1130		uint32_t value;
1131		};
1132
1133		struct AstNodeFloatLiteral {
1134		BigFloat *bigfloat;
1135
1136		// overflow is true if when parsing the number, we discovered it would not
1137		// fit without losing data in a double
1138		bool overflow;
1139		};
1140
1141		struct AstNodeIntLiteral {
1142		BigInt *bigint;
	1124	TokenIndex doc_comments;
	1125	TokenIndex comptime_token;
1143	1126	};
1144	1127
1145	1128	struct AstNodeStructValueField {
...	...	@@ -1158,19 +1141,6 @@ struct AstNodeContainerInitExpr {
1158	1141	ContainerInitKind kind;
1159	1142	};
1160	1143
1161		struct AstNodeNullLiteral {
1162		};
1163
1164		struct AstNodeUndefinedLiteral {
1165		};
1166
1167		struct AstNodeThisLiteral {
1168		};
1169
1170		struct AstNodeSymbolExpr {
1171		Buf *symbol;
1172		};
1173
1174	1144	struct AstNodeBoolLiteral {
1175	1145	bool value;
1176	1146	};
...	...	@@ -1188,13 +1158,6 @@ struct AstNodeContinueExpr {
1188	1158	Buf *name;
1189	1159	};
1190	1160
1191		struct AstNodeUnreachableExpr {
1192		};
1193
1194
1195		struct AstNodeErrorType {
1196		};
1197
1198	1161	struct AstNodeAwaitExpr {
1199	1162	AstNode *expr;
1200	1163	};
...	...	@@ -1207,16 +1170,10 @@ struct AstNodeAnyFrameType {
1207	1170	AstNode *payload_type; // can be NULL
1208	1171	};
1209	1172
1210		struct AstNodeEnumLiteral {
1211		Token *period;
1212		Token *identifier;
1213		};
1214
1215	1173	struct AstNode {
1216	1174	enum NodeType type;
	1175	TokenIndex main_token;
1217	1176	bool already_traced_this_node;
1218		size_t line;
1219		size_t column;
1220	1177	ZigType *owner;
1221	1178	union {
1222	1179	AstNodeFnDef fn_def;
...	...	@@ -1252,30 +1209,19 @@ struct AstNode {
1252	1209	AstNodePtrDerefExpr ptr_deref_expr;
1253	1210	AstNodeContainerDecl container_decl;
1254	1211	AstNodeStructField struct_field;
1255		AstNodeStringLiteral string_literal;
1256		AstNodeCharLiteral char_literal;
1257		AstNodeFloatLiteral float_literal;
1258		AstNodeIntLiteral int_literal;
1259	1212	AstNodeContainerInitExpr container_init_expr;
1260	1213	AstNodeStructValueField struct_val_field;
1261		AstNodeNullLiteral null_literal;
1262		AstNodeUndefinedLiteral undefined_literal;
1263		AstNodeThisLiteral this_literal;
1264		AstNodeSymbolExpr symbol_expr;
1265	1214	AstNodeBoolLiteral bool_literal;
1266	1215	AstNodeBreakExpr break_expr;
1267	1216	AstNodeContinueExpr continue_expr;
1268		AstNodeUnreachableExpr unreachable_expr;
1269	1217	AstNodeArrayType array_type;
1270	1218	AstNodeInferredArrayType inferred_array_type;
1271		AstNodeErrorType error_type;
1272	1219	AstNodeErrorSetDecl err_set_decl;
1273	1220	AstNodeErrorSetField err_set_field;
1274	1221	AstNodeResumeExpr resume_expr;
1275	1222	AstNodeAwaitExpr await_expr;
1276	1223	AstNodeSuspend suspend;
1277	1224	AstNodeAnyFrameType anyframe_type;
1278		AstNodeEnumLiteral enum_literal;
1279	1225	} data;
1280	1226
1281	1227	// This is a function for use in the debugger to print
...	...	@@ -1409,9 +1355,11 @@ struct ZigPackage {
1409	1355	struct RootStruct {
1410	1356	ZigPackage *package;
1411	1357	Buf *path; // relative to root_package->root_src_dir
1412		ZigList<size_t> *line_offsets;
1413	1358	Buf *source_code;
1414	1359	ZigLLVMDIFile *di_file;
	1360	size_t token_count;
	1361	TokenId *token_ids;
	1362	TokenLoc *token_locs;
1415	1363	};
1416	1364
1417	1365	enum StructSpecial {

src/stage1/analyze.cpp+87-68

...	...	@@ -6,9 +6,9 @@
6	6	*/
7	7
8	8	#include "analyze.hpp"
9		#include "ast_render.hpp"
10	9	#include "codegen.hpp"
11	10	#include "error.hpp"
	11	#include "astgen.hpp"
12	12	#include "ir.hpp"
13	13	#include "ir_print.hpp"
14	14	#include "os.hpp"
...	...	@@ -41,41 +41,44 @@ static bool is_top_level_struct(ZigType *import) {
41	41	return import->id == ZigTypeIdStruct && import->data.structure.root_struct != nullptr;
42	42	}
43	43
44		static ErrorMsg add_error_note_token(CodeGen g, ErrorMsg parent_msg, ZigType owner, Token token, Buf msg) {
	44	static ErrorMsg add_error_note_token(CodeGen g, ErrorMsg parent_msg, ZigType owner,
	45	TokenIndex token, Buf *msg)
	46	{
45	47	assert(is_top_level_struct(owner));
46	48	RootStruct *root_struct = owner->data.structure.root_struct;
47
48		ErrorMsg *err = err_msg_create_with_line(root_struct->path, token->start_line, token->start_column,
49		root_struct->source_code, root_struct->line_offsets, msg);
	49	uint32_t byte_offset = root_struct->token_locs[token].offset;
	50	ErrorMsg *err = err_msg_create_with_offset(root_struct->path, byte_offset,
	51	buf_ptr(root_struct->source_code), msg);
50	52
51	53	err_msg_add_note(parent_msg, err);
52	54	return err;
53	55	}
54	56
55		ErrorMsg add_token_error(CodeGen g, ZigType owner, Token token, Buf *msg) {
	57	ErrorMsg add_token_error_offset(CodeGen g, ZigType owner, TokenIndex token, Buf msg,
	58	uint32_t bad_index)
	59	{
56	60	assert(is_top_level_struct(owner));
57	61	RootStruct *root_struct = owner->data.structure.root_struct;
58		ErrorMsg *err = err_msg_create_with_line(root_struct->path, token->start_line, token->start_column,
59		root_struct->source_code, root_struct->line_offsets, msg);
	62	uint32_t byte_offset = root_struct->token_locs[token].offset + bad_index;
	63	ErrorMsg *err = err_msg_create_with_offset(root_struct->path, byte_offset,
	64	buf_ptr(root_struct->source_code), msg);
60	65
61	66	g->errors.append(err);
62	67	g->trace_err = err;
63	68	return err;
64	69	}
65	70
	71	ErrorMsg add_token_error(CodeGen g, ZigType owner, TokenIndex token, Buf msg) {
	72	return add_token_error_offset(g, owner, token, msg, 0);
	73	}
	74
66	75	ErrorMsg add_node_error(CodeGen g, AstNode node, Buf msg) {
67		Token fake_token;
68		fake_token.start_line = node->line;
69		fake_token.start_column = node->column;
70	76	node->already_traced_this_node = true;
71		return add_token_error(g, node->owner, &fake_token, msg);
	77	return add_token_error(g, node->owner, node->main_token, msg);
72	78	}
73	79
74	80	ErrorMsg add_error_note(CodeGen g, ErrorMsg parent_msg, const AstNode node, Buf *msg) {
75		Token fake_token;
76		fake_token.start_line = node->line;
77		fake_token.start_column = node->column;
78		return add_error_note_token(g, parent_msg, node->owner, &fake_token, msg);
	81	return add_error_note_token(g, parent_msg, node->owner, node->main_token, msg);
79	82	}
80	83
81	84	ZigType *new_type_table_entry(ZigTypeId id) {
...	...	@@ -913,13 +916,27 @@ ZigType get_slice_type(CodeGen g, ZigType *ptr_type) {
913	916	return entry;
914	917	}
915	918
916		ZigType get_opaque_type(CodeGen g, Scope scope, AstNode source_node, const char full_name, Buf bare_name) {
	919	static uint32_t node_line_onebased(AstNode *node) {
	920	RootStruct *root_struct = node->owner->data.structure.root_struct;
	921	assert(node->main_token < root_struct->token_count);
	922	return root_struct->token_locs[node->main_token].line + 1;
	923	}
	924
	925	static uint32_t node_column_onebased(AstNode *node) {
	926	RootStruct *root_struct = node->owner->data.structure.root_struct;
	927	assert(node->main_token < root_struct->token_count);
	928	return root_struct->token_locs[node->main_token].column + 1;
	929	}
	930
	931	ZigType get_opaque_type(CodeGen g, Scope scope, AstNode source_node, const char *full_name,
	932	Buf *bare_name)
	933	{
917	934	ZigType *entry = new_type_table_entry(ZigTypeIdOpaque);
918	935
919	936	buf_init_from_str(&entry->name, full_name);
920	937
921	938	ZigType *import = scope ? get_scope_import(scope) : nullptr;
922		unsigned line = source_node ? (unsigned)(source_node->line + 1) : 0;
	939	unsigned line = source_node ? node_line_onebased(source_node) : 0;
923	940
924	941	// Note: duplicated in get_partial_container_type
925	942	entry->llvm_type = LLVMInt8Type();
...	...	@@ -1142,7 +1159,7 @@ ZigType get_partial_container_type(CodeGen g, Scope *scope, ContainerKind kind
1142	1159	break;
1143	1160	case ContainerKindOpaque: {
1144	1161	ZigType *import = scope ? get_scope_import(scope) : nullptr;
1145		unsigned line = decl_node ? (unsigned)(decl_node->line + 1) : 0;
	1162	unsigned line = decl_node ? node_line_onebased(decl_node) : 0;
1146	1163	// Note: duplicated in get_opaque_type
1147	1164	entry->llvm_type = LLVMInt8Type();
1148	1165	entry->llvm_di_type = ZigLLVMCreateDebugForwardDeclType(g->dbuilder,
...	...	@@ -1584,8 +1601,9 @@ static OnePossibleValue type_val_resolve_has_one_possible_value(CodeGen *g, ZigV
1584	1601	ZigType analyze_type_expr(CodeGen g, Scope scope, AstNode node) {
1585	1602	Error err;
1586	1603	// Hot path for simple identifiers, to avoid unnecessary memory allocations.
1587		if (node->type == NodeTypeSymbol) {
1588		Buf *variable_name = node->data.symbol_expr.symbol;
	1604	if (node->type == NodeTypeIdentifier) {
	1605	RootStruct *root_struct = node->owner->data.structure.root_struct;
	1606	Buf *variable_name = token_identifier_buf(root_struct, node->main_token);
1589	1607	if (buf_eql_str(variable_name, "_"))
1590	1608	goto abort_hot_path;
1591	1609	ZigType *primitive_type;
...	...	@@ -2034,7 +2052,7 @@ static ZigType analyze_fn_type(CodeGen g, AstNode proto_node, Scope child_sc
2034	2052	buf_sprintf("var args only allowed in functions with C calling convention"));
2035	2053	return g->builtin_types.entry_invalid;
2036	2054	}
2037		} else if (param_node->data.param_decl.anytype_token != nullptr) {
	2055	} else if (param_node->data.param_decl.anytype_token != 0) {
2038	2056	if (!calling_convention_allows_zig_types(fn_type_id.cc)) {
2039	2057	add_node_error(g, param_node,
2040	2058	buf_sprintf("parameter of type 'anytype' not allowed in function with calling convention '%s'",
...	...	@@ -3021,7 +3039,7 @@ static Error resolve_struct_zero_bits(CodeGen g, ZigType struct_type) {
3021	3039	field_node = decl_node->data.container_decl.fields.at(i);
3022	3040	type_struct_field->name = field_node->data.struct_field.name;
3023	3041	type_struct_field->decl_node = field_node;
3024		if (field_node->data.struct_field.comptime_token != nullptr) {
	3042	if (field_node->data.struct_field.comptime_token != 0) {
3025	3043	if (field_node->data.struct_field.value == nullptr) {
3026	3044	add_token_error(g, field_node->owner,
3027	3045	field_node->data.struct_field.comptime_token,
...	...	@@ -4042,7 +4060,7 @@ void scan_decls(CodeGen g, ScopeDecls decls_scope, AstNode *node) {
4042	4060	case NodeTypeBoolLiteral:
4043	4061	case NodeTypeNullLiteral:
4044	4062	case NodeTypeUndefinedLiteral:
4045		case NodeTypeSymbol:
	4063	case NodeTypeIdentifier:
4046	4064	case NodeTypePrefixOpExpr:
4047	4065	case NodeTypePointerType:
4048	4066	case NodeTypeIfBoolExpr:
...	...	@@ -4238,7 +4256,7 @@ static void resolve_decl_var(CodeGen g, TldVar tld_var, bool allow_lazy) {
4238	4256	bool is_const = var_decl->is_const;
4239	4257	bool is_extern = var_decl->is_extern;
4240	4258	bool is_export = var_decl->is_export;
4241		bool is_thread_local = var_decl->threadlocal_tok != nullptr;
	4259	bool is_thread_local = var_decl->threadlocal_tok != 0;
4242	4260
4243	4261	ZigType *explicit_type = nullptr;
4244	4262	if (var_decl->type) {
...	...	@@ -5225,8 +5243,6 @@ static void analyze_fn_body(CodeGen g, ZigFn fn_table_entry) {
5225	5243	}
5226	5244
5227	5245	if (g->verbose_ir) {
5228		fprintf(stderr, "\n");
5229		ast_render(stderr, fn_table_entry->body_node, 4);
5230	5246	fprintf(stderr, "\nfn %s() { // (IR)\n", buf_ptr(&fn_table_entry->symbol_name));
5231	5247	ir_print_src(g, stderr, fn_table_entry->ir_executable, 4);
5232	5248	fprintf(stderr, "}\n");
...	...	@@ -5238,33 +5254,17 @@ static void analyze_fn_body(CodeGen g, ZigFn fn_table_entry) {
5238	5254	ZigType add_source_file(CodeGen g, ZigPackage package, Buf resolved_path, Buf *source_code,
5239	5255	SourceKind source_kind)
5240	5256	{
5241		if (g->verbose_tokenize) {
5242		fprintf(stderr, "\nOriginal Source (%s):\n", buf_ptr(resolved_path));
5243		fprintf(stderr, "----------------\n");
5244		fprintf(stderr, "%s\n", buf_ptr(source_code));
5245
5246		fprintf(stderr, "\nTokens:\n");
5247		fprintf(stderr, "---------\n");
5248		}
5249
5250	5257	Tokenization tokenization = {0};
5251		tokenize(source_code, &tokenization);
	5258	tokenize(buf_ptr(source_code), &tokenization);
5252	5259
5253	5260	if (tokenization.err) {
5254		ErrorMsg *err = err_msg_create_with_line(resolved_path, tokenization.err_line, tokenization.err_column,
5255		source_code, tokenization.line_offsets, tokenization.err);
	5261	ErrorMsg *err = err_msg_create_with_offset(resolved_path, tokenization.err_byte_offset,
	5262	buf_ptr(source_code), tokenization.err);
5256	5263
5257	5264	print_err_msg(err, g->err_color);
5258	5265	exit(1);
5259	5266	}
5260	5267
5261		if (g->verbose_tokenize) {
5262		print_tokens(source_code, tokenization.tokens);
5263
5264		fprintf(stderr, "\nAST:\n");
5265		fprintf(stderr, "------\n");
5266		}
5267
5268	5268	Buf *src_dirname = buf_alloc();
5269	5269	Buf *src_basename = buf_alloc();
5270	5270	os_path_split(resolved_path, src_dirname, src_basename);
...	...	@@ -5297,9 +5297,20 @@ ZigType add_source_file(CodeGen g, ZigPackage package, Buf resolved_path, Bu
5297	5297	RootStruct *root_struct = heap::c_allocator.create<RootStruct>();
5298	5298	root_struct->package = package;
5299	5299	root_struct->source_code = source_code;
5300		root_struct->line_offsets = tokenization.line_offsets;
5301	5300	root_struct->path = resolved_path;
5302	5301	root_struct->di_file = ZigLLVMCreateFile(g->dbuilder, buf_ptr(src_basename), buf_ptr(src_dirname));
	5302
	5303	assert(tokenization.ids.length == tokenization.locs.length);
	5304	size_t token_count = tokenization.ids.length;
	5305	root_struct->token_count = token_count;
	5306	root_struct->token_ids = g->pass1_arena->allocate<TokenId>(token_count);
	5307	memcpy(root_struct->token_ids, tokenization.ids.items, token_count * sizeof(TokenId));
	5308	root_struct->token_locs = g->pass1_arena->allocate<TokenLoc>(token_count);
	5309	memcpy(root_struct->token_locs, tokenization.locs.items, token_count * sizeof(TokenLoc));
	5310
	5311	tokenization.ids.deinit();
	5312	tokenization.locs.deinit();
	5313
5303	5314	ZigType *import_entry = get_root_container_type(g, buf_ptr(namespace_name), bare_name, root_struct);
5304	5315	if (source_kind == SourceKindRoot) {
5305	5316	assert(g->root_import == nullptr);
...	...	@@ -5307,14 +5318,11 @@ ZigType add_source_file(CodeGen g, ZigPackage package, Buf resolved_path, Bu
5307	5318	}
5308	5319	g->import_table.put(resolved_path, import_entry);
5309	5320
5310		AstNode *root_node = ast_parse(source_code, tokenization.tokens, import_entry, g->err_color);
	5321	AstNode *root_node = ast_parse(source_code, import_entry, g->err_color);
5311	5322	assert(root_node != nullptr);
5312	5323	assert(root_node->type == NodeTypeContainerDecl);
5313	5324	import_entry->data.structure.decl_node = root_node;
5314	5325	import_entry->data.structure.decls_scope->base.source_node = root_node;
5315		if (g->verbose_ast) {
5316		ast_print(stderr, root_node, 0);
5317		}
5318	5326
5319	5327	for (size_t decl_i = 0; decl_i < root_node->data.container_decl.decls.length; decl_i += 1) {
5320	5328	AstNode *top_level_decl = root_node->data.container_decl.decls.at(decl_i);
...	...	@@ -6254,9 +6262,12 @@ ReqCompTime type_requires_comptime(CodeGen g, ZigType ty) {
6254	6262	zig_unreachable();
6255	6263	}
6256	6264
6257		void init_const_str_lit(CodeGen g, ZigValue const_val, Buf *str) {
	6265	void init_const_str_lit(CodeGen g, ZigValue const_val, Buf *str, bool move_str) {
6258	6266	auto entry = g->string_literals_table.maybe_get(str);
6259	6267	if (entry != nullptr) {
	6268	if (move_str) {
	6269	buf_destroy(str);
	6270	}
6260	6271	memcpy(const_val, entry->value, sizeof(ZigValue));
6261	6272	return;
6262	6273	}
...	...	@@ -6280,7 +6291,7 @@ void init_const_str_lit(CodeGen g, ZigValue const_val, Buf *str) {
6280	6291
6281	6292	ZigValue create_const_str_lit(CodeGen g, Buf *str) {
6282	6293	ZigValue *const_val = g->pass1_arena->create<ZigValue>();
6283		init_const_str_lit(g, const_val, str);
	6294	init_const_str_lit(g, const_val, str, false);
6284	6295	return const_val;
6285	6296	}
6286	6297
...	...	@@ -8626,7 +8637,7 @@ static void resolve_llvm_types_struct(CodeGen g, ZigType struct_type, ResolveS
8626	8637	ZigType *import = get_scope_import(scope);
8627	8638	di_file = import->data.structure.root_struct->di_file;
8628	8639	di_scope = ZigLLVMFileToScope(di_file);
8629		line = decl_node->line + 1;
	8640	line = node_line_onebased(decl_node);
8630	8641	} else {
8631	8642	di_file = nullptr;
8632	8643	di_scope = ZigLLVMCompileUnitToScope(g->compile_unit);
...	...	@@ -8830,7 +8841,7 @@ static void resolve_llvm_types_struct(CodeGen g, ZigType struct_type, ResolveS
8830	8841	unsigned line;
8831	8842	if (decl_node != nullptr) {
8832	8843	AstNode *field_node = field->decl_node;
8833		line = field_node->line + 1;
	8844	line = node_line_onebased(field_node);
8834	8845	} else {
8835	8846	line = 0;
8836	8847	}
...	...	@@ -8881,7 +8892,7 @@ static ZigLLVMDIType make_empty_namespace_llvm_di_type(CodeGen g, ZigType *imp
8881	8892	return ZigLLVMCreateDebugStructType(g->dbuilder,
8882	8893	ZigLLVMFileToScope(import->data.structure.root_struct->di_file),
8883	8894	name,
8884		import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
	8895	import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
8885	8896	debug_size_in_bits,
8886	8897	debug_align_in_bits,
8887	8898	ZigLLVM_DIFlags_Zero,
...	...	@@ -8926,7 +8937,7 @@ static void resolve_llvm_types_enum(CodeGen g, ZigType enum_type, ResolveStatu
8926	8937	uint64_t tag_debug_align_in_bits = 8*tag_int_type->abi_align;
8927	8938	ZigLLVMDIType *tag_di_type = ZigLLVMCreateDebugEnumerationType(g->dbuilder,
8928	8939	ZigLLVMFileToScope(import->data.structure.root_struct->di_file), buf_ptr(&enum_type->name),
8929		import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
	8940	import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
8930	8941	tag_debug_size_in_bits,
8931	8942	tag_debug_align_in_bits,
8932	8943	di_enumerators, field_count,
...	...	@@ -8966,12 +8977,12 @@ static void resolve_llvm_types_union(CodeGen g, ZigType union_type, ResolveSta
8966	8977
8967	8978	if (union_type->data.unionation.resolve_status < ResolveStatusLLVMFwdDecl) {
8968	8979	union_type->llvm_type = LLVMStructCreateNamed(LLVMGetGlobalContext(), buf_ptr(&union_type->name));
8969		size_t line = decl_node ? decl_node->line : 0;
	8980	unsigned line = decl_node ? node_line_onebased(decl_node) : 0;
8970	8981	unsigned dwarf_kind = ZigLLVMTag_DW_structure_type();
8971	8982	union_type->llvm_di_type = ZigLLVMCreateReplaceableCompositeType(g->dbuilder,
8972	8983	dwarf_kind, buf_ptr(&union_type->name),
8973	8984	ZigLLVMFileToScope(import->data.structure.root_struct->di_file),
8974		import->data.structure.root_struct->di_file, (unsigned)(line + 1));
	8985	import->data.structure.root_struct->di_file, line);
8975	8986
8976	8987	union_type->data.unionation.resolve_status = ResolveStatusLLVMFwdDecl;
8977	8988	if (ResolveStatusLLVMFwdDecl >= wanted_resolve_status) return;
...	...	@@ -8992,7 +9003,7 @@ static void resolve_llvm_types_union(CodeGen g, ZigType union_type, ResolveSta
8992	9003	AstNode *field_node = union_field->decl_node;
8993	9004	union_inner_di_types[union_field->gen_index] = ZigLLVMCreateDebugMemberType(g->dbuilder,
8994	9005	ZigLLVMTypeToScope(union_type->llvm_di_type), buf_ptr(union_field->enum_field->name),
8995		import->data.structure.root_struct->di_file, (unsigned)(field_node->line + 1),
	9006	import->data.structure.root_struct->di_file, node_line_onebased(field_node),
8996	9007	store_size_in_bits,
8997	9008	abi_align_in_bits,
8998	9009	0,
...	...	@@ -9022,7 +9033,7 @@ static void resolve_llvm_types_union(CodeGen g, ZigType union_type, ResolveSta
9022	9033	// create debug type for union
9023	9034	ZigLLVMDIType *replacement_di_type = ZigLLVMCreateDebugUnionType(g->dbuilder,
9024	9035	ZigLLVMFileToScope(import->data.structure.root_struct->di_file), buf_ptr(&union_type->name),
9025		import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
	9036	import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
9026	9037	union_type->data.unionation.union_abi_size * 8,
9027	9038	most_aligned_union_member->align * 8,
9028	9039	ZigLLVM_DIFlags_Zero, union_inner_di_types,
...	...	@@ -9057,7 +9068,7 @@ static void resolve_llvm_types_union(CodeGen g, ZigType union_type, ResolveSta
9057	9068	// create debug type for union
9058	9069	ZigLLVMDIType *union_di_type = ZigLLVMCreateDebugUnionType(g->dbuilder,
9059	9070	ZigLLVMTypeToScope(union_type->llvm_di_type), "AnonUnion",
9060		import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
	9071	import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
9061	9072	most_aligned_union_member->type_entry->size_in_bits, 8*most_aligned_union_member->align,
9062	9073	ZigLLVM_DIFlags_Zero, union_inner_di_types, gen_field_count, 0, "");
9063	9074
...	...	@@ -9068,7 +9079,7 @@ static void resolve_llvm_types_union(CodeGen g, ZigType union_type, ResolveSta
9068	9079
9069	9080	ZigLLVMDIType *union_member_di_type = ZigLLVMCreateDebugMemberType(g->dbuilder,
9070	9081	ZigLLVMTypeToScope(union_type->llvm_di_type), "payload",
9071		import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
	9082	import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
9072	9083	most_aligned_union_member->type_entry->size_in_bits,
9073	9084	8*most_aligned_union_member->align,
9074	9085	union_offset_in_bits,
...	...	@@ -9079,7 +9090,7 @@ static void resolve_llvm_types_union(CodeGen g, ZigType union_type, ResolveSta
9079	9090
9080	9091	ZigLLVMDIType *tag_member_di_type = ZigLLVMCreateDebugMemberType(g->dbuilder,
9081	9092	ZigLLVMTypeToScope(union_type->llvm_di_type), "tag",
9082		import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
	9093	import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
9083	9094	tag_debug_size_in_bits,
9084	9095	tag_debug_align_in_bits,
9085	9096	tag_offset_in_bits,
...	...	@@ -9094,7 +9105,7 @@ static void resolve_llvm_types_union(CodeGen g, ZigType union_type, ResolveSta
9094	9105	ZigLLVMDIType *replacement_di_type = ZigLLVMCreateDebugStructType(g->dbuilder,
9095	9106	ZigLLVMFileToScope(import->data.structure.root_struct->di_file),
9096	9107	buf_ptr(&union_type->name),
9097		import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
	9108	import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
9098	9109	debug_size_in_bits,
9099	9110	debug_align_in_bits,
9100	9111	ZigLLVM_DIFlags_Zero, nullptr, di_root_members, 2, 0, nullptr, "");
...	...	@@ -9774,9 +9785,9 @@ void src_assert_impl(bool ok, AstNode source_node, char const file, unsigned i
9774	9785	if (source_node == nullptr) {
9775	9786	fprintf(stderr, "when analyzing (unknown source location) ");
9776	9787	} else {
9777		fprintf(stderr, "when analyzing %s:%u:%u ",
9778		buf_ptr(source_node->owner->data.structure.root_struct->path),
9779		(unsigned)source_node->line + 1, (unsigned)source_node->column + 1);
	9788	RootStruct *root_struct = source_node->owner->data.structure.root_struct;
	9789	fprintf(stderr, "when analyzing %s:%u:%u ", buf_ptr(root_struct->path),
	9790	node_line_onebased(source_node), node_column_onebased(source_node));
9780	9791	}
9781	9792	fprintf(stderr, "in compiler source at %s:%u: ", file, line);
9782	9793	const char *msg = "assertion failed. This is a bug in the Zig compiler.";
...	...	@@ -9842,6 +9853,14 @@ Error analyze_import(CodeGen g, ZigType source_import, Buf *import_target_str,
9842	9853	return ErrorNone;
9843	9854	}
9844	9855
	9856	void AstNode::src() {
	9857	RootStruct *root_struct = this->owner->data.structure.root_struct;
	9858	uint32_t line = root_struct->token_locs[this->main_token].line + 1;
	9859	uint32_t column = root_struct->token_locs[this->main_token].column + 1;
	9860	fprintf(stderr, "%s:%" PRIu32 ":%" PRIu32 "\n",
	9861	buf_ptr(root_struct->path),
	9862	line, column);
	9863	}
9845	9864
9846	9865	void IrExecutableSrc::src() {
9847	9866	if (this->source_node != nullptr) {

src/stage1/analyze.hpp+4-2

...	...	@@ -12,7 +12,9 @@
12	12
13	13	void semantic_analyze(CodeGen *g);
14	14	ErrorMsg add_node_error(CodeGen g, AstNode node, Buf msg);
15		ErrorMsg add_token_error(CodeGen g, ZigType owner, Token token, Buf *msg);
	15	ErrorMsg add_token_error(CodeGen g, ZigType owner, TokenIndex token, Buf msg);
	16	ErrorMsg add_token_error_offset(CodeGen g, ZigType owner, TokenIndex token, Buf msg,
	17	uint32_t bad_index);
16	18	ErrorMsg add_error_note(CodeGen g, ErrorMsg parent_msg, const AstNode node, Buf *msg);
17	19	ZigType *new_type_table_entry(ZigTypeId id);
18	20	ZigType get_fn_frame_type(CodeGen g, ZigFn *fn);
...	...	@@ -140,7 +142,7 @@ Scope create_runtime_scope(CodeGen g, AstNode node, Scope parent, IrInstSrc
140	142	Scope create_typeof_scope(CodeGen g, AstNode node, Scope parent);
141	143	ScopeExpr create_expr_scope(CodeGen g, AstNode node, Scope parent);
142	144
143		void init_const_str_lit(CodeGen g, ZigValue const_val, Buf *str);
	145	void init_const_str_lit(CodeGen g, ZigValue const_val, Buf *str, bool move_str);
144	146	ZigValue create_const_str_lit(CodeGen g, Buf *str);
145	147
146	148	void init_const_bigint(ZigValue const_val, ZigType type, const BigInt *bigint);

src/stage1/ast_render.cpp deleted-1241

...	...	@@ -1,1241 +0,0 @@
1		/*
2		* Copyright (c) 2016 Andrew Kelley
3		*
4		* This file is part of zig, which is MIT licensed.
5		* See http://opensource.org/licenses/MIT
6		*/
7
8		#include "analyze.hpp"
9		#include "ast_render.hpp"
10		#include "os.hpp"
11
12		#include <stdio.h>
13
14		static const char *bin_op_str(BinOpType bin_op) {
15		switch (bin_op) {
16		case BinOpTypeInvalid: return "(invalid)";
17		case BinOpTypeBoolOr: return "or";
18		case BinOpTypeBoolAnd: return "and";
19		case BinOpTypeCmpEq: return "==";
20		case BinOpTypeCmpNotEq: return "!=";
21		case BinOpTypeCmpLessThan: return "<";
22		case BinOpTypeCmpGreaterThan: return ">";
23		case BinOpTypeCmpLessOrEq: return "<=";
24		case BinOpTypeCmpGreaterOrEq: return ">=";
25		case BinOpTypeBinOr: return "\|";
26		case BinOpTypeBinXor: return "^";
27		case BinOpTypeBinAnd: return "&";
28		case BinOpTypeBitShiftLeft: return "<<";
29		case BinOpTypeBitShiftRight: return ">>";
30		case BinOpTypeAdd: return "+";
31		case BinOpTypeAddWrap: return "+%";
32		case BinOpTypeSub: return "-";
33		case BinOpTypeSubWrap: return "-%";
34		case BinOpTypeMult: return "*";
35		case BinOpTypeMultWrap: return "*%";
36		case BinOpTypeDiv: return "/";
37		case BinOpTypeMod: return "%";
38		case BinOpTypeAssign: return "=";
39		case BinOpTypeAssignTimes: return "*=";
40		case BinOpTypeAssignTimesWrap: return "*%=";
41		case BinOpTypeAssignDiv: return "/=";
42		case BinOpTypeAssignMod: return "%=";
43		case BinOpTypeAssignPlus: return "+=";
44		case BinOpTypeAssignPlusWrap: return "+%=";
45		case BinOpTypeAssignMinus: return "-=";
46		case BinOpTypeAssignMinusWrap: return "-%=";
47		case BinOpTypeAssignBitShiftLeft: return "<<=";
48		case BinOpTypeAssignBitShiftRight: return ">>=";
49		case BinOpTypeAssignBitAnd: return "&=";
50		case BinOpTypeAssignBitXor: return "^=";
51		case BinOpTypeAssignBitOr: return "\|=";
52		case BinOpTypeUnwrapOptional: return "orelse";
53		case BinOpTypeArrayCat: return "++";
54		case BinOpTypeArrayMult: return "**";
55		case BinOpTypeErrorUnion: return "!";
56		case BinOpTypeMergeErrorSets: return "\|\|";
57		}
58		zig_unreachable();
59		}
60
61		static const char *prefix_op_str(PrefixOp prefix_op) {
62		switch (prefix_op) {
63		case PrefixOpInvalid: return "(invalid)";
64		case PrefixOpNegation: return "-";
65		case PrefixOpNegationWrap: return "-%";
66		case PrefixOpBoolNot: return "!";
67		case PrefixOpBinNot: return "~";
68		case PrefixOpOptional: return "?";
69		case PrefixOpAddrOf: return "&";
70		}
71		zig_unreachable();
72		}
73
74		static const char *visib_mod_string(VisibMod mod) {
75		switch (mod) {
76		case VisibModPub: return "pub ";
77		case VisibModPrivate: return "";
78		}
79		zig_unreachable();
80		}
81
82		static const char *return_string(ReturnKind kind) {
83		switch (kind) {
84		case ReturnKindUnconditional: return "return";
85		case ReturnKindError: return "try";
86		}
87		zig_unreachable();
88		}
89
90		static const char *defer_string(ReturnKind kind) {
91		switch (kind) {
92		case ReturnKindUnconditional: return "defer";
93		case ReturnKindError: return "errdefer";
94		}
95		zig_unreachable();
96		}
97
98		static const char *layout_string(ContainerLayout layout) {
99		switch (layout) {
100		case ContainerLayoutAuto: return "";
101		case ContainerLayoutExtern: return "extern ";
102		case ContainerLayoutPacked: return "packed ";
103		}
104		zig_unreachable();
105		}
106
107		static const char *extern_string(bool is_extern) {
108		return is_extern ? "extern " : "";
109		}
110
111		static const char *export_string(bool is_export) {
112		return is_export ? "export " : "";
113		}
114
115		//static const char *calling_convention_string(CallingConvention cc) {
116		// switch (cc) {
117		// case CallingConventionUnspecified: return "";
118		// case CallingConventionC: return "extern ";
119		// case CallingConventionCold: return "coldcc ";
120		// case CallingConventionNaked: return "nakedcc ";
121		// case CallingConventionStdcall: return "stdcallcc ";
122		// }
123		// zig_unreachable();
124		//}
125
126		static const char *inline_string(FnInline fn_inline) {
127		switch (fn_inline) {
128		case FnInlineAlways: return "inline ";
129		case FnInlineNever: return "noinline ";
130		case FnInlineAuto: return "";
131		}
132		zig_unreachable();
133		}
134
135		static const char *const_or_var_string(bool is_const) {
136		return is_const ? "const" : "var";
137		}
138
139		static const char thread_local_string(Token tok) {
140		return (tok == nullptr) ? "" : "threadlocal ";
141		}
142
143		static const char token_to_ptr_len_str(Token tok) {
144		assert(tok != nullptr);
145		switch (tok->id) {
146		case TokenIdStar:
147		case TokenIdStarStar:
148		return "*";
149		case TokenIdLBracket:
150		return "[*]";
151		case TokenIdSymbol:
152		return "[*c]";
153		default:
154		zig_unreachable();
155		}
156		}
157
158		static const char *node_type_str(NodeType node_type) {
159		switch (node_type) {
160		case NodeTypeFnDef:
161		return "FnDef";
162		case NodeTypeFnProto:
163		return "FnProto";
164		case NodeTypeParamDecl:
165		return "ParamDecl";
166		case NodeTypeBlock:
167		return "Block";
168		case NodeTypeGroupedExpr:
169		return "Parens";
170		case NodeTypeBinOpExpr:
171		return "BinOpExpr";
172		case NodeTypeCatchExpr:
173		return "CatchExpr";
174		case NodeTypeFnCallExpr:
175		return "FnCallExpr";
176		case NodeTypeArrayAccessExpr:
177		return "ArrayAccessExpr";
178		case NodeTypeSliceExpr:
179		return "SliceExpr";
180		case NodeTypeReturnExpr:
181		return "ReturnExpr";
182		case NodeTypeDefer:
183		return "Defer";
184		case NodeTypeVariableDeclaration:
185		return "VariableDeclaration";
186		case NodeTypeTestDecl:
187		return "TestDecl";
188		case NodeTypeIntLiteral:
189		return "IntLiteral";
190		case NodeTypeFloatLiteral:
191		return "FloatLiteral";
192		case NodeTypeStringLiteral:
193		return "StringLiteral";
194		case NodeTypeCharLiteral:
195		return "CharLiteral";
196		case NodeTypeSymbol:
197		return "Symbol";
198		case NodeTypePrefixOpExpr:
199		return "PrefixOpExpr";
200		case NodeTypeUsingNamespace:
201		return "UsingNamespace";
202		case NodeTypeBoolLiteral:
203		return "BoolLiteral";
204		case NodeTypeNullLiteral:
205		return "NullLiteral";
206		case NodeTypeUndefinedLiteral:
207		return "UndefinedLiteral";
208		case NodeTypeIfBoolExpr:
209		return "IfBoolExpr";
210		case NodeTypeWhileExpr:
211		return "WhileExpr";
212		case NodeTypeForExpr:
213		return "ForExpr";
214		case NodeTypeSwitchExpr:
215		return "SwitchExpr";
216		case NodeTypeSwitchProng:
217		return "SwitchProng";
218		case NodeTypeSwitchRange:
219		return "SwitchRange";
220		case NodeTypeCompTime:
221		return "CompTime";
222		case NodeTypeNoSuspend:
223		return "NoSuspend";
224		case NodeTypeBreak:
225		return "Break";
226		case NodeTypeContinue:
227		return "Continue";
228		case NodeTypeUnreachable:
229		return "Unreachable";
230		case NodeTypeAsmExpr:
231		return "AsmExpr";
232		case NodeTypeFieldAccessExpr:
233		return "FieldAccessExpr";
234		case NodeTypePtrDeref:
235		return "PtrDerefExpr";
236		case NodeTypeUnwrapOptional:
237		return "UnwrapOptional";
238		case NodeTypeContainerDecl:
239		return "ContainerDecl";
240		case NodeTypeStructField:
241		return "StructField";
242		case NodeTypeStructValueField:
243		return "StructValueField";
244		case NodeTypeContainerInitExpr:
245		return "ContainerInitExpr";
246		case NodeTypeArrayType:
247		return "ArrayType";
248		case NodeTypeInferredArrayType:
249		return "InferredArrayType";
250		case NodeTypeErrorType:
251		return "ErrorType";
252		case NodeTypeIfErrorExpr:
253		return "IfErrorExpr";
254		case NodeTypeIfOptional:
255		return "IfOptional";
256		case NodeTypeErrorSetDecl:
257		return "ErrorSetDecl";
258		case NodeTypeResume:
259		return "Resume";
260		case NodeTypeAwaitExpr:
261		return "AwaitExpr";
262		case NodeTypeSuspend:
263		return "Suspend";
264		case NodeTypePointerType:
265		return "PointerType";
266		case NodeTypeAnyFrameType:
267		return "AnyFrameType";
268		case NodeTypeEnumLiteral:
269		return "EnumLiteral";
270		case NodeTypeErrorSetField:
271		return "ErrorSetField";
272		case NodeTypeAnyTypeField:
273		return "AnyTypeField";
274		}
275		zig_unreachable();
276		}
277
278		struct AstPrint {
279		int indent;
280		FILE *f;
281		};
282
283		static void ast_print_visit(AstNode *node_ptr, void context) {
284		AstNode node = node_ptr;
285		AstPrint ap = (AstPrint )context;
286
287		for (int i = 0; i < ap->indent; i += 1) {
288		fprintf(ap->f, " ");
289		}
290
291		fprintf(ap->f, "%s\n", node_type_str(node->type));
292
293		AstPrint new_ap;
294		new_ap.indent = ap->indent + 2;
295		new_ap.f = ap->f;
296
297		ast_visit_node_children(node, ast_print_visit, &new_ap);
298		}
299
300		void ast_print(FILE f, AstNode node, int indent) {
301		AstPrint ap;
302		ap.indent = indent;
303		ap.f = f;
304		ast_visit_node_children(node, ast_print_visit, &ap);
305		}
306
307
308		struct AstRender {
309		int indent;
310		int indent_size;
311		FILE *f;
312		};
313
314		static void print_indent(AstRender *ar) {
315		for (int i = 0; i < ar->indent; i += 1) {
316		fprintf(ar->f, " ");
317		}
318		}
319
320		static bool is_alpha_under(uint8_t c) {
321		return (c >= 'a' && c <= 'z') \|\|
322		(c >= 'A' && c <= 'Z') \|\| c == '_';
323		}
324
325		static bool is_digit(uint8_t c) {
326		return (c >= '0' && c <= '9');
327		}
328
329		static bool is_printable(uint8_t c) {
330		if (c == 0) {
331		return false;
332		}
333		static const uint8_t printables[] =
334		" abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.~`!@#$%^&*()_-+=\\{}[];'\"?/<>,:";
335		for (size_t i = 0; i < array_length(printables); i += 1) {
336		if (c == printables[i]) return true;
337		}
338		return false;
339		}
340
341		static void string_literal_escape(Buf source, Buf dest) {
342		buf_resize(dest, 0);
343		for (size_t i = 0; i < buf_len(source); i += 1) {
344		uint8_t c = ((uint8_t)buf_ptr(source) + i);
345		if (c == '\'') {
346		buf_append_str(dest, "\\'");
347		} else if (c == '"') {
348		buf_append_str(dest, "\\\"");
349		} else if (c == '\\') {
350		buf_append_str(dest, "\\\\");
351		} else if (c == '\n') {
352		buf_append_str(dest, "\\n");
353		} else if (c == '\r') {
354		buf_append_str(dest, "\\r");
355		} else if (c == '\t') {
356		buf_append_str(dest, "\\t");
357		} else if (is_printable(c)) {
358		buf_append_char(dest, c);
359		} else {
360		buf_appendf(dest, "\\x%02x", (int)c);
361		}
362		}
363		}
364
365		static bool is_valid_bare_symbol(Buf *symbol) {
366		if (buf_len(symbol) == 0) {
367		return false;
368		}
369		uint8_t first_char = *buf_ptr(symbol);
370		if (!is_alpha_under(first_char)) {
371		return false;
372		}
373		for (size_t i = 1; i < buf_len(symbol); i += 1) {
374		uint8_t c = ((uint8_t)buf_ptr(symbol) + i);
375		if (!is_alpha_under(c) && !is_digit(c)) {
376		return false;
377		}
378		}
379		return true;
380		}
381
382		static void print_symbol(AstRender ar, Buf symbol) {
383		if (is_zig_keyword(symbol)) {
384		fprintf(ar->f, "@\"%s\"", buf_ptr(symbol));
385		return;
386		}
387		if (is_valid_bare_symbol(symbol)) {
388		fprintf(ar->f, "%s", buf_ptr(symbol));
389		return;
390		}
391		Buf escaped = BUF_INIT;
392		string_literal_escape(symbol, &escaped);
393		fprintf(ar->f, "@\"%s\"", buf_ptr(&escaped));
394		}
395
396		static bool statement_terminates_without_semicolon(AstNode *node) {
397		switch (node->type) {
398		case NodeTypeIfBoolExpr:
399		if (node->data.if_bool_expr.else_node)
400		return statement_terminates_without_semicolon(node->data.if_bool_expr.else_node);
401		return node->data.if_bool_expr.then_block->type == NodeTypeBlock;
402		case NodeTypeIfErrorExpr:
403		if (node->data.if_err_expr.else_node)
404		return statement_terminates_without_semicolon(node->data.if_err_expr.else_node);
405		return node->data.if_err_expr.then_node->type == NodeTypeBlock;
406		case NodeTypeIfOptional:
407		if (node->data.test_expr.else_node)
408		return statement_terminates_without_semicolon(node->data.test_expr.else_node);
409		return node->data.test_expr.then_node->type == NodeTypeBlock;
410		case NodeTypeWhileExpr:
411		return node->data.while_expr.body->type == NodeTypeBlock;
412		case NodeTypeForExpr:
413		return node->data.for_expr.body->type == NodeTypeBlock;
414		case NodeTypeCompTime:
415		return node->data.comptime_expr.expr->type == NodeTypeBlock;
416		case NodeTypeDefer:
417		return node->data.defer.expr->type == NodeTypeBlock;
418		case NodeTypeSuspend:
419		return node->data.suspend.block != nullptr && node->data.suspend.block->type == NodeTypeBlock;
420		case NodeTypeSwitchExpr:
421		case NodeTypeBlock:
422		return true;
423		default:
424		return false;
425		}
426		}
427
428		static void render_node_extra(AstRender ar, AstNode node, bool grouped);
429
430		static void render_node_grouped(AstRender ar, AstNode node) {
431		return render_node_extra(ar, node, true);
432		}
433
434		static void render_node_ungrouped(AstRender ar, AstNode node) {
435		return render_node_extra(ar, node, false);
436		}
437
438		static void render_node_extra(AstRender ar, AstNode node, bool grouped) {
439		switch (node->type) {
440		case NodeTypeSwitchProng:
441		case NodeTypeSwitchRange:
442		case NodeTypeStructValueField:
443		zig_unreachable();
444		case NodeTypeFnProto:
445		{
446		const char *pub_str = visib_mod_string(node->data.fn_proto.visib_mod);
447		const char *extern_str = extern_string(node->data.fn_proto.is_extern);
448		const char *export_str = export_string(node->data.fn_proto.is_export);
449		const char *inline_str = inline_string(node->data.fn_proto.fn_inline);
450		fprintf(ar->f, "%s%s%s%sfn ", pub_str, inline_str, export_str, extern_str);
451		if (node->data.fn_proto.name != nullptr) {
452		print_symbol(ar, node->data.fn_proto.name);
453		}
454		fprintf(ar->f, "(");
455		size_t arg_count = node->data.fn_proto.params.length;
456		for (size_t arg_i = 0; arg_i < arg_count; arg_i += 1) {
457		AstNode *param_decl = node->data.fn_proto.params.at(arg_i);
458		assert(param_decl->type == NodeTypeParamDecl);
459		if (param_decl->data.param_decl.name != nullptr) {
460		const char *noalias_str = param_decl->data.param_decl.is_noalias ? "noalias " : "";
461		const char *inline_str = param_decl->data.param_decl.is_comptime ? "comptime " : "";
462		fprintf(ar->f, "%s%s", noalias_str, inline_str);
463		print_symbol(ar, param_decl->data.param_decl.name);
464		fprintf(ar->f, ": ");
465		}
466		if (param_decl->data.param_decl.is_var_args) {
467		fprintf(ar->f, "...");
468		} else if (param_decl->data.param_decl.anytype_token != nullptr) {
469		fprintf(ar->f, "anytype");
470		} else {
471		render_node_grouped(ar, param_decl->data.param_decl.type);
472		}
473
474		if (arg_i + 1 < arg_count) {
475		fprintf(ar->f, ", ");
476		}
477		}
478		if (node->data.fn_proto.is_var_args) {
479		fprintf(ar->f, ", ...");
480		}
481		fprintf(ar->f, ")");
482		if (node->data.fn_proto.align_expr) {
483		fprintf(ar->f, " align(");
484		render_node_grouped(ar, node->data.fn_proto.align_expr);
485		fprintf(ar->f, ")");
486		}
487		if (node->data.fn_proto.section_expr) {
488		fprintf(ar->f, " section(");
489		render_node_grouped(ar, node->data.fn_proto.section_expr);
490		fprintf(ar->f, ")");
491		}
492		if (node->data.fn_proto.callconv_expr) {
493		fprintf(ar->f, " callconv(");
494		render_node_grouped(ar, node->data.fn_proto.callconv_expr);
495		fprintf(ar->f, ")");
496		}
497
498		AstNode *return_type_node = node->data.fn_proto.return_type;
499		assert(return_type_node != nullptr);
500		fprintf(ar->f, " ");
501		if (node->data.fn_proto.auto_err_set) {
502		fprintf(ar->f, "!");
503		}
504		render_node_grouped(ar, return_type_node);
505		break;
506		}
507		case NodeTypeFnDef:
508		{
509		render_node_grouped(ar, node->data.fn_def.fn_proto);
510		fprintf(ar->f, " ");
511		render_node_grouped(ar, node->data.fn_def.body);
512		break;
513		}
514		case NodeTypeBlock:
515		if (node->data.block.name != nullptr) {
516		fprintf(ar->f, "%s: ", buf_ptr(node->data.block.name));
517		}
518		if (node->data.block.statements.length == 0) {
519		fprintf(ar->f, "{}");
520		break;
521		}
522		fprintf(ar->f, "{\n");
523		ar->indent += ar->indent_size;
524		for (size_t i = 0; i < node->data.block.statements.length; i += 1) {
525		AstNode *statement = node->data.block.statements.at(i);
526		print_indent(ar);
527		render_node_grouped(ar, statement);
528
529		if (!statement_terminates_without_semicolon(statement))
530		fprintf(ar->f, ";");
531
532		fprintf(ar->f, "\n");
533		}
534		ar->indent -= ar->indent_size;
535		print_indent(ar);
536		fprintf(ar->f, "}");
537		break;
538		case NodeTypeGroupedExpr:
539		fprintf(ar->f, "(");
540		render_node_ungrouped(ar, node->data.grouped_expr);
541		fprintf(ar->f, ")");
542		break;
543		case NodeTypeReturnExpr:
544		{
545		const char *return_str = return_string(node->data.return_expr.kind);
546		fprintf(ar->f, "%s", return_str);
547		if (node->data.return_expr.expr) {
548		fprintf(ar->f, " ");
549		render_node_grouped(ar, node->data.return_expr.expr);
550		}
551		break;
552		}
553		case NodeTypeBreak:
554		{
555		fprintf(ar->f, "break");
556		if (node->data.break_expr.name != nullptr) {
557		fprintf(ar->f, " :%s", buf_ptr(node->data.break_expr.name));
558		}
559		if (node->data.break_expr.expr) {
560		fprintf(ar->f, " ");
561		render_node_grouped(ar, node->data.break_expr.expr);
562		}
563		break;
564		}
565		case NodeTypeDefer:
566		{
567		const char *defer_str = defer_string(node->data.defer.kind);
568		fprintf(ar->f, "%s ", defer_str);
569		render_node_grouped(ar, node->data.defer.expr);
570		break;
571		}
572		case NodeTypeVariableDeclaration:
573		{
574		const char *pub_str = visib_mod_string(node->data.variable_declaration.visib_mod);
575		const char *extern_str = extern_string(node->data.variable_declaration.is_extern);
576		const char *thread_local_str = thread_local_string(node->data.variable_declaration.threadlocal_tok);
577		const char *const_or_var = const_or_var_string(node->data.variable_declaration.is_const);
578		fprintf(ar->f, "%s%s%s%s ", pub_str, extern_str, thread_local_str, const_or_var);
579		print_symbol(ar, node->data.variable_declaration.symbol);
580
581		if (node->data.variable_declaration.type) {
582		fprintf(ar->f, ": ");
583		render_node_grouped(ar, node->data.variable_declaration.type);
584		}
585		if (node->data.variable_declaration.align_expr) {
586		fprintf(ar->f, "align(");
587		render_node_grouped(ar, node->data.variable_declaration.align_expr);
588		fprintf(ar->f, ") ");
589		}
590		if (node->data.variable_declaration.section_expr) {
591		fprintf(ar->f, "section(");
592		render_node_grouped(ar, node->data.variable_declaration.section_expr);
593		fprintf(ar->f, ") ");
594		}
595		if (node->data.variable_declaration.expr) {
596		fprintf(ar->f, " = ");
597		render_node_grouped(ar, node->data.variable_declaration.expr);
598		}
599		break;
600		}
601		case NodeTypeBinOpExpr:
602		if (!grouped) fprintf(ar->f, "(");
603		render_node_ungrouped(ar, node->data.bin_op_expr.op1);
604		fprintf(ar->f, " %s ", bin_op_str(node->data.bin_op_expr.bin_op));
605		render_node_ungrouped(ar, node->data.bin_op_expr.op2);
606		if (!grouped) fprintf(ar->f, ")");
607		break;
608		case NodeTypeFloatLiteral:
609		{
610		Buf rendered_buf = BUF_INIT;
611		buf_resize(&rendered_buf, 0);
612		bigfloat_append_buf(&rendered_buf, node->data.float_literal.bigfloat);
613		fprintf(ar->f, "%s", buf_ptr(&rendered_buf));
614		}
615		break;
616		case NodeTypeIntLiteral:
617		{
618		Buf rendered_buf = BUF_INIT;
619		buf_resize(&rendered_buf, 0);
620		bigint_append_buf(&rendered_buf, node->data.int_literal.bigint, 10);
621		fprintf(ar->f, "%s", buf_ptr(&rendered_buf));
622		}
623		break;
624		case NodeTypeStringLiteral:
625		{
626		Buf tmp_buf = BUF_INIT;
627		string_literal_escape(node->data.string_literal.buf, &tmp_buf);
628		fprintf(ar->f, "\"%s\"", buf_ptr(&tmp_buf));
629		}
630		break;
631		case NodeTypeCharLiteral:
632		{
633		uint8_t c = node->data.char_literal.value;
634		if (c == '\'') {
635		fprintf(ar->f, "'\\''");
636		} else if (c == '\"') {
637		fprintf(ar->f, "'\\\"'");
638		} else if (c == '\\') {
639		fprintf(ar->f, "'\\\\'");
640		} else if (c == '\n') {
641		fprintf(ar->f, "'\\n'");
642		} else if (c == '\r') {
643		fprintf(ar->f, "'\\r'");
644		} else if (c == '\t') {
645		fprintf(ar->f, "'\\t'");
646		} else if (is_printable(c)) {
647		fprintf(ar->f, "'%c'", c);
648		} else {
649		fprintf(ar->f, "'\\x%02x'", (int)c);
650		}
651		break;
652		}
653		case NodeTypeSymbol:
654		print_symbol(ar, node->data.symbol_expr.symbol);
655		break;
656		case NodeTypePrefixOpExpr:
657		{
658		if (!grouped) fprintf(ar->f, "(");
659		PrefixOp op = node->data.prefix_op_expr.prefix_op;
660		fprintf(ar->f, "%s", prefix_op_str(op));
661
662		AstNode *child_node = node->data.prefix_op_expr.primary_expr;
663		bool new_grouped = child_node->type == NodeTypePrefixOpExpr \|\| child_node->type == NodeTypePointerType;
664		render_node_extra(ar, child_node, new_grouped);
665		if (!grouped) fprintf(ar->f, ")");
666		break;
667		}
668		case NodeTypePointerType:
669		{
670		if (!grouped) fprintf(ar->f, "(");
671		const char *ptr_len_str = token_to_ptr_len_str(node->data.pointer_type.star_token);
672		fprintf(ar->f, "%s", ptr_len_str);
673		if (node->data.pointer_type.align_expr != nullptr) {
674		fprintf(ar->f, "align(");
675		render_node_grouped(ar, node->data.pointer_type.align_expr);
676		if (node->data.pointer_type.bit_offset_start != nullptr) {
677		assert(node->data.pointer_type.host_int_bytes != nullptr);
678
679		Buf offset_start_buf = BUF_INIT;
680		buf_resize(&offset_start_buf, 0);
681		bigint_append_buf(&offset_start_buf, node->data.pointer_type.bit_offset_start, 10);
682
683		Buf offset_end_buf = BUF_INIT;
684		buf_resize(&offset_end_buf, 0);
685		bigint_append_buf(&offset_end_buf, node->data.pointer_type.host_int_bytes, 10);
686
687		fprintf(ar->f, ":%s:%s ", buf_ptr(&offset_start_buf), buf_ptr(&offset_end_buf));
688		}
689		fprintf(ar->f, ") ");
690		}
691		if (node->data.pointer_type.is_const) {
692		fprintf(ar->f, "const ");
693		}
694		if (node->data.pointer_type.is_volatile) {
695		fprintf(ar->f, "volatile ");
696		}
697
698		render_node_ungrouped(ar, node->data.pointer_type.op_expr);
699		if (!grouped) fprintf(ar->f, ")");
700		break;
701		}
702		case NodeTypeFnCallExpr:
703		{
704		switch (node->data.fn_call_expr.modifier) {
705		case CallModifierNone:
706		break;
707		case CallModifierNoSuspend:
708		fprintf(ar->f, "nosuspend ");
709		break;
710		case CallModifierAsync:
711		fprintf(ar->f, "async ");
712		break;
713		case CallModifierNeverTail:
714		fprintf(ar->f, "notail ");
715		break;
716		case CallModifierNeverInline:
717		fprintf(ar->f, "noinline ");
718		break;
719		case CallModifierAlwaysTail:
720		fprintf(ar->f, "tail ");
721		break;
722		case CallModifierAlwaysInline:
723		fprintf(ar->f, "inline ");
724		break;
725		case CallModifierCompileTime:
726		fprintf(ar->f, "comptime ");
727		break;
728		case CallModifierBuiltin:
729		fprintf(ar->f, "@");
730		break;
731		}
732		AstNode *fn_ref_node = node->data.fn_call_expr.fn_ref_expr;
733		bool grouped = (fn_ref_node->type != NodeTypePrefixOpExpr && fn_ref_node->type != NodeTypePointerType);
734		render_node_extra(ar, fn_ref_node, grouped);
735		fprintf(ar->f, "(");
736		for (size_t i = 0; i < node->data.fn_call_expr.params.length; i += 1) {
737		AstNode *param = node->data.fn_call_expr.params.at(i);
738		if (i != 0) {
739		fprintf(ar->f, ", ");
740		}
741		render_node_grouped(ar, param);
742		}
743		fprintf(ar->f, ")");
744		break;
745		}
746		case NodeTypeArrayAccessExpr:
747		render_node_ungrouped(ar, node->data.array_access_expr.array_ref_expr);
748		fprintf(ar->f, "[");
749		render_node_grouped(ar, node->data.array_access_expr.subscript);
750		fprintf(ar->f, "]");
751		break;
752		case NodeTypeFieldAccessExpr:
753		{
754		AstNode *lhs = node->data.field_access_expr.struct_expr;
755		Buf *rhs = node->data.field_access_expr.field_name;
756		if (lhs->type == NodeTypeErrorType) {
757		fprintf(ar->f, "error");
758		} else {
759		render_node_ungrouped(ar, lhs);
760		}
761		fprintf(ar->f, ".");
762		print_symbol(ar, rhs);
763		break;
764		}
765		case NodeTypePtrDeref:
766		{
767		AstNode *lhs = node->data.ptr_deref_expr.target;
768		render_node_ungrouped(ar, lhs);
769		fprintf(ar->f, ".*");
770		break;
771		}
772		case NodeTypeUnwrapOptional:
773		{
774		AstNode *lhs = node->data.unwrap_optional.expr;
775		render_node_ungrouped(ar, lhs);
776		fprintf(ar->f, ".?");
777		break;
778		}
779		case NodeTypeUndefinedLiteral:
780		fprintf(ar->f, "undefined");
781		break;
782		case NodeTypeContainerDecl:
783		{
784		if (!node->data.container_decl.is_root) {
785		const char *layout_str = layout_string(node->data.container_decl.layout);
786		const char *container_str = container_string(node->data.container_decl.kind);
787		fprintf(ar->f, "%s%s", layout_str, container_str);
788		if (node->data.container_decl.auto_enum) {
789		fprintf(ar->f, "(enum");
790		}
791		if (node->data.container_decl.init_arg_expr != nullptr) {
792		fprintf(ar->f, "(");
793		render_node_grouped(ar, node->data.container_decl.init_arg_expr);
794		fprintf(ar->f, ")");
795		}
796		if (node->data.container_decl.auto_enum) {
797		fprintf(ar->f, ")");
798		}
799
800		fprintf(ar->f, " {\n");
801		ar->indent += ar->indent_size;
802		}
803		for (size_t field_i = 0; field_i < node->data.container_decl.fields.length; field_i += 1) {
804		AstNode *field_node = node->data.container_decl.fields.at(field_i);
805		assert(field_node->type == NodeTypeStructField);
806		print_indent(ar);
807		print_symbol(ar, field_node->data.struct_field.name);
808		if (field_node->data.struct_field.type != nullptr) {
809		fprintf(ar->f, ": ");
810		render_node_grouped(ar, field_node->data.struct_field.type);
811		}
812		if (field_node->data.struct_field.value != nullptr) {
813		fprintf(ar->f, " = ");
814		render_node_grouped(ar, field_node->data.struct_field.value);
815		}
816		fprintf(ar->f, ",\n");
817		}
818
819		for (size_t decl_i = 0; decl_i < node->data.container_decl.decls.length; decl_i += 1) {
820		AstNode *decls_node = node->data.container_decl.decls.at(decl_i);
821		render_node_grouped(ar, decls_node);
822
823		if (decls_node->type == NodeTypeUsingNamespace \|\|
824		decls_node->type == NodeTypeVariableDeclaration \|\|
825		decls_node->type == NodeTypeFnProto)
826		{
827		fprintf(ar->f, ";");
828		}
829		fprintf(ar->f, "\n");
830		}
831
832		if (!node->data.container_decl.is_root) {
833		ar->indent -= ar->indent_size;
834		print_indent(ar);
835		fprintf(ar->f, "}");
836		}
837		break;
838		}
839		case NodeTypeContainerInitExpr:
840		if (node->data.container_init_expr.type != nullptr) {
841		render_node_ungrouped(ar, node->data.container_init_expr.type);
842		}
843		if (node->data.container_init_expr.kind == ContainerInitKindStruct) {
844		fprintf(ar->f, "{\n");
845		ar->indent += ar->indent_size;
846		} else {
847		fprintf(ar->f, "{");
848		}
849		for (size_t i = 0; i < node->data.container_init_expr.entries.length; i += 1) {
850		AstNode *entry = node->data.container_init_expr.entries.at(i);
851		if (entry->type == NodeTypeStructValueField) {
852		Buf *name = entry->data.struct_val_field.name;
853		AstNode *expr = entry->data.struct_val_field.expr;
854		print_indent(ar);
855		fprintf(ar->f, ".%s = ", buf_ptr(name));
856		render_node_grouped(ar, expr);
857		fprintf(ar->f, ",\n");
858		} else {
859		if (i != 0)
860		fprintf(ar->f, ", ");
861		render_node_grouped(ar, entry);
862		}
863		}
864		if (node->data.container_init_expr.kind == ContainerInitKindStruct) {
865		ar->indent -= ar->indent_size;
866		}
867		print_indent(ar);
868		fprintf(ar->f, "}");
869		break;
870		case NodeTypeArrayType:
871		{
872		fprintf(ar->f, "[");
873		if (node->data.array_type.size) {
874		render_node_grouped(ar, node->data.array_type.size);
875		}
876		fprintf(ar->f, "]");
877		if (node->data.array_type.is_const) {
878		fprintf(ar->f, "const ");
879		}
880		render_node_ungrouped(ar, node->data.array_type.child_type);
881		break;
882		}
883		case NodeTypeInferredArrayType:
884		{
885		fprintf(ar->f, "[_]");
886		render_node_ungrouped(ar, node->data.inferred_array_type.child_type);
887		break;
888		}
889		case NodeTypeAnyFrameType: {
890		fprintf(ar->f, "anyframe");
891		if (node->data.anyframe_type.payload_type != nullptr) {
892		fprintf(ar->f, "->");
893		render_node_grouped(ar, node->data.anyframe_type.payload_type);
894		}
895		break;
896		}
897		case NodeTypeErrorType:
898		fprintf(ar->f, "anyerror");
899		break;
900		case NodeTypeAsmExpr:
901		{
902		AstNodeAsmExpr *asm_expr = &node->data.asm_expr;
903		const char *volatile_str = (asm_expr->volatile_token != nullptr) ? " volatile" : "";
904		fprintf(ar->f, "asm%s (", volatile_str);
905		render_node_ungrouped(ar, asm_expr->asm_template);
906		fprintf(ar->f, ")");
907		print_indent(ar);
908		fprintf(ar->f, ": ");
909		for (size_t i = 0; i < asm_expr->output_list.length; i += 1) {
910		AsmOutput *asm_output = asm_expr->output_list.at(i);
911
912		if (i != 0) {
913		fprintf(ar->f, ",\n");
914		print_indent(ar);
915		}
916
917		fprintf(ar->f, "[%s] \"%s\" (",
918		buf_ptr(asm_output->asm_symbolic_name),
919		buf_ptr(asm_output->constraint));
920		if (asm_output->return_type) {
921		fprintf(ar->f, "-> ");
922		render_node_grouped(ar, asm_output->return_type);
923		} else {
924		fprintf(ar->f, "%s", buf_ptr(asm_output->variable_name));
925		}
926		fprintf(ar->f, ")");
927		}
928		fprintf(ar->f, "\n");
929		print_indent(ar);
930		fprintf(ar->f, ": ");
931		for (size_t i = 0; i < asm_expr->input_list.length; i += 1) {
932		AsmInput *asm_input = asm_expr->input_list.at(i);
933
934		if (i != 0) {
935		fprintf(ar->f, ",\n");
936		print_indent(ar);
937		}
938
939		fprintf(ar->f, "[%s] \"%s\" (",
940		buf_ptr(asm_input->asm_symbolic_name),
941		buf_ptr(asm_input->constraint));
942		render_node_grouped(ar, asm_input->expr);
943		fprintf(ar->f, ")");
944		}
945		fprintf(ar->f, "\n");
946		print_indent(ar);
947		fprintf(ar->f, ": ");
948		for (size_t i = 0; i < asm_expr->clobber_list.length; i += 1) {
949		Buf *reg_name = asm_expr->clobber_list.at(i);
950		if (i != 0) fprintf(ar->f, ", ");
951		fprintf(ar->f, "\"%s\"", buf_ptr(reg_name));
952		}
953		fprintf(ar->f, ")");
954		break;
955		}
956		case NodeTypeWhileExpr:
957		{
958		if (node->data.while_expr.name != nullptr) {
959		fprintf(ar->f, "%s: ", buf_ptr(node->data.while_expr.name));
960		}
961		const char *inline_str = node->data.while_expr.is_inline ? "inline " : "";
962		fprintf(ar->f, "%swhile (", inline_str);
963		render_node_grouped(ar, node->data.while_expr.condition);
964		fprintf(ar->f, ") ");
965		if (node->data.while_expr.var_symbol) {
966		fprintf(ar->f, "\|%s\| ", buf_ptr(node->data.while_expr.var_symbol));
967		}
968		if (node->data.while_expr.continue_expr) {
969		fprintf(ar->f, ": (");
970		render_node_grouped(ar, node->data.while_expr.continue_expr);
971		fprintf(ar->f, ") ");
972		}
973		render_node_grouped(ar, node->data.while_expr.body);
974		if (node->data.while_expr.else_node) {
975		fprintf(ar->f, " else ");
976		if (node->data.while_expr.err_symbol) {
977		fprintf(ar->f, "\|%s\| ", buf_ptr(node->data.while_expr.err_symbol));
978		}
979		render_node_grouped(ar, node->data.while_expr.else_node);
980		}
981		break;
982		}
983		case NodeTypeBoolLiteral:
984		{
985		const char *bool_str = node->data.bool_literal.value ? "true" : "false";
986		fprintf(ar->f, "%s", bool_str);
987		break;
988		}
989		case NodeTypeIfBoolExpr:
990		{
991		fprintf(ar->f, "if (");
992		render_node_grouped(ar, node->data.if_bool_expr.condition);
993		fprintf(ar->f, ") ");
994		render_node_grouped(ar, node->data.if_bool_expr.then_block);
995		if (node->data.if_bool_expr.else_node) {
996		fprintf(ar->f, " else ");
997		render_node_grouped(ar, node->data.if_bool_expr.else_node);
998		}
999		break;
1000		}
1001		case NodeTypeNullLiteral:
1002		{
1003		fprintf(ar->f, "null");
1004		break;
1005		}
1006		case NodeTypeIfErrorExpr:
1007		{
1008		fprintf(ar->f, "if (");
1009		render_node_grouped(ar, node->data.if_err_expr.target_node);
1010		fprintf(ar->f, ") ");
1011		if (node->data.if_err_expr.var_symbol) {
1012		const char ptr_str = node->data.if_err_expr.var_is_ptr ? "" : "";
1013		const char *var_name = buf_ptr(node->data.if_err_expr.var_symbol);
1014		fprintf(ar->f, "\|%s%s\| ", ptr_str, var_name);
1015		}
1016		render_node_grouped(ar, node->data.if_err_expr.then_node);
1017		if (node->data.if_err_expr.else_node) {
1018		fprintf(ar->f, " else ");
1019		if (node->data.if_err_expr.err_symbol) {
1020		fprintf(ar->f, "\|%s\| ", buf_ptr(node->data.if_err_expr.err_symbol));
1021		}
1022		render_node_grouped(ar, node->data.if_err_expr.else_node);
1023		}
1024		break;
1025		}
1026		case NodeTypeIfOptional:
1027		{
1028		fprintf(ar->f, "if (");
1029		render_node_grouped(ar, node->data.test_expr.target_node);
1030		fprintf(ar->f, ") ");
1031		if (node->data.test_expr.var_symbol) {
1032		const char ptr_str = node->data.test_expr.var_is_ptr ? "" : "";
1033		const char *var_name = buf_ptr(node->data.test_expr.var_symbol);
1034		fprintf(ar->f, "\|%s%s\| ", ptr_str, var_name);
1035		}
1036		render_node_grouped(ar, node->data.test_expr.then_node);
1037		if (node->data.test_expr.else_node) {
1038		fprintf(ar->f, " else ");
1039		render_node_grouped(ar, node->data.test_expr.else_node);
1040		}
1041		break;
1042		}
1043		case NodeTypeSwitchExpr:
1044		{
1045		AstNodeSwitchExpr *switch_expr = &node->data.switch_expr;
1046		fprintf(ar->f, "switch (");
1047		render_node_grouped(ar, switch_expr->expr);
1048		fprintf(ar->f, ") {\n");
1049		ar->indent += ar->indent_size;
1050
1051		for (size_t prong_i = 0; prong_i < switch_expr->prongs.length; prong_i += 1) {
1052		AstNode *prong_node = switch_expr->prongs.at(prong_i);
1053		AstNodeSwitchProng *switch_prong = &prong_node->data.switch_prong;
1054		print_indent(ar);
1055		for (size_t item_i = 0; item_i < switch_prong->items.length; item_i += 1) {
1056		AstNode *item_node = switch_prong->items.at(item_i);
1057		if (item_i != 0)
1058		fprintf(ar->f, ", ");
1059		if (item_node->type == NodeTypeSwitchRange) {
1060		AstNode *start_node = item_node->data.switch_range.start;
1061		AstNode *end_node = item_node->data.switch_range.end;
1062		render_node_grouped(ar, start_node);
1063		fprintf(ar->f, "...");
1064		render_node_grouped(ar, end_node);
1065		} else {
1066		render_node_grouped(ar, item_node);
1067		}
1068		}
1069		const char *else_str = (switch_prong->items.length == 0) ? "else" : "";
1070		fprintf(ar->f, "%s => ", else_str);
1071		if (switch_prong->var_symbol) {
1072		const char star_str = switch_prong->var_is_ptr ? "" : "";
1073		Buf *var_name = switch_prong->var_symbol->data.symbol_expr.symbol;
1074		fprintf(ar->f, "\|%s%s\| ", star_str, buf_ptr(var_name));
1075		}
1076		render_node_grouped(ar, switch_prong->expr);
1077		fprintf(ar->f, ",\n");
1078		}
1079
1080		ar->indent -= ar->indent_size;
1081		print_indent(ar);
1082		fprintf(ar->f, "}");
1083		break;
1084		}
1085		case NodeTypeCompTime:
1086		{
1087		fprintf(ar->f, "comptime ");
1088		render_node_grouped(ar, node->data.comptime_expr.expr);
1089		break;
1090		}
1091		case NodeTypeNoSuspend:
1092		{
1093		fprintf(ar->f, "nosuspend ");
1094		render_node_grouped(ar, node->data.nosuspend_expr.expr);
1095		break;
1096		}
1097		case NodeTypeForExpr:
1098		{
1099		if (node->data.for_expr.name != nullptr) {
1100		fprintf(ar->f, "%s: ", buf_ptr(node->data.for_expr.name));
1101		}
1102		const char *inline_str = node->data.for_expr.is_inline ? "inline " : "";
1103		fprintf(ar->f, "%sfor (", inline_str);
1104		render_node_grouped(ar, node->data.for_expr.array_expr);
1105		fprintf(ar->f, ") ");
1106		if (node->data.for_expr.elem_node) {
1107		fprintf(ar->f, "\|");
1108		if (node->data.for_expr.elem_is_ptr)
1109		fprintf(ar->f, "*");
1110		render_node_grouped(ar, node->data.for_expr.elem_node);
1111		if (node->data.for_expr.index_node) {
1112		fprintf(ar->f, ", ");
1113		render_node_grouped(ar, node->data.for_expr.index_node);
1114		}
1115		fprintf(ar->f, "\| ");
1116		}
1117		render_node_grouped(ar, node->data.for_expr.body);
1118		if (node->data.for_expr.else_node) {
1119		fprintf(ar->f, " else");
1120		render_node_grouped(ar, node->data.for_expr.else_node);
1121		}
1122		break;
1123		}
1124		case NodeTypeContinue:
1125		{
1126		fprintf(ar->f, "continue");
1127		if (node->data.continue_expr.name != nullptr) {
1128		fprintf(ar->f, " :%s", buf_ptr(node->data.continue_expr.name));
1129		}
1130		break;
1131		}
1132		case NodeTypeUnreachable:
1133		{
1134		fprintf(ar->f, "unreachable");
1135		break;
1136		}
1137		case NodeTypeSliceExpr:
1138		{
1139		render_node_ungrouped(ar, node->data.slice_expr.array_ref_expr);
1140		fprintf(ar->f, "[");
1141		render_node_grouped(ar, node->data.slice_expr.start);
1142		fprintf(ar->f, "..");
1143		if (node->data.slice_expr.end)
1144		render_node_grouped(ar, node->data.slice_expr.end);
1145		fprintf(ar->f, "]");
1146		break;
1147		}
1148		case NodeTypeCatchExpr:
1149		{
1150		render_node_ungrouped(ar, node->data.unwrap_err_expr.op1);
1151		fprintf(ar->f, " catch ");
1152		if (node->data.unwrap_err_expr.symbol) {
1153		Buf *var_name = node->data.unwrap_err_expr.symbol->data.symbol_expr.symbol;
1154		fprintf(ar->f, "\|%s\| ", buf_ptr(var_name));
1155		}
1156		render_node_ungrouped(ar, node->data.unwrap_err_expr.op2);
1157		break;
1158		}
1159		case NodeTypeErrorSetDecl:
1160		{
1161		fprintf(ar->f, "error {\n");
1162		ar->indent += ar->indent_size;
1163
1164		for (size_t i = 0; i < node->data.err_set_decl.decls.length; i += 1) {
1165		AstNode *field_node = node->data.err_set_decl.decls.at(i);
1166		switch (field_node->type) {
1167		case NodeTypeSymbol:
1168		print_indent(ar);
1169		print_symbol(ar, field_node->data.symbol_expr.symbol);
1170		fprintf(ar->f, ",\n");
1171		break;
1172		case NodeTypeErrorSetField:
1173		print_indent(ar);
1174		print_symbol(ar, field_node->data.err_set_field.field_name->data.symbol_expr.symbol);
1175		fprintf(ar->f, ",\n");
1176		break;
1177		default:
1178		zig_unreachable();
1179		}
1180		}
1181
1182		ar->indent -= ar->indent_size;
1183		print_indent(ar);
1184		fprintf(ar->f, "}");
1185		break;
1186		}
1187		case NodeTypeResume:
1188		{
1189		fprintf(ar->f, "resume ");
1190		render_node_grouped(ar, node->data.resume_expr.expr);
1191		break;
1192		}
1193		case NodeTypeAwaitExpr:
1194		{
1195		fprintf(ar->f, "await ");
1196		render_node_grouped(ar, node->data.await_expr.expr);
1197		break;
1198		}
1199		case NodeTypeSuspend:
1200		{
1201		if (node->data.suspend.block != nullptr) {
1202		fprintf(ar->f, "suspend ");
1203		render_node_grouped(ar, node->data.suspend.block);
1204		} else {
1205		fprintf(ar->f, "suspend\n");
1206		}
1207		break;
1208		}
1209		case NodeTypeEnumLiteral:
1210		{
1211		fprintf(ar->f, ".%s", buf_ptr(&node->data.enum_literal.identifier->data.str_lit.str));
1212		break;
1213		}
1214		case NodeTypeAnyTypeField: {
1215		fprintf(ar->f, "anytype");
1216		break;
1217		}
1218		case NodeTypeParamDecl:
1219		case NodeTypeTestDecl:
1220		case NodeTypeStructField:
1221		case NodeTypeUsingNamespace:
1222		case NodeTypeErrorSetField:
1223		zig_panic("TODO more ast rendering");
1224		}
1225		}
1226
1227
1228		void ast_render(FILE f, AstNode node, int indent_size) {
1229		AstRender ar = {0};
1230		ar.f = f;
1231		ar.indent_size = indent_size;
1232		ar.indent = 0;
1233
1234		render_node_grouped(&ar, node);
1235		}
1236
1237		void AstNode::src() {
1238		fprintf(stderr, "%s:%" ZIG_PRI_usize ":%" ZIG_PRI_usize "\n",
1239		buf_ptr(this->owner->data.structure.root_struct->path),
1240		this->line + 1, this->column + 1);
1241		}

src/stage1/ast_render.hpp deleted-20

...	...	@@ -1,20 +0,0 @@
1		/*
2		* Copyright (c) 2015 Andrew Kelley
3		*
4		* This file is part of zig, which is MIT licensed.
5		* See http://opensource.org/licenses/MIT
6		*/
7
8		#ifndef ZIG_AST_RENDER_HPP
9		#define ZIG_AST_RENDER_HPP
10
11		#include "all_types.hpp"
12		#include "parser.hpp"
13
14		#include <stdio.h>
15
16		void ast_print(FILE f, AstNode node, int indent);
17
18		void ast_render(FILE f, AstNode node, int indent_size);
19
20		#endif

src/stage1/astgen.cpp created+8120

...	...	@@ -0,0 +1,8120 @@
	1	/*
	2	* Copyright (c) 2021 Andrew Kelley
	3	*
	4	* This file is part of zig, which is MIT licensed.
	5	* See http://opensource.org/licenses/MIT
	6	*/
	7
	8	#include "astgen.hpp"
	9	#include "analyze.hpp"
	10	#include "util.hpp"
	11	#include "os.hpp"
	12	#include "parser.hpp"
	13
	14	static IrInstSrc ir_gen_node(IrBuilderSrc irb, AstNode node, Scope scope);
	15	static IrInstSrc ir_gen_node_extra(IrBuilderSrc irb, AstNode node, Scope scope, LVal lval,
	16	ResultLoc *result_loc);
	17
	18	static IrInstSrc ir_lval_wrap(IrBuilderSrc irb, Scope scope, IrInstSrc value, LVal lval,
	19	ResultLoc *result_loc);
	20	static IrInstSrc ir_expr_wrap(IrBuilderSrc irb, Scope scope, IrInstSrc inst,
	21	ResultLoc *result_loc);
	22	static IrInstSrc ir_gen_union_init_expr(IrBuilderSrc irb, Scope scope, AstNode source_node,
	23	IrInstSrc union_type, IrInstSrc field_name, AstNode *expr_node,
	24	LVal lval, ResultLoc *parent_result_loc);
	25	static ResultLocCast ir_build_cast_result_loc(IrBuilderSrc irb, IrInstSrc *dest_type,
	26	ResultLoc *parent_result_loc);
	27	static ZigVar ir_create_var(IrBuilderSrc irb, AstNode node, Scope scope, Buf *name,
	28	bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstSrc *is_comptime);
	29	static void build_decl_var_and_init(IrBuilderSrc irb, Scope scope, AstNode *source_node,
	30	ZigVar var, IrInstSrc init, const char name_hint, IrInstSrc is_comptime);
	31
	32	static void ir_assert_impl(bool ok, IrInst source_instruction, char const file, unsigned int line) {
	33	if (ok) return;
	34	src_assert_impl(ok, source_instruction->source_node, file, line);
	35	}
	36
	37	static void ir_add_call_stack_errors(CodeGen codegen, IrExecutableSrc exec, ErrorMsg *err_msg, int limit) {
	38	if (!exec \|\| !exec->source_node \|\| limit < 0) return;
	39	add_error_note(codegen, err_msg, exec->source_node, buf_sprintf("called from here"));
	40
	41	ir_add_call_stack_errors_gen(codegen, exec->parent_exec, err_msg, limit - 1);
	42	}
	43
	44	static ErrorMsg exec_add_error_node(CodeGen codegen, IrExecutableSrc exec, AstNode source_node, Buf *msg) {
	45	ErrorMsg *err_msg = add_node_error(codegen, source_node, msg);
	46	invalidate_exec(exec, err_msg);
	47	if (exec->parent_exec) {
	48	ir_add_call_stack_errors(codegen, exec, err_msg, 10);
	49	}
	50	return err_msg;
	51	}
	52
	53
	54	#define ir_assert(OK, SOURCE_INSTRUCTION) ir_assert_impl((OK), (SOURCE_INSTRUCTION), __FILE__, __LINE__)
	55
	56
	57	static bool instr_is_unreachable(IrInstSrc *instruction) {
	58	return instruction->is_noreturn;
	59	}
	60
	61	void destroy_instruction_src(IrInstSrc *inst) {
	62	switch (inst->id) {
	63	case IrInstSrcIdInvalid:
	64	zig_unreachable();
	65	case IrInstSrcIdReturn:
	66	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcReturn *>(inst));
	67	case IrInstSrcIdConst:
	68	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcConst *>(inst));
	69	case IrInstSrcIdBinOp:
	70	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBinOp *>(inst));
	71	case IrInstSrcIdMergeErrSets:
	72	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcMergeErrSets *>(inst));
	73	case IrInstSrcIdDeclVar:
	74	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcDeclVar *>(inst));
	75	case IrInstSrcIdCall:
	76	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCall *>(inst));
	77	case IrInstSrcIdCallExtra:
	78	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCallExtra *>(inst));
	79	case IrInstSrcIdAsyncCallExtra:
	80	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAsyncCallExtra *>(inst));
	81	case IrInstSrcIdUnOp:
	82	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUnOp *>(inst));
	83	case IrInstSrcIdCondBr:
	84	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCondBr *>(inst));
	85	case IrInstSrcIdBr:
	86	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBr *>(inst));
	87	case IrInstSrcIdPhi:
	88	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPhi *>(inst));
	89	case IrInstSrcIdContainerInitList:
	90	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcContainerInitList *>(inst));
	91	case IrInstSrcIdContainerInitFields:
	92	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcContainerInitFields *>(inst));
	93	case IrInstSrcIdUnreachable:
	94	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUnreachable *>(inst));
	95	case IrInstSrcIdElemPtr:
	96	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcElemPtr *>(inst));
	97	case IrInstSrcIdVarPtr:
	98	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcVarPtr *>(inst));
	99	case IrInstSrcIdLoadPtr:
	100	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcLoadPtr *>(inst));
	101	case IrInstSrcIdStorePtr:
	102	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcStorePtr *>(inst));
	103	case IrInstSrcIdTypeOf:
	104	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTypeOf *>(inst));
	105	case IrInstSrcIdFieldPtr:
	106	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFieldPtr *>(inst));
	107	case IrInstSrcIdSetCold:
	108	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSetCold *>(inst));
	109	case IrInstSrcIdSetRuntimeSafety:
	110	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSetRuntimeSafety *>(inst));
	111	case IrInstSrcIdSetFloatMode:
	112	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSetFloatMode *>(inst));
	113	case IrInstSrcIdArrayType:
	114	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcArrayType *>(inst));
	115	case IrInstSrcIdSliceType:
	116	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSliceType *>(inst));
	117	case IrInstSrcIdAnyFrameType:
	118	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAnyFrameType *>(inst));
	119	case IrInstSrcIdAsm:
	120	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAsm *>(inst));
	121	case IrInstSrcIdSizeOf:
	122	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSizeOf *>(inst));
	123	case IrInstSrcIdTestNonNull:
	124	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTestNonNull *>(inst));
	125	case IrInstSrcIdOptionalUnwrapPtr:
	126	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcOptionalUnwrapPtr *>(inst));
	127	case IrInstSrcIdPopCount:
	128	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPopCount *>(inst));
	129	case IrInstSrcIdClz:
	130	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcClz *>(inst));
	131	case IrInstSrcIdCtz:
	132	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCtz *>(inst));
	133	case IrInstSrcIdBswap:
	134	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBswap *>(inst));
	135	case IrInstSrcIdBitReverse:
	136	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBitReverse *>(inst));
	137	case IrInstSrcIdSwitchBr:
	138	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSwitchBr *>(inst));
	139	case IrInstSrcIdSwitchVar:
	140	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSwitchVar *>(inst));
	141	case IrInstSrcIdSwitchElseVar:
	142	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSwitchElseVar *>(inst));
	143	case IrInstSrcIdSwitchTarget:
	144	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSwitchTarget *>(inst));
	145	case IrInstSrcIdImport:
	146	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcImport *>(inst));
	147	case IrInstSrcIdRef:
	148	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcRef *>(inst));
	149	case IrInstSrcIdCompileErr:
	150	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCompileErr *>(inst));
	151	case IrInstSrcIdCompileLog:
	152	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCompileLog *>(inst));
	153	case IrInstSrcIdErrName:
	154	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcErrName *>(inst));
	155	case IrInstSrcIdCImport:
	156	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCImport *>(inst));
	157	case IrInstSrcIdCInclude:
	158	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCInclude *>(inst));
	159	case IrInstSrcIdCDefine:
	160	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCDefine *>(inst));
	161	case IrInstSrcIdCUndef:
	162	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCUndef *>(inst));
	163	case IrInstSrcIdEmbedFile:
	164	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcEmbedFile *>(inst));
	165	case IrInstSrcIdCmpxchg:
	166	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCmpxchg *>(inst));
	167	case IrInstSrcIdFence:
	168	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFence *>(inst));
	169	case IrInstSrcIdReduce:
	170	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcReduce *>(inst));
	171	case IrInstSrcIdTruncate:
	172	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTruncate *>(inst));
	173	case IrInstSrcIdIntCast:
	174	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcIntCast *>(inst));
	175	case IrInstSrcIdFloatCast:
	176	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFloatCast *>(inst));
	177	case IrInstSrcIdErrSetCast:
	178	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcErrSetCast *>(inst));
	179	case IrInstSrcIdIntToFloat:
	180	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcIntToFloat *>(inst));
	181	case IrInstSrcIdFloatToInt:
	182	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFloatToInt *>(inst));
	183	case IrInstSrcIdBoolToInt:
	184	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBoolToInt *>(inst));
	185	case IrInstSrcIdVectorType:
	186	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcVectorType *>(inst));
	187	case IrInstSrcIdShuffleVector:
	188	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcShuffleVector *>(inst));
	189	case IrInstSrcIdSplat:
	190	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSplat *>(inst));
	191	case IrInstSrcIdBoolNot:
	192	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBoolNot *>(inst));
	193	case IrInstSrcIdMemset:
	194	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcMemset *>(inst));
	195	case IrInstSrcIdMemcpy:
	196	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcMemcpy *>(inst));
	197	case IrInstSrcIdSlice:
	198	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSlice *>(inst));
	199	case IrInstSrcIdBreakpoint:
	200	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBreakpoint *>(inst));
	201	case IrInstSrcIdReturnAddress:
	202	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcReturnAddress *>(inst));
	203	case IrInstSrcIdFrameAddress:
	204	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFrameAddress *>(inst));
	205	case IrInstSrcIdFrameHandle:
	206	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFrameHandle *>(inst));
	207	case IrInstSrcIdFrameType:
	208	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFrameType *>(inst));
	209	case IrInstSrcIdFrameSize:
	210	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFrameSize *>(inst));
	211	case IrInstSrcIdAlignOf:
	212	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAlignOf *>(inst));
	213	case IrInstSrcIdOverflowOp:
	214	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcOverflowOp *>(inst));
	215	case IrInstSrcIdTestErr:
	216	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTestErr *>(inst));
	217	case IrInstSrcIdUnwrapErrCode:
	218	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUnwrapErrCode *>(inst));
	219	case IrInstSrcIdUnwrapErrPayload:
	220	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUnwrapErrPayload *>(inst));
	221	case IrInstSrcIdFnProto:
	222	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFnProto *>(inst));
	223	case IrInstSrcIdTestComptime:
	224	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTestComptime *>(inst));
	225	case IrInstSrcIdPtrCast:
	226	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPtrCast *>(inst));
	227	case IrInstSrcIdBitCast:
	228	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBitCast *>(inst));
	229	case IrInstSrcIdPtrToInt:
	230	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPtrToInt *>(inst));
	231	case IrInstSrcIdIntToPtr:
	232	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcIntToPtr *>(inst));
	233	case IrInstSrcIdIntToEnum:
	234	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcIntToEnum *>(inst));
	235	case IrInstSrcIdIntToErr:
	236	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcIntToErr *>(inst));
	237	case IrInstSrcIdErrToInt:
	238	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcErrToInt *>(inst));
	239	case IrInstSrcIdCheckSwitchProngsUnderNo:
	240	case IrInstSrcIdCheckSwitchProngsUnderYes:
	241	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCheckSwitchProngs *>(inst));
	242	case IrInstSrcIdCheckStatementIsVoid:
	243	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCheckStatementIsVoid *>(inst));
	244	case IrInstSrcIdTypeName:
	245	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTypeName *>(inst));
	246	case IrInstSrcIdTagName:
	247	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTagName *>(inst));
	248	case IrInstSrcIdPtrType:
	249	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPtrType *>(inst));
	250	case IrInstSrcIdPtrTypeSimple:
	251	case IrInstSrcIdPtrTypeSimpleConst:
	252	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPtrTypeSimple *>(inst));
	253	case IrInstSrcIdDeclRef:
	254	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcDeclRef *>(inst));
	255	case IrInstSrcIdPanic:
	256	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPanic *>(inst));
	257	case IrInstSrcIdFieldParentPtr:
	258	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFieldParentPtr *>(inst));
	259	case IrInstSrcIdByteOffsetOf:
	260	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcByteOffsetOf *>(inst));
	261	case IrInstSrcIdBitOffsetOf:
	262	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBitOffsetOf *>(inst));
	263	case IrInstSrcIdTypeInfo:
	264	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTypeInfo *>(inst));
	265	case IrInstSrcIdType:
	266	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcType *>(inst));
	267	case IrInstSrcIdHasField:
	268	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcHasField *>(inst));
	269	case IrInstSrcIdSetEvalBranchQuota:
	270	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSetEvalBranchQuota *>(inst));
	271	case IrInstSrcIdAlignCast:
	272	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAlignCast *>(inst));
	273	case IrInstSrcIdImplicitCast:
	274	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcImplicitCast *>(inst));
	275	case IrInstSrcIdResolveResult:
	276	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcResolveResult *>(inst));
	277	case IrInstSrcIdResetResult:
	278	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcResetResult *>(inst));
	279	case IrInstSrcIdSetAlignStack:
	280	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSetAlignStack *>(inst));
	281	case IrInstSrcIdArgTypeAllowVarFalse:
	282	case IrInstSrcIdArgTypeAllowVarTrue:
	283	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcArgType *>(inst));
	284	case IrInstSrcIdExport:
	285	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcExport *>(inst));
	286	case IrInstSrcIdExtern:
	287	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcExtern *>(inst));
	288	case IrInstSrcIdErrorReturnTrace:
	289	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcErrorReturnTrace *>(inst));
	290	case IrInstSrcIdErrorUnion:
	291	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcErrorUnion *>(inst));
	292	case IrInstSrcIdAtomicRmw:
	293	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAtomicRmw *>(inst));
	294	case IrInstSrcIdSaveErrRetAddr:
	295	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSaveErrRetAddr *>(inst));
	296	case IrInstSrcIdAddImplicitReturnType:
	297	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAddImplicitReturnType *>(inst));
	298	case IrInstSrcIdFloatOp:
	299	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFloatOp *>(inst));
	300	case IrInstSrcIdMulAdd:
	301	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcMulAdd *>(inst));
	302	case IrInstSrcIdAtomicLoad:
	303	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAtomicLoad *>(inst));
	304	case IrInstSrcIdAtomicStore:
	305	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAtomicStore *>(inst));
	306	case IrInstSrcIdEnumToInt:
	307	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcEnumToInt *>(inst));
	308	case IrInstSrcIdCheckRuntimeScope:
	309	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCheckRuntimeScope *>(inst));
	310	case IrInstSrcIdHasDecl:
	311	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcHasDecl *>(inst));
	312	case IrInstSrcIdUndeclaredIdent:
	313	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUndeclaredIdent *>(inst));
	314	case IrInstSrcIdAlloca:
	315	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAlloca *>(inst));
	316	case IrInstSrcIdEndExpr:
	317	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcEndExpr *>(inst));
	318	case IrInstSrcIdUnionInitNamedField:
	319	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUnionInitNamedField *>(inst));
	320	case IrInstSrcIdSuspendBegin:
	321	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSuspendBegin *>(inst));
	322	case IrInstSrcIdSuspendFinish:
	323	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSuspendFinish *>(inst));
	324	case IrInstSrcIdResume:
	325	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcResume *>(inst));
	326	case IrInstSrcIdAwait:
	327	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAwait *>(inst));
	328	case IrInstSrcIdSpillBegin:
	329	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSpillBegin *>(inst));
	330	case IrInstSrcIdSpillEnd:
	331	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSpillEnd *>(inst));
	332	case IrInstSrcIdCallArgs:
	333	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCallArgs *>(inst));
	334	case IrInstSrcIdWasmMemorySize:
	335	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcWasmMemorySize *>(inst));
	336	case IrInstSrcIdWasmMemoryGrow:
	337	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcWasmMemoryGrow *>(inst));
	338	case IrInstSrcIdSrc:
	339	return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSrc *>(inst));
	340	}
	341	zig_unreachable();
	342	}
	343
	344
	345	bool ir_should_inline(IrExecutableSrc exec, Scope scope) {
	346	if (exec->is_inline)
	347	return true;
	348
	349	while (scope != nullptr) {
	350	if (scope->id == ScopeIdCompTime)
	351	return true;
	352	if (scope->id == ScopeIdTypeOf)
	353	return false;
	354	if (scope->id == ScopeIdFnDef)
	355	break;
	356	scope = scope->parent;
	357	}
	358	return false;
	359	}
	360
	361	static void ir_instruction_append(IrBasicBlockSrc basic_block, IrInstSrc instruction) {
	362	assert(basic_block);
	363	assert(instruction);
	364	basic_block->instruction_list.append(instruction);
	365	}
	366
	367	static size_t exec_next_debug_id(IrExecutableSrc *exec) {
	368	size_t result = exec->next_debug_id;
	369	exec->next_debug_id += 1;
	370	return result;
	371	}
	372
	373	static ZigFn exec_fn_entry(IrExecutableSrc exec) {
	374	return exec->fn_entry;
	375	}
	376
	377	static Buf exec_c_import_buf(IrExecutableSrc exec) {
	378	return exec->c_import_buf;
	379	}
	380
	381	static void ir_ref_bb(IrBasicBlockSrc *bb) {
	382	bb->ref_count += 1;
	383	}
	384
	385	static void ir_ref_instruction(IrInstSrc instruction, IrBasicBlockSrc cur_bb) {
	386	assert(instruction->id != IrInstSrcIdInvalid);
	387	instruction->base.ref_count += 1;
	388	if (instruction->owner_bb != cur_bb && !instr_is_unreachable(instruction)
	389	&& instruction->id != IrInstSrcIdConst)
	390	{
	391	ir_ref_bb(instruction->owner_bb);
	392	}
	393	}
	394
	395	static IrBasicBlockSrc ir_create_basic_block(IrBuilderSrc irb, Scope scope, const char name_hint) {
	396	IrBasicBlockSrc *result = heap::c_allocator.create<IrBasicBlockSrc>();
	397	result->scope = scope;
	398	result->name_hint = name_hint;
	399	result->debug_id = exec_next_debug_id(irb->exec);
	400	result->index = UINT32_MAX; // set later
	401	return result;
	402	}
	403
	404	static constexpr IrInstSrcId ir_inst_id(IrInstSrcDeclVar *) {
	405	return IrInstSrcIdDeclVar;
	406	}
	407
	408	static constexpr IrInstSrcId ir_inst_id(IrInstSrcBr *) {
	409	return IrInstSrcIdBr;
	410	}
	411
	412	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCondBr *) {
	413	return IrInstSrcIdCondBr;
	414	}
	415
	416	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSwitchBr *) {
	417	return IrInstSrcIdSwitchBr;
	418	}
	419
	420	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSwitchVar *) {
	421	return IrInstSrcIdSwitchVar;
	422	}
	423
	424	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSwitchElseVar *) {
	425	return IrInstSrcIdSwitchElseVar;
	426	}
	427
	428	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSwitchTarget *) {
	429	return IrInstSrcIdSwitchTarget;
	430	}
	431
	432	static constexpr IrInstSrcId ir_inst_id(IrInstSrcPhi *) {
	433	return IrInstSrcIdPhi;
	434	}
	435
	436	static constexpr IrInstSrcId ir_inst_id(IrInstSrcUnOp *) {
	437	return IrInstSrcIdUnOp;
	438	}
	439
	440	static constexpr IrInstSrcId ir_inst_id(IrInstSrcBinOp *) {
	441	return IrInstSrcIdBinOp;
	442	}
	443
	444	static constexpr IrInstSrcId ir_inst_id(IrInstSrcMergeErrSets *) {
	445	return IrInstSrcIdMergeErrSets;
	446	}
	447
	448	static constexpr IrInstSrcId ir_inst_id(IrInstSrcLoadPtr *) {
	449	return IrInstSrcIdLoadPtr;
	450	}
	451
	452	static constexpr IrInstSrcId ir_inst_id(IrInstSrcStorePtr *) {
	453	return IrInstSrcIdStorePtr;
	454	}
	455
	456	static constexpr IrInstSrcId ir_inst_id(IrInstSrcFieldPtr *) {
	457	return IrInstSrcIdFieldPtr;
	458	}
	459
	460	static constexpr IrInstSrcId ir_inst_id(IrInstSrcElemPtr *) {
	461	return IrInstSrcIdElemPtr;
	462	}
	463
	464	static constexpr IrInstSrcId ir_inst_id(IrInstSrcVarPtr *) {
	465	return IrInstSrcIdVarPtr;
	466	}
	467
	468	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCall *) {
	469	return IrInstSrcIdCall;
	470	}
	471
	472	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCallArgs *) {
	473	return IrInstSrcIdCallArgs;
	474	}
	475
	476	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCallExtra *) {
	477	return IrInstSrcIdCallExtra;
	478	}
	479
	480	static constexpr IrInstSrcId ir_inst_id(IrInstSrcAsyncCallExtra *) {
	481	return IrInstSrcIdAsyncCallExtra;
	482	}
	483
	484	static constexpr IrInstSrcId ir_inst_id(IrInstSrcConst *) {
	485	return IrInstSrcIdConst;
	486	}
	487
	488	static constexpr IrInstSrcId ir_inst_id(IrInstSrcReturn *) {
	489	return IrInstSrcIdReturn;
	490	}
	491
	492	static constexpr IrInstSrcId ir_inst_id(IrInstSrcContainerInitList *) {
	493	return IrInstSrcIdContainerInitList;
	494	}
	495
	496	static constexpr IrInstSrcId ir_inst_id(IrInstSrcContainerInitFields *) {
	497	return IrInstSrcIdContainerInitFields;
	498	}
	499
	500	static constexpr IrInstSrcId ir_inst_id(IrInstSrcUnreachable *) {
	501	return IrInstSrcIdUnreachable;
	502	}
	503
	504	static constexpr IrInstSrcId ir_inst_id(IrInstSrcTypeOf *) {
	505	return IrInstSrcIdTypeOf;
	506	}
	507
	508	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSetCold *) {
	509	return IrInstSrcIdSetCold;
	510	}
	511
	512	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSetRuntimeSafety *) {
	513	return IrInstSrcIdSetRuntimeSafety;
	514	}
	515
	516	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSetFloatMode *) {
	517	return IrInstSrcIdSetFloatMode;
	518	}
	519
	520	static constexpr IrInstSrcId ir_inst_id(IrInstSrcArrayType *) {
	521	return IrInstSrcIdArrayType;
	522	}
	523
	524	static constexpr IrInstSrcId ir_inst_id(IrInstSrcAnyFrameType *) {
	525	return IrInstSrcIdAnyFrameType;
	526	}
	527
	528	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSliceType *) {
	529	return IrInstSrcIdSliceType;
	530	}
	531
	532	static constexpr IrInstSrcId ir_inst_id(IrInstSrcAsm *) {
	533	return IrInstSrcIdAsm;
	534	}
	535
	536	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSizeOf *) {
	537	return IrInstSrcIdSizeOf;
	538	}
	539
	540	static constexpr IrInstSrcId ir_inst_id(IrInstSrcTestNonNull *) {
	541	return IrInstSrcIdTestNonNull;
	542	}
	543
	544	static constexpr IrInstSrcId ir_inst_id(IrInstSrcOptionalUnwrapPtr *) {
	545	return IrInstSrcIdOptionalUnwrapPtr;
	546	}
	547
	548	static constexpr IrInstSrcId ir_inst_id(IrInstSrcClz *) {
	549	return IrInstSrcIdClz;
	550	}
	551
	552	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCtz *) {
	553	return IrInstSrcIdCtz;
	554	}
	555
	556	static constexpr IrInstSrcId ir_inst_id(IrInstSrcPopCount *) {
	557	return IrInstSrcIdPopCount;
	558	}
	559
	560	static constexpr IrInstSrcId ir_inst_id(IrInstSrcBswap *) {
	561	return IrInstSrcIdBswap;
	562	}
	563
	564	static constexpr IrInstSrcId ir_inst_id(IrInstSrcBitReverse *) {
	565	return IrInstSrcIdBitReverse;
	566	}
	567
	568	static constexpr IrInstSrcId ir_inst_id(IrInstSrcImport *) {
	569	return IrInstSrcIdImport;
	570	}
	571
	572	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCImport *) {
	573	return IrInstSrcIdCImport;
	574	}
	575
	576	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCInclude *) {
	577	return IrInstSrcIdCInclude;
	578	}
	579
	580	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCDefine *) {
	581	return IrInstSrcIdCDefine;
	582	}
	583
	584	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCUndef *) {
	585	return IrInstSrcIdCUndef;
	586	}
	587
	588	static constexpr IrInstSrcId ir_inst_id(IrInstSrcRef *) {
	589	return IrInstSrcIdRef;
	590	}
	591
	592	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCompileErr *) {
	593	return IrInstSrcIdCompileErr;
	594	}
	595
	596	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCompileLog *) {
	597	return IrInstSrcIdCompileLog;
	598	}
	599
	600	static constexpr IrInstSrcId ir_inst_id(IrInstSrcErrName *) {
	601	return IrInstSrcIdErrName;
	602	}
	603
	604	static constexpr IrInstSrcId ir_inst_id(IrInstSrcEmbedFile *) {
	605	return IrInstSrcIdEmbedFile;
	606	}
	607
	608	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCmpxchg *) {
	609	return IrInstSrcIdCmpxchg;
	610	}
	611
	612	static constexpr IrInstSrcId ir_inst_id(IrInstSrcFence *) {
	613	return IrInstSrcIdFence;
	614	}
	615
	616	static constexpr IrInstSrcId ir_inst_id(IrInstSrcReduce *) {
	617	return IrInstSrcIdReduce;
	618	}
	619
	620	static constexpr IrInstSrcId ir_inst_id(IrInstSrcTruncate *) {
	621	return IrInstSrcIdTruncate;
	622	}
	623
	624	static constexpr IrInstSrcId ir_inst_id(IrInstSrcIntCast *) {
	625	return IrInstSrcIdIntCast;
	626	}
	627
	628	static constexpr IrInstSrcId ir_inst_id(IrInstSrcFloatCast *) {
	629	return IrInstSrcIdFloatCast;
	630	}
	631
	632	static constexpr IrInstSrcId ir_inst_id(IrInstSrcIntToFloat *) {
	633	return IrInstSrcIdIntToFloat;
	634	}
	635
	636	static constexpr IrInstSrcId ir_inst_id(IrInstSrcFloatToInt *) {
	637	return IrInstSrcIdFloatToInt;
	638	}
	639
	640	static constexpr IrInstSrcId ir_inst_id(IrInstSrcBoolToInt *) {
	641	return IrInstSrcIdBoolToInt;
	642	}
	643
	644	static constexpr IrInstSrcId ir_inst_id(IrInstSrcVectorType *) {
	645	return IrInstSrcIdVectorType;
	646	}
	647
	648	static constexpr IrInstSrcId ir_inst_id(IrInstSrcShuffleVector *) {
	649	return IrInstSrcIdShuffleVector;
	650	}
	651
	652	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSplat *) {
	653	return IrInstSrcIdSplat;
	654	}
	655
	656	static constexpr IrInstSrcId ir_inst_id(IrInstSrcBoolNot *) {
	657	return IrInstSrcIdBoolNot;
	658	}
	659
	660	static constexpr IrInstSrcId ir_inst_id(IrInstSrcMemset *) {
	661	return IrInstSrcIdMemset;
	662	}
	663
	664	static constexpr IrInstSrcId ir_inst_id(IrInstSrcMemcpy *) {
	665	return IrInstSrcIdMemcpy;
	666	}
	667
	668	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSlice *) {
	669	return IrInstSrcIdSlice;
	670	}
	671
	672	static constexpr IrInstSrcId ir_inst_id(IrInstSrcBreakpoint *) {
	673	return IrInstSrcIdBreakpoint;
	674	}
	675
	676	static constexpr IrInstSrcId ir_inst_id(IrInstSrcReturnAddress *) {
	677	return IrInstSrcIdReturnAddress;
	678	}
	679
	680	static constexpr IrInstSrcId ir_inst_id(IrInstSrcFrameAddress *) {
	681	return IrInstSrcIdFrameAddress;
	682	}
	683
	684	static constexpr IrInstSrcId ir_inst_id(IrInstSrcFrameHandle *) {
	685	return IrInstSrcIdFrameHandle;
	686	}
	687
	688	static constexpr IrInstSrcId ir_inst_id(IrInstSrcFrameType *) {
	689	return IrInstSrcIdFrameType;
	690	}
	691
	692	static constexpr IrInstSrcId ir_inst_id(IrInstSrcFrameSize *) {
	693	return IrInstSrcIdFrameSize;
	694	}
	695
	696	static constexpr IrInstSrcId ir_inst_id(IrInstSrcAlignOf *) {
	697	return IrInstSrcIdAlignOf;
	698	}
	699
	700	static constexpr IrInstSrcId ir_inst_id(IrInstSrcOverflowOp *) {
	701	return IrInstSrcIdOverflowOp;
	702	}
	703
	704	static constexpr IrInstSrcId ir_inst_id(IrInstSrcTestErr *) {
	705	return IrInstSrcIdTestErr;
	706	}
	707
	708	static constexpr IrInstSrcId ir_inst_id(IrInstSrcMulAdd *) {
	709	return IrInstSrcIdMulAdd;
	710	}
	711
	712	static constexpr IrInstSrcId ir_inst_id(IrInstSrcFloatOp *) {
	713	return IrInstSrcIdFloatOp;
	714	}
	715
	716	static constexpr IrInstSrcId ir_inst_id(IrInstSrcUnwrapErrCode *) {
	717	return IrInstSrcIdUnwrapErrCode;
	718	}
	719
	720	static constexpr IrInstSrcId ir_inst_id(IrInstSrcUnwrapErrPayload *) {
	721	return IrInstSrcIdUnwrapErrPayload;
	722	}
	723
	724	static constexpr IrInstSrcId ir_inst_id(IrInstSrcFnProto *) {
	725	return IrInstSrcIdFnProto;
	726	}
	727
	728	static constexpr IrInstSrcId ir_inst_id(IrInstSrcTestComptime *) {
	729	return IrInstSrcIdTestComptime;
	730	}
	731
	732	static constexpr IrInstSrcId ir_inst_id(IrInstSrcPtrCast *) {
	733	return IrInstSrcIdPtrCast;
	734	}
	735
	736	static constexpr IrInstSrcId ir_inst_id(IrInstSrcBitCast *) {
	737	return IrInstSrcIdBitCast;
	738	}
	739
	740	static constexpr IrInstSrcId ir_inst_id(IrInstSrcIntToPtr *) {
	741	return IrInstSrcIdIntToPtr;
	742	}
	743
	744	static constexpr IrInstSrcId ir_inst_id(IrInstSrcPtrToInt *) {
	745	return IrInstSrcIdPtrToInt;
	746	}
	747
	748	static constexpr IrInstSrcId ir_inst_id(IrInstSrcIntToEnum *) {
	749	return IrInstSrcIdIntToEnum;
	750	}
	751
	752	static constexpr IrInstSrcId ir_inst_id(IrInstSrcEnumToInt *) {
	753	return IrInstSrcIdEnumToInt;
	754	}
	755
	756	static constexpr IrInstSrcId ir_inst_id(IrInstSrcIntToErr *) {
	757	return IrInstSrcIdIntToErr;
	758	}
	759
	760	static constexpr IrInstSrcId ir_inst_id(IrInstSrcErrToInt *) {
	761	return IrInstSrcIdErrToInt;
	762	}
	763
	764	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCheckStatementIsVoid *) {
	765	return IrInstSrcIdCheckStatementIsVoid;
	766	}
	767
	768	static constexpr IrInstSrcId ir_inst_id(IrInstSrcTypeName *) {
	769	return IrInstSrcIdTypeName;
	770	}
	771
	772	static constexpr IrInstSrcId ir_inst_id(IrInstSrcDeclRef *) {
	773	return IrInstSrcIdDeclRef;
	774	}
	775
	776	static constexpr IrInstSrcId ir_inst_id(IrInstSrcPanic *) {
	777	return IrInstSrcIdPanic;
	778	}
	779
	780	static constexpr IrInstSrcId ir_inst_id(IrInstSrcTagName *) {
	781	return IrInstSrcIdTagName;
	782	}
	783
	784	static constexpr IrInstSrcId ir_inst_id(IrInstSrcFieldParentPtr *) {
	785	return IrInstSrcIdFieldParentPtr;
	786	}
	787
	788	static constexpr IrInstSrcId ir_inst_id(IrInstSrcByteOffsetOf *) {
	789	return IrInstSrcIdByteOffsetOf;
	790	}
	791
	792	static constexpr IrInstSrcId ir_inst_id(IrInstSrcBitOffsetOf *) {
	793	return IrInstSrcIdBitOffsetOf;
	794	}
	795
	796	static constexpr IrInstSrcId ir_inst_id(IrInstSrcTypeInfo *) {
	797	return IrInstSrcIdTypeInfo;
	798	}
	799
	800	static constexpr IrInstSrcId ir_inst_id(IrInstSrcType *) {
	801	return IrInstSrcIdType;
	802	}
	803
	804	static constexpr IrInstSrcId ir_inst_id(IrInstSrcHasField *) {
	805	return IrInstSrcIdHasField;
	806	}
	807
	808	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSetEvalBranchQuota *) {
	809	return IrInstSrcIdSetEvalBranchQuota;
	810	}
	811
	812	static constexpr IrInstSrcId ir_inst_id(IrInstSrcPtrType *) {
	813	return IrInstSrcIdPtrType;
	814	}
	815
	816	static constexpr IrInstSrcId ir_inst_id(IrInstSrcAlignCast *) {
	817	return IrInstSrcIdAlignCast;
	818	}
	819
	820	static constexpr IrInstSrcId ir_inst_id(IrInstSrcImplicitCast *) {
	821	return IrInstSrcIdImplicitCast;
	822	}
	823
	824	static constexpr IrInstSrcId ir_inst_id(IrInstSrcResolveResult *) {
	825	return IrInstSrcIdResolveResult;
	826	}
	827
	828	static constexpr IrInstSrcId ir_inst_id(IrInstSrcResetResult *) {
	829	return IrInstSrcIdResetResult;
	830	}
	831
	832	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSetAlignStack *) {
	833	return IrInstSrcIdSetAlignStack;
	834	}
	835
	836	static constexpr IrInstSrcId ir_inst_id(IrInstSrcExport *) {
	837	return IrInstSrcIdExport;
	838	}
	839
	840	static constexpr IrInstSrcId ir_inst_id(IrInstSrcExtern *) {
	841	return IrInstSrcIdExtern;
	842	}
	843
	844	static constexpr IrInstSrcId ir_inst_id(IrInstSrcErrorReturnTrace *) {
	845	return IrInstSrcIdErrorReturnTrace;
	846	}
	847
	848	static constexpr IrInstSrcId ir_inst_id(IrInstSrcErrorUnion *) {
	849	return IrInstSrcIdErrorUnion;
	850	}
	851
	852	static constexpr IrInstSrcId ir_inst_id(IrInstSrcAtomicRmw *) {
	853	return IrInstSrcIdAtomicRmw;
	854	}
	855
	856	static constexpr IrInstSrcId ir_inst_id(IrInstSrcAtomicLoad *) {
	857	return IrInstSrcIdAtomicLoad;
	858	}
	859
	860	static constexpr IrInstSrcId ir_inst_id(IrInstSrcAtomicStore *) {
	861	return IrInstSrcIdAtomicStore;
	862	}
	863
	864	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSaveErrRetAddr *) {
	865	return IrInstSrcIdSaveErrRetAddr;
	866	}
	867
	868	static constexpr IrInstSrcId ir_inst_id(IrInstSrcAddImplicitReturnType *) {
	869	return IrInstSrcIdAddImplicitReturnType;
	870	}
	871
	872	static constexpr IrInstSrcId ir_inst_id(IrInstSrcErrSetCast *) {
	873	return IrInstSrcIdErrSetCast;
	874	}
	875
	876	static constexpr IrInstSrcId ir_inst_id(IrInstSrcCheckRuntimeScope *) {
	877	return IrInstSrcIdCheckRuntimeScope;
	878	}
	879
	880	static constexpr IrInstSrcId ir_inst_id(IrInstSrcHasDecl *) {
	881	return IrInstSrcIdHasDecl;
	882	}
	883
	884	static constexpr IrInstSrcId ir_inst_id(IrInstSrcUndeclaredIdent *) {
	885	return IrInstSrcIdUndeclaredIdent;
	886	}
	887
	888	static constexpr IrInstSrcId ir_inst_id(IrInstSrcAlloca *) {
	889	return IrInstSrcIdAlloca;
	890	}
	891
	892	static constexpr IrInstSrcId ir_inst_id(IrInstSrcEndExpr *) {
	893	return IrInstSrcIdEndExpr;
	894	}
	895
	896	static constexpr IrInstSrcId ir_inst_id(IrInstSrcUnionInitNamedField *) {
	897	return IrInstSrcIdUnionInitNamedField;
	898	}
	899
	900	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSuspendBegin *) {
	901	return IrInstSrcIdSuspendBegin;
	902	}
	903
	904	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSuspendFinish *) {
	905	return IrInstSrcIdSuspendFinish;
	906	}
	907
	908	static constexpr IrInstSrcId ir_inst_id(IrInstSrcAwait *) {
	909	return IrInstSrcIdAwait;
	910	}
	911
	912	static constexpr IrInstSrcId ir_inst_id(IrInstSrcResume *) {
	913	return IrInstSrcIdResume;
	914	}
	915
	916	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSpillBegin *) {
	917	return IrInstSrcIdSpillBegin;
	918	}
	919
	920	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSpillEnd *) {
	921	return IrInstSrcIdSpillEnd;
	922	}
	923
	924	static constexpr IrInstSrcId ir_inst_id(IrInstSrcWasmMemorySize *) {
	925	return IrInstSrcIdWasmMemorySize;
	926	}
	927
	928	static constexpr IrInstSrcId ir_inst_id(IrInstSrcWasmMemoryGrow *) {
	929	return IrInstSrcIdWasmMemoryGrow;
	930	}
	931
	932	static constexpr IrInstSrcId ir_inst_id(IrInstSrcSrc *) {
	933	return IrInstSrcIdSrc;
	934	}
	935
	936	template<typename T>
	937	static T ir_create_instruction(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	938	T *special_instruction = heap::c_allocator.create<T>();
	939	special_instruction->base.id = ir_inst_id(special_instruction);
	940	special_instruction->base.base.scope = scope;
	941	special_instruction->base.base.source_node = source_node;
	942	special_instruction->base.base.debug_id = exec_next_debug_id(irb->exec);
	943	special_instruction->base.owner_bb = irb->current_basic_block;
	944	return special_instruction;
	945	}
	946
	947	template<typename T>
	948	static T ir_build_instruction(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	949	T *special_instruction = ir_create_instruction<T>(irb, scope, source_node);
	950	ir_instruction_append(irb->current_basic_block, &special_instruction->base);
	951	return special_instruction;
	952	}
	953
	954	static IrInstSrc ir_build_cond_br(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *condition,
	955	IrBasicBlockSrc then_block, IrBasicBlockSrc else_block, IrInstSrc *is_comptime)
	956	{
	957	IrInstSrcCondBr *inst = ir_build_instruction<IrInstSrcCondBr>(irb, scope, source_node);
	958	inst->base.is_noreturn = true;
	959	inst->condition = condition;
	960	inst->then_block = then_block;
	961	inst->else_block = else_block;
	962	inst->is_comptime = is_comptime;
	963
	964	ir_ref_instruction(condition, irb->current_basic_block);
	965	ir_ref_bb(then_block);
	966	ir_ref_bb(else_block);
	967	if (is_comptime != nullptr) ir_ref_instruction(is_comptime, irb->current_basic_block);
	968
	969	return &inst->base;
	970	}
	971
	972	static IrInstSrc ir_build_return_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *operand) {
	973	IrInstSrcReturn *inst = ir_build_instruction<IrInstSrcReturn>(irb, scope, source_node);
	974	inst->base.is_noreturn = true;
	975	inst->operand = operand;
	976
	977	if (operand != nullptr) ir_ref_instruction(operand, irb->current_basic_block);
	978
	979	return &inst->base;
	980	}
	981
	982	static IrInstSrc ir_build_const_void(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	983	IrInstSrcConst *const_instruction = ir_create_instruction<IrInstSrcConst>(irb, scope, source_node);
	984	ir_instruction_append(irb->current_basic_block, &const_instruction->base);
	985	const_instruction->value = irb->codegen->intern.for_void();
	986	return &const_instruction->base;
	987	}
	988
	989	static IrInstSrc ir_build_const_undefined(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	990	IrInstSrcConst *const_instruction = ir_create_instruction<IrInstSrcConst>(irb, scope, source_node);
	991	ir_instruction_append(irb->current_basic_block, &const_instruction->base);
	992	const_instruction->value = irb->codegen->intern.for_undefined();
	993	const_instruction->value->special = ConstValSpecialUndef;
	994	return &const_instruction->base;
	995	}
	996
	997	static IrInstSrc ir_build_const_uint(IrBuilderSrc irb, Scope scope, AstNode source_node, uint64_t value) {
	998	IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
	999	const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
	1000	const_instruction->value->type = irb->codegen->builtin_types.entry_num_lit_int;
	1001	const_instruction->value->special = ConstValSpecialStatic;
	1002	bigint_init_unsigned(&const_instruction->value->data.x_bigint, value);
	1003	return &const_instruction->base;
	1004	}
	1005
	1006	static IrInstSrc ir_build_const_bigint(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1007	BigInt bigint)
	1008	{
	1009	IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
	1010	const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
	1011	const_instruction->value->type = irb->codegen->builtin_types.entry_num_lit_int;
	1012	const_instruction->value->special = ConstValSpecialStatic;
	1013	const_instruction->value->data.x_bigint = bigint;
	1014	return &const_instruction->base;
	1015	}
	1016
	1017	static IrInstSrc ir_build_const_bigfloat(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1018	BigFloat bigfloat)
	1019	{
	1020	IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
	1021	const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
	1022	const_instruction->value->type = irb->codegen->builtin_types.entry_num_lit_float;
	1023	const_instruction->value->special = ConstValSpecialStatic;
	1024	const_instruction->value->data.x_bigfloat = bigfloat;
	1025	return &const_instruction->base;
	1026	}
	1027
	1028	static IrInstSrc ir_build_const_null(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	1029	IrInstSrcConst *const_instruction = ir_create_instruction<IrInstSrcConst>(irb, scope, source_node);
	1030	ir_instruction_append(irb->current_basic_block, &const_instruction->base);
	1031	const_instruction->value = irb->codegen->intern.for_null();
	1032	return &const_instruction->base;
	1033	}
	1034
	1035	static IrInstSrc ir_build_const_usize(IrBuilderSrc irb, Scope scope, AstNode source_node, uint64_t value) {
	1036	IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
	1037	const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
	1038	const_instruction->value->type = irb->codegen->builtin_types.entry_usize;
	1039	const_instruction->value->special = ConstValSpecialStatic;
	1040	bigint_init_unsigned(&const_instruction->value->data.x_bigint, value);
	1041	return &const_instruction->base;
	1042	}
	1043
	1044	static IrInstSrc ir_create_const_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1045	ZigType *type_entry)
	1046	{
	1047	IrInstSrcConst *const_instruction = ir_create_instruction<IrInstSrcConst>(irb, scope, source_node);
	1048	const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
	1049	const_instruction->value->type = irb->codegen->builtin_types.entry_type;
	1050	const_instruction->value->special = ConstValSpecialStatic;
	1051	const_instruction->value->data.x_type = type_entry;
	1052	return &const_instruction->base;
	1053	}
	1054
	1055	static IrInstSrc ir_build_const_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1056	ZigType *type_entry)
	1057	{
	1058	IrInstSrc *instruction = ir_create_const_type(irb, scope, source_node, type_entry);
	1059	ir_instruction_append(irb->current_basic_block, instruction);
	1060	return instruction;
	1061	}
	1062
	1063	static IrInstSrc ir_build_const_import(IrBuilderSrc irb, Scope scope, AstNode source_node, ZigType *import) {
	1064	IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
	1065	const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
	1066	const_instruction->value->type = irb->codegen->builtin_types.entry_type;
	1067	const_instruction->value->special = ConstValSpecialStatic;
	1068	const_instruction->value->data.x_type = import;
	1069	return &const_instruction->base;
	1070	}
	1071
	1072	static IrInstSrc ir_build_const_bool(IrBuilderSrc irb, Scope scope, AstNode source_node, bool value) {
	1073	IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
	1074	const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
	1075	const_instruction->value->type = irb->codegen->builtin_types.entry_bool;
	1076	const_instruction->value->special = ConstValSpecialStatic;
	1077	const_instruction->value->data.x_bool = value;
	1078	return &const_instruction->base;
	1079	}
	1080
	1081	static IrInstSrc ir_build_const_enum_literal(IrBuilderSrc irb, Scope scope, AstNode source_node, Buf *name) {
	1082	IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
	1083	const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
	1084	const_instruction->value->type = irb->codegen->builtin_types.entry_enum_literal;
	1085	const_instruction->value->special = ConstValSpecialStatic;
	1086	const_instruction->value->data.x_enum_literal = name;
	1087	return &const_instruction->base;
	1088	}
	1089
	1090	// Consumes `str`.
	1091	static IrInstSrc ir_create_const_str_lit(IrBuilderSrc irb, Scope scope, AstNode source_node, Buf *str) {
	1092	IrInstSrcConst *const_instruction = ir_create_instruction<IrInstSrcConst>(irb, scope, source_node);
	1093	const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
	1094	init_const_str_lit(irb->codegen, const_instruction->value, str, true);
	1095
	1096	return &const_instruction->base;
	1097	}
	1098
	1099	// Consumes `str`.
	1100	static IrInstSrc ir_build_const_str_lit(IrBuilderSrc irb, Scope scope, AstNode source_node, Buf *str) {
	1101	IrInstSrc *instruction = ir_create_const_str_lit(irb, scope, source_node, str);
	1102	ir_instruction_append(irb->current_basic_block, instruction);
	1103	return instruction;
	1104	}
	1105
	1106	static IrInstSrc ir_build_bin_op(IrBuilderSrc irb, Scope scope, AstNode source_node, IrBinOp op_id,
	1107	IrInstSrc op1, IrInstSrc op2, bool safety_check_on)
	1108	{
	1109	IrInstSrcBinOp *inst = ir_build_instruction<IrInstSrcBinOp>(irb, scope, source_node);
	1110	inst->op_id = op_id;
	1111	inst->op1 = op1;
	1112	inst->op2 = op2;
	1113	inst->safety_check_on = safety_check_on;
	1114
	1115	ir_ref_instruction(op1, irb->current_basic_block);
	1116	ir_ref_instruction(op2, irb->current_basic_block);
	1117
	1118	return &inst->base;
	1119	}
	1120
	1121	static IrInstSrc ir_build_merge_err_sets(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1122	IrInstSrc op1, IrInstSrc op2, Buf *type_name)
	1123	{
	1124	IrInstSrcMergeErrSets *inst = ir_build_instruction<IrInstSrcMergeErrSets>(irb, scope, source_node);
	1125	inst->op1 = op1;
	1126	inst->op2 = op2;
	1127	inst->type_name = type_name;
	1128
	1129	ir_ref_instruction(op1, irb->current_basic_block);
	1130	ir_ref_instruction(op2, irb->current_basic_block);
	1131
	1132	return &inst->base;
	1133	}
	1134
	1135	static IrInstSrc ir_build_var_ptr_x(IrBuilderSrc irb, Scope scope, AstNode source_node, ZigVar *var,
	1136	ScopeFnDef *crossed_fndef_scope)
	1137	{
	1138	IrInstSrcVarPtr *instruction = ir_build_instruction<IrInstSrcVarPtr>(irb, scope, source_node);
	1139	instruction->var = var;
	1140	instruction->crossed_fndef_scope = crossed_fndef_scope;
	1141
	1142	var->ref_count += 1;
	1143
	1144	return &instruction->base;
	1145	}
	1146
	1147	static IrInstSrc ir_build_var_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node, ZigVar *var) {
	1148	return ir_build_var_ptr_x(irb, scope, source_node, var, nullptr);
	1149	}
	1150
	1151	static IrInstSrc ir_build_elem_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1152	IrInstSrc array_ptr, IrInstSrc elem_index, bool safety_check_on, PtrLen ptr_len,
	1153	AstNode *init_array_type_source_node)
	1154	{
	1155	IrInstSrcElemPtr *instruction = ir_build_instruction<IrInstSrcElemPtr>(irb, scope, source_node);
	1156	instruction->array_ptr = array_ptr;
	1157	instruction->elem_index = elem_index;
	1158	instruction->safety_check_on = safety_check_on;
	1159	instruction->ptr_len = ptr_len;
	1160	instruction->init_array_type_source_node = init_array_type_source_node;
	1161
	1162	ir_ref_instruction(array_ptr, irb->current_basic_block);
	1163	ir_ref_instruction(elem_index, irb->current_basic_block);
	1164
	1165	return &instruction->base;
	1166	}
	1167
	1168	static IrInstSrc ir_build_field_ptr_instruction(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1169	IrInstSrc container_ptr, IrInstSrc field_name_expr, bool initializing)
	1170	{
	1171	IrInstSrcFieldPtr *instruction = ir_build_instruction<IrInstSrcFieldPtr>(irb, scope, source_node);
	1172	instruction->container_ptr = container_ptr;
	1173	instruction->field_name_buffer = nullptr;
	1174	instruction->field_name_expr = field_name_expr;
	1175	instruction->initializing = initializing;
	1176
	1177	ir_ref_instruction(container_ptr, irb->current_basic_block);
	1178	ir_ref_instruction(field_name_expr, irb->current_basic_block);
	1179
	1180	return &instruction->base;
	1181	}
	1182
	1183	static IrInstSrc ir_build_field_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1184	IrInstSrc container_ptr, Buf field_name, bool initializing)
	1185	{
	1186	IrInstSrcFieldPtr *instruction = ir_build_instruction<IrInstSrcFieldPtr>(irb, scope, source_node);
	1187	instruction->container_ptr = container_ptr;
	1188	instruction->field_name_buffer = field_name;
	1189	instruction->field_name_expr = nullptr;
	1190	instruction->initializing = initializing;
	1191
	1192	ir_ref_instruction(container_ptr, irb->current_basic_block);
	1193
	1194	return &instruction->base;
	1195	}
	1196
	1197	static IrInstSrc ir_build_has_field(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1198	IrInstSrc container_type, IrInstSrc field_name)
	1199	{
	1200	IrInstSrcHasField *instruction = ir_build_instruction<IrInstSrcHasField>(irb, scope, source_node);
	1201	instruction->container_type = container_type;
	1202	instruction->field_name = field_name;
	1203
	1204	ir_ref_instruction(container_type, irb->current_basic_block);
	1205	ir_ref_instruction(field_name, irb->current_basic_block);
	1206
	1207	return &instruction->base;
	1208	}
	1209
	1210	static IrInstSrc ir_build_call_extra(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1211	IrInstSrc options, IrInstSrc fn_ref, IrInstSrc args, ResultLoc result_loc)
	1212	{
	1213	IrInstSrcCallExtra *call_instruction = ir_build_instruction<IrInstSrcCallExtra>(irb, scope, source_node);
	1214	call_instruction->options = options;
	1215	call_instruction->fn_ref = fn_ref;
	1216	call_instruction->args = args;
	1217	call_instruction->result_loc = result_loc;
	1218
	1219	ir_ref_instruction(options, irb->current_basic_block);
	1220	ir_ref_instruction(fn_ref, irb->current_basic_block);
	1221	ir_ref_instruction(args, irb->current_basic_block);
	1222
	1223	return &call_instruction->base;
	1224	}
	1225
	1226	static IrInstSrc ir_build_async_call_extra(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1227	CallModifier modifier, IrInstSrc fn_ref, IrInstSrc ret_ptr, IrInstSrc new_stack, IrInstSrc args, ResultLoc *result_loc)
	1228	{
	1229	IrInstSrcAsyncCallExtra *call_instruction = ir_build_instruction<IrInstSrcAsyncCallExtra>(irb, scope, source_node);
	1230	call_instruction->modifier = modifier;
	1231	call_instruction->fn_ref = fn_ref;
	1232	call_instruction->ret_ptr = ret_ptr;
	1233	call_instruction->new_stack = new_stack;
	1234	call_instruction->args = args;
	1235	call_instruction->result_loc = result_loc;
	1236
	1237	ir_ref_instruction(fn_ref, irb->current_basic_block);
	1238	if (ret_ptr != nullptr) ir_ref_instruction(ret_ptr, irb->current_basic_block);
	1239	ir_ref_instruction(new_stack, irb->current_basic_block);
	1240	ir_ref_instruction(args, irb->current_basic_block);
	1241
	1242	return &call_instruction->base;
	1243	}
	1244
	1245	static IrInstSrc ir_build_call_args(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1246	IrInstSrc options, IrInstSrc fn_ref, IrInstSrc **args_ptr, size_t args_len,
	1247	ResultLoc *result_loc)
	1248	{
	1249	IrInstSrcCallArgs *call_instruction = ir_build_instruction<IrInstSrcCallArgs>(irb, scope, source_node);
	1250	call_instruction->options = options;
	1251	call_instruction->fn_ref = fn_ref;
	1252	call_instruction->args_ptr = args_ptr;
	1253	call_instruction->args_len = args_len;
	1254	call_instruction->result_loc = result_loc;
	1255
	1256	ir_ref_instruction(options, irb->current_basic_block);
	1257	ir_ref_instruction(fn_ref, irb->current_basic_block);
	1258	for (size_t i = 0; i < args_len; i += 1)
	1259	ir_ref_instruction(args_ptr[i], irb->current_basic_block);
	1260
	1261	return &call_instruction->base;
	1262	}
	1263
	1264	static IrInstSrc ir_build_call_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1265	ZigFn fn_entry, IrInstSrc fn_ref, size_t arg_count, IrInstSrc **args,
	1266	IrInstSrc *ret_ptr, CallModifier modifier, bool is_async_call_builtin,
	1267	IrInstSrc new_stack, ResultLoc result_loc)
	1268	{
	1269	IrInstSrcCall *call_instruction = ir_build_instruction<IrInstSrcCall>(irb, scope, source_node);
	1270	call_instruction->fn_entry = fn_entry;
	1271	call_instruction->fn_ref = fn_ref;
	1272	call_instruction->args = args;
	1273	call_instruction->arg_count = arg_count;
	1274	call_instruction->modifier = modifier;
	1275	call_instruction->is_async_call_builtin = is_async_call_builtin;
	1276	call_instruction->new_stack = new_stack;
	1277	call_instruction->result_loc = result_loc;
	1278	call_instruction->ret_ptr = ret_ptr;
	1279
	1280	if (fn_ref != nullptr) ir_ref_instruction(fn_ref, irb->current_basic_block);
	1281	for (size_t i = 0; i < arg_count; i += 1)
	1282	ir_ref_instruction(args[i], irb->current_basic_block);
	1283	if (ret_ptr != nullptr) ir_ref_instruction(ret_ptr, irb->current_basic_block);
	1284	if (new_stack != nullptr) ir_ref_instruction(new_stack, irb->current_basic_block);
	1285
	1286	return &call_instruction->base;
	1287	}
	1288
	1289	static IrInstSrc ir_build_phi(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1290	size_t incoming_count, IrBasicBlockSrc incoming_blocks, IrInstSrc incoming_values,
	1291	ResultLocPeerParent *peer_parent)
	1292	{
	1293	assert(incoming_count != 0);
	1294	assert(incoming_count != SIZE_MAX);
	1295
	1296	IrInstSrcPhi *phi_instruction = ir_build_instruction<IrInstSrcPhi>(irb, scope, source_node);
	1297	phi_instruction->incoming_count = incoming_count;
	1298	phi_instruction->incoming_blocks = incoming_blocks;
	1299	phi_instruction->incoming_values = incoming_values;
	1300	phi_instruction->peer_parent = peer_parent;
	1301
	1302	for (size_t i = 0; i < incoming_count; i += 1) {
	1303	ir_ref_bb(incoming_blocks[i]);
	1304	ir_ref_instruction(incoming_values[i], irb->current_basic_block);
	1305	}
	1306
	1307	return &phi_instruction->base;
	1308	}
	1309
	1310	static IrInstSrc ir_build_br(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1311	IrBasicBlockSrc dest_block, IrInstSrc is_comptime)
	1312	{
	1313	IrInstSrcBr *inst = ir_build_instruction<IrInstSrcBr>(irb, scope, source_node);
	1314	inst->base.is_noreturn = true;
	1315	inst->dest_block = dest_block;
	1316	inst->is_comptime = is_comptime;
	1317
	1318	ir_ref_bb(dest_block);
	1319	if (is_comptime) ir_ref_instruction(is_comptime, irb->current_basic_block);
	1320
	1321	return &inst->base;
	1322	}
	1323
	1324	static IrInstSrc ir_build_ptr_type_simple(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1325	IrInstSrc *child_type, bool is_const)
	1326	{
	1327	IrInstSrcPtrTypeSimple *inst = heap::c_allocator.create<IrInstSrcPtrTypeSimple>();
	1328	inst->base.id = is_const ? IrInstSrcIdPtrTypeSimpleConst : IrInstSrcIdPtrTypeSimple;
	1329	inst->base.base.scope = scope;
	1330	inst->base.base.source_node = source_node;
	1331	inst->base.base.debug_id = exec_next_debug_id(irb->exec);
	1332	inst->base.owner_bb = irb->current_basic_block;
	1333	ir_instruction_append(irb->current_basic_block, &inst->base);
	1334
	1335	inst->child_type = child_type;
	1336
	1337	ir_ref_instruction(child_type, irb->current_basic_block);
	1338
	1339	return &inst->base;
	1340	}
	1341
	1342	static IrInstSrc ir_build_ptr_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1343	IrInstSrc *child_type, bool is_const, bool is_volatile, PtrLen ptr_len,
	1344	IrInstSrc sentinel, IrInstSrc align_value,
	1345	uint32_t bit_offset_start, uint32_t host_int_bytes, bool is_allow_zero)
	1346	{
	1347	if (!is_volatile && ptr_len == PtrLenSingle && sentinel == nullptr && align_value == nullptr &&
	1348	bit_offset_start == 0 && host_int_bytes == 0 && is_allow_zero == 0)
	1349	{
	1350	return ir_build_ptr_type_simple(irb, scope, source_node, child_type, is_const);
	1351	}
	1352
	1353	IrInstSrcPtrType *inst = ir_build_instruction<IrInstSrcPtrType>(irb, scope, source_node);
	1354	inst->sentinel = sentinel;
	1355	inst->align_value = align_value;
	1356	inst->child_type = child_type;
	1357	inst->is_const = is_const;
	1358	inst->is_volatile = is_volatile;
	1359	inst->ptr_len = ptr_len;
	1360	inst->bit_offset_start = bit_offset_start;
	1361	inst->host_int_bytes = host_int_bytes;
	1362	inst->is_allow_zero = is_allow_zero;
	1363
	1364	if (sentinel) ir_ref_instruction(sentinel, irb->current_basic_block);
	1365	if (align_value) ir_ref_instruction(align_value, irb->current_basic_block);
	1366	ir_ref_instruction(child_type, irb->current_basic_block);
	1367
	1368	return &inst->base;
	1369	}
	1370
	1371	static IrInstSrc ir_build_un_op_lval(IrBuilderSrc irb, Scope scope, AstNode source_node, IrUnOp op_id,
	1372	IrInstSrc value, LVal lval, ResultLoc result_loc)
	1373	{
	1374	IrInstSrcUnOp *instruction = ir_build_instruction<IrInstSrcUnOp>(irb, scope, source_node);
	1375	instruction->op_id = op_id;
	1376	instruction->value = value;
	1377	instruction->lval = lval;
	1378	instruction->result_loc = result_loc;
	1379
	1380	ir_ref_instruction(value, irb->current_basic_block);
	1381
	1382	return &instruction->base;
	1383	}
	1384
	1385	static IrInstSrc ir_build_un_op(IrBuilderSrc irb, Scope scope, AstNode source_node, IrUnOp op_id,
	1386	IrInstSrc *value)
	1387	{
	1388	return ir_build_un_op_lval(irb, scope, source_node, op_id, value, LValNone, nullptr);
	1389	}
	1390
	1391	static IrInstSrc ir_build_container_init_list(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1392	size_t item_count, IrInstSrc *elem_result_loc_list, IrInstSrc result_loc,
	1393	AstNode *init_array_type_source_node)
	1394	{
	1395	IrInstSrcContainerInitList *container_init_list_instruction =
	1396	ir_build_instruction<IrInstSrcContainerInitList>(irb, scope, source_node);
	1397	container_init_list_instruction->item_count = item_count;
	1398	container_init_list_instruction->elem_result_loc_list = elem_result_loc_list;
	1399	container_init_list_instruction->result_loc = result_loc;
	1400	container_init_list_instruction->init_array_type_source_node = init_array_type_source_node;
	1401
	1402	for (size_t i = 0; i < item_count; i += 1) {
	1403	ir_ref_instruction(elem_result_loc_list[i], irb->current_basic_block);
	1404	}
	1405	if (result_loc != nullptr) ir_ref_instruction(result_loc, irb->current_basic_block);
	1406
	1407	return &container_init_list_instruction->base;
	1408	}
	1409
	1410	static IrInstSrc ir_build_container_init_fields(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1411	size_t field_count, IrInstSrcContainerInitFieldsField fields, IrInstSrc result_loc)
	1412	{
	1413	IrInstSrcContainerInitFields *container_init_fields_instruction =
	1414	ir_build_instruction<IrInstSrcContainerInitFields>(irb, scope, source_node);
	1415	container_init_fields_instruction->field_count = field_count;
	1416	container_init_fields_instruction->fields = fields;
	1417	container_init_fields_instruction->result_loc = result_loc;
	1418
	1419	for (size_t i = 0; i < field_count; i += 1) {
	1420	ir_ref_instruction(fields[i].result_loc, irb->current_basic_block);
	1421	}
	1422	if (result_loc != nullptr) ir_ref_instruction(result_loc, irb->current_basic_block);
	1423
	1424	return &container_init_fields_instruction->base;
	1425	}
	1426
	1427	static IrInstSrc ir_build_unreachable(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	1428	IrInstSrcUnreachable *inst = ir_build_instruction<IrInstSrcUnreachable>(irb, scope, source_node);
	1429	inst->base.is_noreturn = true;
	1430	return &inst->base;
	1431	}
	1432
	1433	static IrInstSrcStorePtr ir_build_store_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1434	IrInstSrc ptr, IrInstSrc value)
	1435	{
	1436	IrInstSrcStorePtr *instruction = ir_build_instruction<IrInstSrcStorePtr>(irb, scope, source_node);
	1437	instruction->ptr = ptr;
	1438	instruction->value = value;
	1439
	1440	ir_ref_instruction(ptr, irb->current_basic_block);
	1441	ir_ref_instruction(value, irb->current_basic_block);
	1442
	1443	return instruction;
	1444	}
	1445
	1446	static IrInstSrc ir_build_var_decl_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1447	ZigVar var, IrInstSrc align_value, IrInstSrc *ptr)
	1448	{
	1449	IrInstSrcDeclVar *inst = ir_build_instruction<IrInstSrcDeclVar>(irb, scope, source_node);
	1450	inst->var = var;
	1451	inst->align_value = align_value;
	1452	inst->ptr = ptr;
	1453
	1454	if (align_value != nullptr) ir_ref_instruction(align_value, irb->current_basic_block);
	1455	ir_ref_instruction(ptr, irb->current_basic_block);
	1456
	1457	return &inst->base;
	1458	}
	1459
	1460	static IrInstSrc ir_build_export(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1461	IrInstSrc target, IrInstSrc options)
	1462	{
	1463	IrInstSrcExport *export_instruction = ir_build_instruction<IrInstSrcExport>(
	1464	irb, scope, source_node);
	1465	export_instruction->target = target;
	1466	export_instruction->options = options;
	1467
	1468	ir_ref_instruction(target, irb->current_basic_block);
	1469	ir_ref_instruction(options, irb->current_basic_block);
	1470
	1471	return &export_instruction->base;
	1472	}
	1473
	1474	static IrInstSrc ir_build_extern(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1475	IrInstSrc type, IrInstSrc options)
	1476	{
	1477	IrInstSrcExtern *extern_instruction = ir_build_instruction<IrInstSrcExtern>(
	1478	irb, scope, source_node);
	1479	extern_instruction->type = type;
	1480	extern_instruction->options = options;
	1481
	1482	ir_ref_instruction(type, irb->current_basic_block);
	1483	ir_ref_instruction(options, irb->current_basic_block);
	1484
	1485	return &extern_instruction->base;
	1486	}
	1487
	1488	static IrInstSrc ir_build_load_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *ptr) {
	1489	IrInstSrcLoadPtr *instruction = ir_build_instruction<IrInstSrcLoadPtr>(irb, scope, source_node);
	1490	instruction->ptr = ptr;
	1491
	1492	ir_ref_instruction(ptr, irb->current_basic_block);
	1493
	1494	return &instruction->base;
	1495	}
	1496
	1497	static IrInstSrc ir_build_typeof_n(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1498	IrInstSrc **values, size_t value_count)
	1499	{
	1500	assert(value_count >= 2);
	1501
	1502	IrInstSrcTypeOf *instruction = ir_build_instruction<IrInstSrcTypeOf>(irb, scope, source_node);
	1503	instruction->value.list = values;
	1504	instruction->value_count = value_count;
	1505
	1506	for (size_t i = 0; i < value_count; i++)
	1507	ir_ref_instruction(values[i], irb->current_basic_block);
	1508
	1509	return &instruction->base;
	1510	}
	1511
	1512	static IrInstSrc ir_build_typeof_1(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *value) {
	1513	IrInstSrcTypeOf *instruction = ir_build_instruction<IrInstSrcTypeOf>(irb, scope, source_node);
	1514	instruction->value.scalar = value;
	1515
	1516	ir_ref_instruction(value, irb->current_basic_block);
	1517
	1518	return &instruction->base;
	1519	}
	1520
	1521	static IrInstSrc ir_build_set_cold(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *is_cold) {
	1522	IrInstSrcSetCold *instruction = ir_build_instruction<IrInstSrcSetCold>(irb, scope, source_node);
	1523	instruction->is_cold = is_cold;
	1524
	1525	ir_ref_instruction(is_cold, irb->current_basic_block);
	1526
	1527	return &instruction->base;
	1528	}
	1529
	1530	static IrInstSrc ir_build_set_runtime_safety(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1531	IrInstSrc *safety_on)
	1532	{
	1533	IrInstSrcSetRuntimeSafety *inst = ir_build_instruction<IrInstSrcSetRuntimeSafety>(irb, scope, source_node);
	1534	inst->safety_on = safety_on;
	1535
	1536	ir_ref_instruction(safety_on, irb->current_basic_block);
	1537
	1538	return &inst->base;
	1539	}
	1540
	1541	static IrInstSrc ir_build_set_float_mode(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1542	IrInstSrc *mode_value)
	1543	{
	1544	IrInstSrcSetFloatMode *instruction = ir_build_instruction<IrInstSrcSetFloatMode>(irb, scope, source_node);
	1545	instruction->mode_value = mode_value;
	1546
	1547	ir_ref_instruction(mode_value, irb->current_basic_block);
	1548
	1549	return &instruction->base;
	1550	}
	1551
	1552	static IrInstSrc ir_build_array_type(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *size,
	1553	IrInstSrc sentinel, IrInstSrc child_type)
	1554	{
	1555	IrInstSrcArrayType *instruction = ir_build_instruction<IrInstSrcArrayType>(irb, scope, source_node);
	1556	instruction->size = size;
	1557	instruction->sentinel = sentinel;
	1558	instruction->child_type = child_type;
	1559
	1560	ir_ref_instruction(size, irb->current_basic_block);
	1561	if (sentinel != nullptr) ir_ref_instruction(sentinel, irb->current_basic_block);
	1562	ir_ref_instruction(child_type, irb->current_basic_block);
	1563
	1564	return &instruction->base;
	1565	}
	1566
	1567	static IrInstSrc ir_build_anyframe_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1568	IrInstSrc *payload_type)
	1569	{
	1570	IrInstSrcAnyFrameType *instruction = ir_build_instruction<IrInstSrcAnyFrameType>(irb, scope, source_node);
	1571	instruction->payload_type = payload_type;
	1572
	1573	if (payload_type != nullptr) ir_ref_instruction(payload_type, irb->current_basic_block);
	1574
	1575	return &instruction->base;
	1576	}
	1577
	1578	static IrInstSrc ir_build_slice_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1579	IrInstSrc *child_type, bool is_const, bool is_volatile,
	1580	IrInstSrc sentinel, IrInstSrc align_value, bool is_allow_zero)
	1581	{
	1582	IrInstSrcSliceType *instruction = ir_build_instruction<IrInstSrcSliceType>(irb, scope, source_node);
	1583	instruction->is_const = is_const;
	1584	instruction->is_volatile = is_volatile;
	1585	instruction->child_type = child_type;
	1586	instruction->sentinel = sentinel;
	1587	instruction->align_value = align_value;
	1588	instruction->is_allow_zero = is_allow_zero;
	1589
	1590	if (sentinel != nullptr) ir_ref_instruction(sentinel, irb->current_basic_block);
	1591	if (align_value != nullptr) ir_ref_instruction(align_value, irb->current_basic_block);
	1592	ir_ref_instruction(child_type, irb->current_basic_block);
	1593
	1594	return &instruction->base;
	1595	}
	1596
	1597	static IrInstSrc ir_build_asm_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1598	IrInstSrc asm_template, IrInstSrc input_list, IrInstSrc *output_types,
	1599	ZigVar **output_vars, size_t return_count, bool has_side_effects, bool is_global)
	1600	{
	1601	IrInstSrcAsm *instruction = ir_build_instruction<IrInstSrcAsm>(irb, scope, source_node);
	1602	instruction->asm_template = asm_template;
	1603	instruction->input_list = input_list;
	1604	instruction->output_types = output_types;
	1605	instruction->output_vars = output_vars;
	1606	instruction->return_count = return_count;
	1607	instruction->has_side_effects = has_side_effects;
	1608	instruction->is_global = is_global;
	1609
	1610	assert(source_node->type == NodeTypeAsmExpr);
	1611	for (size_t i = 0; i < source_node->data.asm_expr.output_list.length; i += 1) {
	1612	IrInstSrc *output_type = output_types[i];
	1613	if (output_type) ir_ref_instruction(output_type, irb->current_basic_block);
	1614	}
	1615
	1616	for (size_t i = 0; i < source_node->data.asm_expr.input_list.length; i += 1) {
	1617	IrInstSrc *input_value = input_list[i];
	1618	ir_ref_instruction(input_value, irb->current_basic_block);
	1619	}
	1620
	1621	return &instruction->base;
	1622	}
	1623
	1624	static IrInstSrc ir_build_size_of(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type_value,
	1625	bool bit_size)
	1626	{
	1627	IrInstSrcSizeOf *instruction = ir_build_instruction<IrInstSrcSizeOf>(irb, scope, source_node);
	1628	instruction->type_value = type_value;
	1629	instruction->bit_size = bit_size;
	1630
	1631	ir_ref_instruction(type_value, irb->current_basic_block);
	1632
	1633	return &instruction->base;
	1634	}
	1635
	1636	static IrInstSrc ir_build_test_non_null_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1637	IrInstSrc *value)
	1638	{
	1639	IrInstSrcTestNonNull *instruction = ir_build_instruction<IrInstSrcTestNonNull>(irb, scope, source_node);
	1640	instruction->value = value;
	1641
	1642	ir_ref_instruction(value, irb->current_basic_block);
	1643
	1644	return &instruction->base;
	1645	}
	1646
	1647	static IrInstSrc ir_build_optional_unwrap_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1648	IrInstSrc *base_ptr, bool safety_check_on)
	1649	{
	1650	IrInstSrcOptionalUnwrapPtr *instruction = ir_build_instruction<IrInstSrcOptionalUnwrapPtr>(irb, scope, source_node);
	1651	instruction->base_ptr = base_ptr;
	1652	instruction->safety_check_on = safety_check_on;
	1653
	1654	ir_ref_instruction(base_ptr, irb->current_basic_block);
	1655
	1656	return &instruction->base;
	1657	}
	1658
	1659	static IrInstSrc ir_build_clz(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type,
	1660	IrInstSrc *op)
	1661	{
	1662	IrInstSrcClz *instruction = ir_build_instruction<IrInstSrcClz>(irb, scope, source_node);
	1663	instruction->type = type;
	1664	instruction->op = op;
	1665
	1666	ir_ref_instruction(type, irb->current_basic_block);
	1667	ir_ref_instruction(op, irb->current_basic_block);
	1668
	1669	return &instruction->base;
	1670	}
	1671
	1672	static IrInstSrc ir_build_ctz(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type,
	1673	IrInstSrc *op)
	1674	{
	1675	IrInstSrcCtz *instruction = ir_build_instruction<IrInstSrcCtz>(irb, scope, source_node);
	1676	instruction->type = type;
	1677	instruction->op = op;
	1678
	1679	ir_ref_instruction(type, irb->current_basic_block);
	1680	ir_ref_instruction(op, irb->current_basic_block);
	1681
	1682	return &instruction->base;
	1683	}
	1684
	1685	static IrInstSrc ir_build_pop_count(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type,
	1686	IrInstSrc *op)
	1687	{
	1688	IrInstSrcPopCount *instruction = ir_build_instruction<IrInstSrcPopCount>(irb, scope, source_node);
	1689	instruction->type = type;
	1690	instruction->op = op;
	1691
	1692	ir_ref_instruction(type, irb->current_basic_block);
	1693	ir_ref_instruction(op, irb->current_basic_block);
	1694
	1695	return &instruction->base;
	1696	}
	1697
	1698	static IrInstSrc ir_build_bswap(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type,
	1699	IrInstSrc *op)
	1700	{
	1701	IrInstSrcBswap *instruction = ir_build_instruction<IrInstSrcBswap>(irb, scope, source_node);
	1702	instruction->type = type;
	1703	instruction->op = op;
	1704
	1705	ir_ref_instruction(type, irb->current_basic_block);
	1706	ir_ref_instruction(op, irb->current_basic_block);
	1707
	1708	return &instruction->base;
	1709	}
	1710
	1711	static IrInstSrc ir_build_bit_reverse(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type,
	1712	IrInstSrc *op)
	1713	{
	1714	IrInstSrcBitReverse *instruction = ir_build_instruction<IrInstSrcBitReverse>(irb, scope, source_node);
	1715	instruction->type = type;
	1716	instruction->op = op;
	1717
	1718	ir_ref_instruction(type, irb->current_basic_block);
	1719	ir_ref_instruction(op, irb->current_basic_block);
	1720
	1721	return &instruction->base;
	1722	}
	1723
	1724	static IrInstSrcSwitchBr ir_build_switch_br_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1725	IrInstSrc target_value, IrBasicBlockSrc else_block, size_t case_count, IrInstSrcSwitchBrCase *cases,
	1726	IrInstSrc is_comptime, IrInstSrc switch_prongs_void)
	1727	{
	1728	IrInstSrcSwitchBr *instruction = ir_build_instruction<IrInstSrcSwitchBr>(irb, scope, source_node);
	1729	instruction->base.is_noreturn = true;
	1730	instruction->target_value = target_value;
	1731	instruction->else_block = else_block;
	1732	instruction->case_count = case_count;
	1733	instruction->cases = cases;
	1734	instruction->is_comptime = is_comptime;
	1735	instruction->switch_prongs_void = switch_prongs_void;
	1736
	1737	ir_ref_instruction(target_value, irb->current_basic_block);
	1738	ir_ref_instruction(is_comptime, irb->current_basic_block);
	1739	ir_ref_bb(else_block);
	1740	ir_ref_instruction(switch_prongs_void, irb->current_basic_block);
	1741
	1742	for (size_t i = 0; i < case_count; i += 1) {
	1743	ir_ref_instruction(cases[i].value, irb->current_basic_block);
	1744	ir_ref_bb(cases[i].block);
	1745	}
	1746
	1747	return instruction;
	1748	}
	1749
	1750	static IrInstSrc ir_build_switch_target(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1751	IrInstSrc *target_value_ptr)
	1752	{
	1753	IrInstSrcSwitchTarget *instruction = ir_build_instruction<IrInstSrcSwitchTarget>(irb, scope, source_node);
	1754	instruction->target_value_ptr = target_value_ptr;
	1755
	1756	ir_ref_instruction(target_value_ptr, irb->current_basic_block);
	1757
	1758	return &instruction->base;
	1759	}
	1760
	1761	static IrInstSrc ir_build_switch_var(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1762	IrInstSrc target_value_ptr, IrInstSrc *prongs_ptr, size_t prongs_len)
	1763	{
	1764	IrInstSrcSwitchVar *instruction = ir_build_instruction<IrInstSrcSwitchVar>(irb, scope, source_node);
	1765	instruction->target_value_ptr = target_value_ptr;
	1766	instruction->prongs_ptr = prongs_ptr;
	1767	instruction->prongs_len = prongs_len;
	1768
	1769	ir_ref_instruction(target_value_ptr, irb->current_basic_block);
	1770	for (size_t i = 0; i < prongs_len; i += 1) {
	1771	ir_ref_instruction(prongs_ptr[i], irb->current_basic_block);
	1772	}
	1773
	1774	return &instruction->base;
	1775	}
	1776
	1777	// For this instruction the switch_br must be set later.
	1778	static IrInstSrcSwitchElseVar ir_build_switch_else_var(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1779	IrInstSrc *target_value_ptr)
	1780	{
	1781	IrInstSrcSwitchElseVar *instruction = ir_build_instruction<IrInstSrcSwitchElseVar>(irb, scope, source_node);
	1782	instruction->target_value_ptr = target_value_ptr;
	1783
	1784	ir_ref_instruction(target_value_ptr, irb->current_basic_block);
	1785
	1786	return instruction;
	1787	}
	1788
	1789	static IrInstSrc ir_build_import(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *name) {
	1790	IrInstSrcImport *instruction = ir_build_instruction<IrInstSrcImport>(irb, scope, source_node);
	1791	instruction->name = name;
	1792
	1793	ir_ref_instruction(name, irb->current_basic_block);
	1794
	1795	return &instruction->base;
	1796	}
	1797
	1798	static IrInstSrc ir_build_ref_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *value) {
	1799	IrInstSrcRef *instruction = ir_build_instruction<IrInstSrcRef>(irb, scope, source_node);
	1800	instruction->value = value;
	1801
	1802	ir_ref_instruction(value, irb->current_basic_block);
	1803
	1804	return &instruction->base;
	1805	}
	1806
	1807	static IrInstSrc ir_build_compile_err(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *msg) {
	1808	IrInstSrcCompileErr *instruction = ir_build_instruction<IrInstSrcCompileErr>(irb, scope, source_node);
	1809	instruction->msg = msg;
	1810
	1811	ir_ref_instruction(msg, irb->current_basic_block);
	1812
	1813	return &instruction->base;
	1814	}
	1815
	1816	static IrInstSrc ir_build_compile_log(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1817	size_t msg_count, IrInstSrc **msg_list)
	1818	{
	1819	IrInstSrcCompileLog *instruction = ir_build_instruction<IrInstSrcCompileLog>(irb, scope, source_node);
	1820	instruction->msg_count = msg_count;
	1821	instruction->msg_list = msg_list;
	1822
	1823	for (size_t i = 0; i < msg_count; i += 1) {
	1824	ir_ref_instruction(msg_list[i], irb->current_basic_block);
	1825	}
	1826
	1827	return &instruction->base;
	1828	}
	1829
	1830	static IrInstSrc ir_build_err_name(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *value) {
	1831	IrInstSrcErrName *instruction = ir_build_instruction<IrInstSrcErrName>(irb, scope, source_node);
	1832	instruction->value = value;
	1833
	1834	ir_ref_instruction(value, irb->current_basic_block);
	1835
	1836	return &instruction->base;
	1837	}
	1838
	1839	static IrInstSrc ir_build_c_import(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	1840	IrInstSrcCImport *instruction = ir_build_instruction<IrInstSrcCImport>(irb, scope, source_node);
	1841	return &instruction->base;
	1842	}
	1843
	1844	static IrInstSrc ir_build_c_include(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *name) {
	1845	IrInstSrcCInclude *instruction = ir_build_instruction<IrInstSrcCInclude>(irb, scope, source_node);
	1846	instruction->name = name;
	1847
	1848	ir_ref_instruction(name, irb->current_basic_block);
	1849
	1850	return &instruction->base;
	1851	}
	1852
	1853	static IrInstSrc ir_build_c_define(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc name, IrInstSrc value) {
	1854	IrInstSrcCDefine *instruction = ir_build_instruction<IrInstSrcCDefine>(irb, scope, source_node);
	1855	instruction->name = name;
	1856	instruction->value = value;
	1857
	1858	ir_ref_instruction(name, irb->current_basic_block);
	1859	ir_ref_instruction(value, irb->current_basic_block);
	1860
	1861	return &instruction->base;
	1862	}
	1863
	1864	static IrInstSrc ir_build_c_undef(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *name) {
	1865	IrInstSrcCUndef *instruction = ir_build_instruction<IrInstSrcCUndef>(irb, scope, source_node);
	1866	instruction->name = name;
	1867
	1868	ir_ref_instruction(name, irb->current_basic_block);
	1869
	1870	return &instruction->base;
	1871	}
	1872
	1873	static IrInstSrc ir_build_embed_file(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *name) {
	1874	IrInstSrcEmbedFile *instruction = ir_build_instruction<IrInstSrcEmbedFile>(irb, scope, source_node);
	1875	instruction->name = name;
	1876
	1877	ir_ref_instruction(name, irb->current_basic_block);
	1878
	1879	return &instruction->base;
	1880	}
	1881
	1882	static IrInstSrc ir_build_cmpxchg_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1883	IrInstSrc type_value, IrInstSrc ptr, IrInstSrc cmp_value, IrInstSrc new_value,
	1884	IrInstSrc success_order_value, IrInstSrc failure_order_value, bool is_weak, ResultLoc *result_loc)
	1885	{
	1886	IrInstSrcCmpxchg *instruction = ir_build_instruction<IrInstSrcCmpxchg>(irb, scope, source_node);
	1887	instruction->type_value = type_value;
	1888	instruction->ptr = ptr;
	1889	instruction->cmp_value = cmp_value;
	1890	instruction->new_value = new_value;
	1891	instruction->success_order_value = success_order_value;
	1892	instruction->failure_order_value = failure_order_value;
	1893	instruction->is_weak = is_weak;
	1894	instruction->result_loc = result_loc;
	1895
	1896	ir_ref_instruction(type_value, irb->current_basic_block);
	1897	ir_ref_instruction(ptr, irb->current_basic_block);
	1898	ir_ref_instruction(cmp_value, irb->current_basic_block);
	1899	ir_ref_instruction(new_value, irb->current_basic_block);
	1900	ir_ref_instruction(success_order_value, irb->current_basic_block);
	1901	ir_ref_instruction(failure_order_value, irb->current_basic_block);
	1902
	1903	return &instruction->base;
	1904	}
	1905
	1906	static IrInstSrc ir_build_fence(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *order) {
	1907	IrInstSrcFence *instruction = ir_build_instruction<IrInstSrcFence>(irb, scope, source_node);
	1908	instruction->order = order;
	1909
	1910	ir_ref_instruction(order, irb->current_basic_block);
	1911
	1912	return &instruction->base;
	1913	}
	1914
	1915	static IrInstSrc ir_build_reduce(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc op, IrInstSrc value) {
	1916	IrInstSrcReduce *instruction = ir_build_instruction<IrInstSrcReduce>(irb, scope, source_node);
	1917	instruction->op = op;
	1918	instruction->value = value;
	1919
	1920	ir_ref_instruction(op, irb->current_basic_block);
	1921	ir_ref_instruction(value, irb->current_basic_block);
	1922
	1923	return &instruction->base;
	1924	}
	1925
	1926	static IrInstSrc ir_build_truncate(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1927	IrInstSrc dest_type, IrInstSrc target)
	1928	{
	1929	IrInstSrcTruncate *instruction = ir_build_instruction<IrInstSrcTruncate>(irb, scope, source_node);
	1930	instruction->dest_type = dest_type;
	1931	instruction->target = target;
	1932
	1933	ir_ref_instruction(dest_type, irb->current_basic_block);
	1934	ir_ref_instruction(target, irb->current_basic_block);
	1935
	1936	return &instruction->base;
	1937	}
	1938
	1939	static IrInstSrc ir_build_int_cast(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *dest_type,
	1940	IrInstSrc *target)
	1941	{
	1942	IrInstSrcIntCast *instruction = ir_build_instruction<IrInstSrcIntCast>(irb, scope, source_node);
	1943	instruction->dest_type = dest_type;
	1944	instruction->target = target;
	1945
	1946	ir_ref_instruction(dest_type, irb->current_basic_block);
	1947	ir_ref_instruction(target, irb->current_basic_block);
	1948
	1949	return &instruction->base;
	1950	}
	1951
	1952	static IrInstSrc ir_build_float_cast(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *dest_type,
	1953	IrInstSrc *target)
	1954	{
	1955	IrInstSrcFloatCast *instruction = ir_build_instruction<IrInstSrcFloatCast>(irb, scope, source_node);
	1956	instruction->dest_type = dest_type;
	1957	instruction->target = target;
	1958
	1959	ir_ref_instruction(dest_type, irb->current_basic_block);
	1960	ir_ref_instruction(target, irb->current_basic_block);
	1961
	1962	return &instruction->base;
	1963	}
	1964
	1965	static IrInstSrc ir_build_err_set_cast(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1966	IrInstSrc dest_type, IrInstSrc target)
	1967	{
	1968	IrInstSrcErrSetCast *instruction = ir_build_instruction<IrInstSrcErrSetCast>(irb, scope, source_node);
	1969	instruction->dest_type = dest_type;
	1970	instruction->target = target;
	1971
	1972	ir_ref_instruction(dest_type, irb->current_basic_block);
	1973	ir_ref_instruction(target, irb->current_basic_block);
	1974
	1975	return &instruction->base;
	1976	}
	1977
	1978	static IrInstSrc ir_build_int_to_float(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1979	IrInstSrc dest_type, IrInstSrc target)
	1980	{
	1981	IrInstSrcIntToFloat *instruction = ir_build_instruction<IrInstSrcIntToFloat>(irb, scope, source_node);
	1982	instruction->dest_type = dest_type;
	1983	instruction->target = target;
	1984
	1985	ir_ref_instruction(dest_type, irb->current_basic_block);
	1986	ir_ref_instruction(target, irb->current_basic_block);
	1987
	1988	return &instruction->base;
	1989	}
	1990
	1991	static IrInstSrc ir_build_float_to_int(IrBuilderSrc irb, Scope scope, AstNode source_node,
	1992	IrInstSrc dest_type, IrInstSrc target)
	1993	{
	1994	IrInstSrcFloatToInt *instruction = ir_build_instruction<IrInstSrcFloatToInt>(irb, scope, source_node);
	1995	instruction->dest_type = dest_type;
	1996	instruction->target = target;
	1997
	1998	ir_ref_instruction(dest_type, irb->current_basic_block);
	1999	ir_ref_instruction(target, irb->current_basic_block);
	2000
	2001	return &instruction->base;
	2002	}
	2003
	2004	static IrInstSrc ir_build_bool_to_int(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *target) {
	2005	IrInstSrcBoolToInt *instruction = ir_build_instruction<IrInstSrcBoolToInt>(irb, scope, source_node);
	2006	instruction->target = target;
	2007
	2008	ir_ref_instruction(target, irb->current_basic_block);
	2009
	2010	return &instruction->base;
	2011	}
	2012
	2013	static IrInstSrc ir_build_vector_type(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *len,
	2014	IrInstSrc *elem_type)
	2015	{
	2016	IrInstSrcVectorType *instruction = ir_build_instruction<IrInstSrcVectorType>(irb, scope, source_node);
	2017	instruction->len = len;
	2018	instruction->elem_type = elem_type;
	2019
	2020	ir_ref_instruction(len, irb->current_basic_block);
	2021	ir_ref_instruction(elem_type, irb->current_basic_block);
	2022
	2023	return &instruction->base;
	2024	}
	2025
	2026	static IrInstSrc ir_build_shuffle_vector(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2027	IrInstSrc scalar_type, IrInstSrc a, IrInstSrc b, IrInstSrc mask)
	2028	{
	2029	IrInstSrcShuffleVector *instruction = ir_build_instruction<IrInstSrcShuffleVector>(irb, scope, source_node);
	2030	instruction->scalar_type = scalar_type;
	2031	instruction->a = a;
	2032	instruction->b = b;
	2033	instruction->mask = mask;
	2034
	2035	if (scalar_type != nullptr) ir_ref_instruction(scalar_type, irb->current_basic_block);
	2036	ir_ref_instruction(a, irb->current_basic_block);
	2037	ir_ref_instruction(b, irb->current_basic_block);
	2038	ir_ref_instruction(mask, irb->current_basic_block);
	2039
	2040	return &instruction->base;
	2041	}
	2042
	2043	static IrInstSrc ir_build_splat_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2044	IrInstSrc len, IrInstSrc scalar)
	2045	{
	2046	IrInstSrcSplat *instruction = ir_build_instruction<IrInstSrcSplat>(irb, scope, source_node);
	2047	instruction->len = len;
	2048	instruction->scalar = scalar;
	2049
	2050	ir_ref_instruction(len, irb->current_basic_block);
	2051	ir_ref_instruction(scalar, irb->current_basic_block);
	2052
	2053	return &instruction->base;
	2054	}
	2055
	2056	static IrInstSrc ir_build_bool_not(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *value) {
	2057	IrInstSrcBoolNot *instruction = ir_build_instruction<IrInstSrcBoolNot>(irb, scope, source_node);
	2058	instruction->value = value;
	2059
	2060	ir_ref_instruction(value, irb->current_basic_block);
	2061
	2062	return &instruction->base;
	2063	}
	2064
	2065	static IrInstSrc ir_build_memset_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2066	IrInstSrc dest_ptr, IrInstSrc byte, IrInstSrc *count)
	2067	{
	2068	IrInstSrcMemset *instruction = ir_build_instruction<IrInstSrcMemset>(irb, scope, source_node);
	2069	instruction->dest_ptr = dest_ptr;
	2070	instruction->byte = byte;
	2071	instruction->count = count;
	2072
	2073	ir_ref_instruction(dest_ptr, irb->current_basic_block);
	2074	ir_ref_instruction(byte, irb->current_basic_block);
	2075	ir_ref_instruction(count, irb->current_basic_block);
	2076
	2077	return &instruction->base;
	2078	}
	2079
	2080	static IrInstSrc ir_build_memcpy_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2081	IrInstSrc dest_ptr, IrInstSrc src_ptr, IrInstSrc *count)
	2082	{
	2083	IrInstSrcMemcpy *instruction = ir_build_instruction<IrInstSrcMemcpy>(irb, scope, source_node);
	2084	instruction->dest_ptr = dest_ptr;
	2085	instruction->src_ptr = src_ptr;
	2086	instruction->count = count;
	2087
	2088	ir_ref_instruction(dest_ptr, irb->current_basic_block);
	2089	ir_ref_instruction(src_ptr, irb->current_basic_block);
	2090	ir_ref_instruction(count, irb->current_basic_block);
	2091
	2092	return &instruction->base;
	2093	}
	2094
	2095	static IrInstSrc ir_build_slice_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2096	IrInstSrc ptr, IrInstSrc start, IrInstSrc end, IrInstSrc sentinel,
	2097	bool safety_check_on, ResultLoc *result_loc)
	2098	{
	2099	IrInstSrcSlice *instruction = ir_build_instruction<IrInstSrcSlice>(irb, scope, source_node);
	2100	instruction->ptr = ptr;
	2101	instruction->start = start;
	2102	instruction->end = end;
	2103	instruction->sentinel = sentinel;
	2104	instruction->safety_check_on = safety_check_on;
	2105	instruction->result_loc = result_loc;
	2106
	2107	ir_ref_instruction(ptr, irb->current_basic_block);
	2108	ir_ref_instruction(start, irb->current_basic_block);
	2109	if (end) ir_ref_instruction(end, irb->current_basic_block);
	2110	if (sentinel) ir_ref_instruction(sentinel, irb->current_basic_block);
	2111
	2112	return &instruction->base;
	2113	}
	2114
	2115	static IrInstSrc ir_build_breakpoint(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	2116	IrInstSrcBreakpoint *instruction = ir_build_instruction<IrInstSrcBreakpoint>(irb, scope, source_node);
	2117	return &instruction->base;
	2118	}
	2119
	2120	static IrInstSrc ir_build_return_address_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	2121	IrInstSrcReturnAddress *instruction = ir_build_instruction<IrInstSrcReturnAddress>(irb, scope, source_node);
	2122	return &instruction->base;
	2123	}
	2124
	2125	static IrInstSrc ir_build_frame_address_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	2126	IrInstSrcFrameAddress *inst = ir_build_instruction<IrInstSrcFrameAddress>(irb, scope, source_node);
	2127	return &inst->base;
	2128	}
	2129
	2130	static IrInstSrc ir_build_handle_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	2131	IrInstSrcFrameHandle *inst = ir_build_instruction<IrInstSrcFrameHandle>(irb, scope, source_node);
	2132	return &inst->base;
	2133	}
	2134
	2135	static IrInstSrc ir_build_frame_type(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *fn) {
	2136	IrInstSrcFrameType *inst = ir_build_instruction<IrInstSrcFrameType>(irb, scope, source_node);
	2137	inst->fn = fn;
	2138
	2139	ir_ref_instruction(fn, irb->current_basic_block);
	2140
	2141	return &inst->base;
	2142	}
	2143
	2144	static IrInstSrc ir_build_frame_size_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *fn) {
	2145	IrInstSrcFrameSize *inst = ir_build_instruction<IrInstSrcFrameSize>(irb, scope, source_node);
	2146	inst->fn = fn;
	2147
	2148	ir_ref_instruction(fn, irb->current_basic_block);
	2149
	2150	return &inst->base;
	2151	}
	2152
	2153	static IrInstSrc ir_build_overflow_op_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2154	IrOverflowOp op, IrInstSrc type_value, IrInstSrc op1, IrInstSrc op2, IrInstSrc result_ptr)
	2155	{
	2156	IrInstSrcOverflowOp *instruction = ir_build_instruction<IrInstSrcOverflowOp>(irb, scope, source_node);
	2157	instruction->op = op;
	2158	instruction->type_value = type_value;
	2159	instruction->op1 = op1;
	2160	instruction->op2 = op2;
	2161	instruction->result_ptr = result_ptr;
	2162
	2163	ir_ref_instruction(type_value, irb->current_basic_block);
	2164	ir_ref_instruction(op1, irb->current_basic_block);
	2165	ir_ref_instruction(op2, irb->current_basic_block);
	2166	ir_ref_instruction(result_ptr, irb->current_basic_block);
	2167
	2168	return &instruction->base;
	2169	}
	2170
	2171	static IrInstSrc ir_build_float_op_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *operand,
	2172	BuiltinFnId fn_id)
	2173	{
	2174	IrInstSrcFloatOp *instruction = ir_build_instruction<IrInstSrcFloatOp>(irb, scope, source_node);
	2175	instruction->operand = operand;
	2176	instruction->fn_id = fn_id;
	2177
	2178	ir_ref_instruction(operand, irb->current_basic_block);
	2179
	2180	return &instruction->base;
	2181	}
	2182
	2183	static IrInstSrc ir_build_mul_add_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2184	IrInstSrc type_value, IrInstSrc op1, IrInstSrc op2, IrInstSrc op3)
	2185	{
	2186	IrInstSrcMulAdd *instruction = ir_build_instruction<IrInstSrcMulAdd>(irb, scope, source_node);
	2187	instruction->type_value = type_value;
	2188	instruction->op1 = op1;
	2189	instruction->op2 = op2;
	2190	instruction->op3 = op3;
	2191
	2192	ir_ref_instruction(type_value, irb->current_basic_block);
	2193	ir_ref_instruction(op1, irb->current_basic_block);
	2194	ir_ref_instruction(op2, irb->current_basic_block);
	2195	ir_ref_instruction(op3, irb->current_basic_block);
	2196
	2197	return &instruction->base;
	2198	}
	2199
	2200	static IrInstSrc ir_build_align_of(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type_value) {
	2201	IrInstSrcAlignOf *instruction = ir_build_instruction<IrInstSrcAlignOf>(irb, scope, source_node);
	2202	instruction->type_value = type_value;
	2203
	2204	ir_ref_instruction(type_value, irb->current_basic_block);
	2205
	2206	return &instruction->base;
	2207	}
	2208
	2209	static IrInstSrc ir_build_test_err_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2210	IrInstSrc *base_ptr, bool resolve_err_set, bool base_ptr_is_payload)
	2211	{
	2212	IrInstSrcTestErr *instruction = ir_build_instruction<IrInstSrcTestErr>(irb, scope, source_node);
	2213	instruction->base_ptr = base_ptr;
	2214	instruction->resolve_err_set = resolve_err_set;
	2215	instruction->base_ptr_is_payload = base_ptr_is_payload;
	2216
	2217	ir_ref_instruction(base_ptr, irb->current_basic_block);
	2218
	2219	return &instruction->base;
	2220	}
	2221
	2222	static IrInstSrc ir_build_unwrap_err_code_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2223	IrInstSrc *err_union_ptr)
	2224	{
	2225	IrInstSrcUnwrapErrCode *inst = ir_build_instruction<IrInstSrcUnwrapErrCode>(irb, scope, source_node);
	2226	inst->err_union_ptr = err_union_ptr;
	2227
	2228	ir_ref_instruction(err_union_ptr, irb->current_basic_block);
	2229
	2230	return &inst->base;
	2231	}
	2232
	2233	static IrInstSrc ir_build_unwrap_err_payload_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2234	IrInstSrc *value, bool safety_check_on, bool initializing)
	2235	{
	2236	IrInstSrcUnwrapErrPayload *inst = ir_build_instruction<IrInstSrcUnwrapErrPayload>(irb, scope, source_node);
	2237	inst->value = value;
	2238	inst->safety_check_on = safety_check_on;
	2239	inst->initializing = initializing;
	2240
	2241	ir_ref_instruction(value, irb->current_basic_block);
	2242
	2243	return &inst->base;
	2244	}
	2245
	2246	static IrInstSrc ir_build_fn_proto(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2247	IrInstSrc *param_types, IrInstSrc align_value, IrInstSrc *callconv_value,
	2248	IrInstSrc *return_type, bool is_var_args)
	2249	{
	2250	IrInstSrcFnProto *instruction = ir_build_instruction<IrInstSrcFnProto>(irb, scope, source_node);
	2251	instruction->param_types = param_types;
	2252	instruction->align_value = align_value;
	2253	instruction->callconv_value = callconv_value;
	2254	instruction->return_type = return_type;
	2255	instruction->is_var_args = is_var_args;
	2256
	2257	assert(source_node->type == NodeTypeFnProto);
	2258	size_t param_count = source_node->data.fn_proto.params.length;
	2259	if (is_var_args) param_count -= 1;
	2260	for (size_t i = 0; i < param_count; i += 1) {
	2261	if (param_types[i] != nullptr) ir_ref_instruction(param_types[i], irb->current_basic_block);
	2262	}
	2263	if (align_value != nullptr) ir_ref_instruction(align_value, irb->current_basic_block);
	2264	if (callconv_value != nullptr) ir_ref_instruction(callconv_value, irb->current_basic_block);
	2265	ir_ref_instruction(return_type, irb->current_basic_block);
	2266
	2267	return &instruction->base;
	2268	}
	2269
	2270	static IrInstSrc ir_build_test_comptime(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *value) {
	2271	IrInstSrcTestComptime *instruction = ir_build_instruction<IrInstSrcTestComptime>(irb, scope, source_node);
	2272	instruction->value = value;
	2273
	2274	ir_ref_instruction(value, irb->current_basic_block);
	2275
	2276	return &instruction->base;
	2277	}
	2278
	2279	static IrInstSrc ir_build_ptr_cast_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2280	IrInstSrc dest_type, IrInstSrc ptr, bool safety_check_on)
	2281	{
	2282	IrInstSrcPtrCast *instruction = ir_build_instruction<IrInstSrcPtrCast>(
	2283	irb, scope, source_node);
	2284	instruction->dest_type = dest_type;
	2285	instruction->ptr = ptr;
	2286	instruction->safety_check_on = safety_check_on;
	2287
	2288	ir_ref_instruction(dest_type, irb->current_basic_block);
	2289	ir_ref_instruction(ptr, irb->current_basic_block);
	2290
	2291	return &instruction->base;
	2292	}
	2293
	2294	static IrInstSrc ir_build_implicit_cast(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2295	IrInstSrc operand, ResultLocCast result_loc_cast)
	2296	{
	2297	IrInstSrcImplicitCast *instruction = ir_build_instruction<IrInstSrcImplicitCast>(irb, scope, source_node);
	2298	instruction->operand = operand;
	2299	instruction->result_loc_cast = result_loc_cast;
	2300
	2301	ir_ref_instruction(operand, irb->current_basic_block);
	2302
	2303	return &instruction->base;
	2304	}
	2305
	2306	static IrInstSrc ir_build_bit_cast_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2307	IrInstSrc operand, ResultLocBitCast result_loc_bit_cast)
	2308	{
	2309	IrInstSrcBitCast *instruction = ir_build_instruction<IrInstSrcBitCast>(irb, scope, source_node);
	2310	instruction->operand = operand;
	2311	instruction->result_loc_bit_cast = result_loc_bit_cast;
	2312
	2313	ir_ref_instruction(operand, irb->current_basic_block);
	2314
	2315	return &instruction->base;
	2316	}
	2317
	2318	static IrInstSrc ir_build_int_to_ptr_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2319	IrInstSrc dest_type, IrInstSrc target)
	2320	{
	2321	IrInstSrcIntToPtr *instruction = ir_build_instruction<IrInstSrcIntToPtr>(irb, scope, source_node);
	2322	instruction->dest_type = dest_type;
	2323	instruction->target = target;
	2324
	2325	ir_ref_instruction(dest_type, irb->current_basic_block);
	2326	ir_ref_instruction(target, irb->current_basic_block);
	2327
	2328	return &instruction->base;
	2329	}
	2330
	2331	static IrInstSrc ir_build_ptr_to_int_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2332	IrInstSrc *target)
	2333	{
	2334	IrInstSrcPtrToInt *inst = ir_build_instruction<IrInstSrcPtrToInt>(irb, scope, source_node);
	2335	inst->target = target;
	2336
	2337	ir_ref_instruction(target, irb->current_basic_block);
	2338
	2339	return &inst->base;
	2340	}
	2341
	2342	static IrInstSrc ir_build_int_to_enum_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2343	IrInstSrc dest_type, IrInstSrc target)
	2344	{
	2345	IrInstSrcIntToEnum *instruction = ir_build_instruction<IrInstSrcIntToEnum>(irb, scope, source_node);
	2346	instruction->dest_type = dest_type;
	2347	instruction->target = target;
	2348
	2349	if (dest_type) ir_ref_instruction(dest_type, irb->current_basic_block);
	2350	ir_ref_instruction(target, irb->current_basic_block);
	2351
	2352	return &instruction->base;
	2353	}
	2354
	2355	static IrInstSrc ir_build_enum_to_int(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2356	IrInstSrc *target)
	2357	{
	2358	IrInstSrcEnumToInt *instruction = ir_build_instruction<IrInstSrcEnumToInt>(
	2359	irb, scope, source_node);
	2360	instruction->target = target;
	2361
	2362	ir_ref_instruction(target, irb->current_basic_block);
	2363
	2364	return &instruction->base;
	2365	}
	2366
	2367	static IrInstSrc ir_build_int_to_err_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2368	IrInstSrc *target)
	2369	{
	2370	IrInstSrcIntToErr *instruction = ir_build_instruction<IrInstSrcIntToErr>(irb, scope, source_node);
	2371	instruction->target = target;
	2372
	2373	ir_ref_instruction(target, irb->current_basic_block);
	2374
	2375	return &instruction->base;
	2376	}
	2377
	2378	static IrInstSrc ir_build_err_to_int_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2379	IrInstSrc *target)
	2380	{
	2381	IrInstSrcErrToInt *instruction = ir_build_instruction<IrInstSrcErrToInt>(
	2382	irb, scope, source_node);
	2383	instruction->target = target;
	2384
	2385	ir_ref_instruction(target, irb->current_basic_block);
	2386
	2387	return &instruction->base;
	2388	}
	2389
	2390	static IrInstSrc ir_build_check_switch_prongs(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2391	IrInstSrc target_value, IrInstSrcCheckSwitchProngsRange ranges, size_t range_count,
	2392	AstNode* else_prong, bool have_underscore_prong)
	2393	{
	2394	IrInstSrcCheckSwitchProngs *instruction = heap::c_allocator.create<IrInstSrcCheckSwitchProngs>();
	2395	instruction->base.id = have_underscore_prong ?
	2396	IrInstSrcIdCheckSwitchProngsUnderYes : IrInstSrcIdCheckSwitchProngsUnderNo;
	2397	instruction->base.base.scope = scope;
	2398	instruction->base.base.source_node = source_node;
	2399	instruction->base.base.debug_id = exec_next_debug_id(irb->exec);
	2400	instruction->base.owner_bb = irb->current_basic_block;
	2401	ir_instruction_append(irb->current_basic_block, &instruction->base);
	2402
	2403	instruction->target_value = target_value;
	2404	instruction->ranges = ranges;
	2405	instruction->range_count = range_count;
	2406	instruction->else_prong = else_prong;
	2407
	2408	ir_ref_instruction(target_value, irb->current_basic_block);
	2409	for (size_t i = 0; i < range_count; i += 1) {
	2410	ir_ref_instruction(ranges[i].start, irb->current_basic_block);
	2411	ir_ref_instruction(ranges[i].end, irb->current_basic_block);
	2412	}
	2413
	2414	return &instruction->base;
	2415	}
	2416
	2417	static IrInstSrc ir_build_check_statement_is_void(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2418	IrInstSrc* statement_value)
	2419	{
	2420	IrInstSrcCheckStatementIsVoid *instruction = ir_build_instruction<IrInstSrcCheckStatementIsVoid>(
	2421	irb, scope, source_node);
	2422	instruction->statement_value = statement_value;
	2423
	2424	ir_ref_instruction(statement_value, irb->current_basic_block);
	2425
	2426	return &instruction->base;
	2427	}
	2428
	2429	static IrInstSrc ir_build_type_name(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2430	IrInstSrc *type_value)
	2431	{
	2432	IrInstSrcTypeName *instruction = ir_build_instruction<IrInstSrcTypeName>(irb, scope, source_node);
	2433	instruction->type_value = type_value;
	2434
	2435	ir_ref_instruction(type_value, irb->current_basic_block);
	2436
	2437	return &instruction->base;
	2438	}
	2439
	2440	static IrInstSrc ir_build_decl_ref(IrBuilderSrc irb, Scope scope, AstNode source_node, Tld *tld, LVal lval) {
	2441	IrInstSrcDeclRef *instruction = ir_build_instruction<IrInstSrcDeclRef>(irb, scope, source_node);
	2442	instruction->tld = tld;
	2443	instruction->lval = lval;
	2444
	2445	return &instruction->base;
	2446	}
	2447
	2448	static IrInstSrc ir_build_panic_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *msg) {
	2449	IrInstSrcPanic *instruction = ir_build_instruction<IrInstSrcPanic>(irb, scope, source_node);
	2450	instruction->base.is_noreturn = true;
	2451	instruction->msg = msg;
	2452
	2453	ir_ref_instruction(msg, irb->current_basic_block);
	2454
	2455	return &instruction->base;
	2456	}
	2457
	2458	static IrInstSrc ir_build_tag_name_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *target) {
	2459	IrInstSrcTagName *instruction = ir_build_instruction<IrInstSrcTagName>(irb, scope, source_node);
	2460	instruction->target = target;
	2461
	2462	ir_ref_instruction(target, irb->current_basic_block);
	2463
	2464	return &instruction->base;
	2465	}
	2466
	2467	static IrInstSrc ir_build_field_parent_ptr_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2468	IrInstSrc type_value, IrInstSrc field_name, IrInstSrc *field_ptr)
	2469	{
	2470	IrInstSrcFieldParentPtr *inst = ir_build_instruction<IrInstSrcFieldParentPtr>(
	2471	irb, scope, source_node);
	2472	inst->type_value = type_value;
	2473	inst->field_name = field_name;
	2474	inst->field_ptr = field_ptr;
	2475
	2476	ir_ref_instruction(type_value, irb->current_basic_block);
	2477	ir_ref_instruction(field_name, irb->current_basic_block);
	2478	ir_ref_instruction(field_ptr, irb->current_basic_block);
	2479
	2480	return &inst->base;
	2481	}
	2482
	2483	static IrInstSrc ir_build_byte_offset_of(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2484	IrInstSrc type_value, IrInstSrc field_name)
	2485	{
	2486	IrInstSrcByteOffsetOf *instruction = ir_build_instruction<IrInstSrcByteOffsetOf>(irb, scope, source_node);
	2487	instruction->type_value = type_value;
	2488	instruction->field_name = field_name;
	2489
	2490	ir_ref_instruction(type_value, irb->current_basic_block);
	2491	ir_ref_instruction(field_name, irb->current_basic_block);
	2492
	2493	return &instruction->base;
	2494	}
	2495
	2496	static IrInstSrc ir_build_bit_offset_of(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2497	IrInstSrc type_value, IrInstSrc field_name)
	2498	{
	2499	IrInstSrcBitOffsetOf *instruction = ir_build_instruction<IrInstSrcBitOffsetOf>(irb, scope, source_node);
	2500	instruction->type_value = type_value;
	2501	instruction->field_name = field_name;
	2502
	2503	ir_ref_instruction(type_value, irb->current_basic_block);
	2504	ir_ref_instruction(field_name, irb->current_basic_block);
	2505
	2506	return &instruction->base;
	2507	}
	2508
	2509	static IrInstSrc ir_build_type_info(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type_value) {
	2510	IrInstSrcTypeInfo *instruction = ir_build_instruction<IrInstSrcTypeInfo>(irb, scope, source_node);
	2511	instruction->type_value = type_value;
	2512
	2513	ir_ref_instruction(type_value, irb->current_basic_block);
	2514
	2515	return &instruction->base;
	2516	}
	2517
	2518	static IrInstSrc ir_build_type(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type_info) {
	2519	IrInstSrcType *instruction = ir_build_instruction<IrInstSrcType>(irb, scope, source_node);
	2520	instruction->type_info = type_info;
	2521
	2522	ir_ref_instruction(type_info, irb->current_basic_block);
	2523
	2524	return &instruction->base;
	2525	}
	2526
	2527	static IrInstSrc ir_build_set_eval_branch_quota(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2528	IrInstSrc *new_quota)
	2529	{
	2530	IrInstSrcSetEvalBranchQuota *instruction = ir_build_instruction<IrInstSrcSetEvalBranchQuota>(irb, scope, source_node);
	2531	instruction->new_quota = new_quota;
	2532
	2533	ir_ref_instruction(new_quota, irb->current_basic_block);
	2534
	2535	return &instruction->base;
	2536	}
	2537
	2538	static IrInstSrc ir_build_align_cast_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2539	IrInstSrc align_bytes, IrInstSrc target)
	2540	{
	2541	IrInstSrcAlignCast *instruction = ir_build_instruction<IrInstSrcAlignCast>(irb, scope, source_node);
	2542	instruction->align_bytes = align_bytes;
	2543	instruction->target = target;
	2544
	2545	ir_ref_instruction(align_bytes, irb->current_basic_block);
	2546	ir_ref_instruction(target, irb->current_basic_block);
	2547
	2548	return &instruction->base;
	2549	}
	2550
	2551	static IrInstSrc ir_build_resolve_result(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2552	ResultLoc result_loc, IrInstSrc ty)
	2553	{
	2554	IrInstSrcResolveResult *instruction = ir_build_instruction<IrInstSrcResolveResult>(irb, scope, source_node);
	2555	instruction->result_loc = result_loc;
	2556	instruction->ty = ty;
	2557
	2558	if (ty != nullptr) ir_ref_instruction(ty, irb->current_basic_block);
	2559
	2560	return &instruction->base;
	2561	}
	2562
	2563	static IrInstSrc ir_build_reset_result(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2564	ResultLoc *result_loc)
	2565	{
	2566	IrInstSrcResetResult *instruction = ir_build_instruction<IrInstSrcResetResult>(irb, scope, source_node);
	2567	instruction->result_loc = result_loc;
	2568	instruction->base.is_gen = true;
	2569
	2570	return &instruction->base;
	2571	}
	2572
	2573	static IrInstSrc ir_build_set_align_stack(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2574	IrInstSrc *align_bytes)
	2575	{
	2576	IrInstSrcSetAlignStack *instruction = ir_build_instruction<IrInstSrcSetAlignStack>(irb, scope, source_node);
	2577	instruction->align_bytes = align_bytes;
	2578
	2579	ir_ref_instruction(align_bytes, irb->current_basic_block);
	2580
	2581	return &instruction->base;
	2582	}
	2583
	2584	static IrInstSrc ir_build_arg_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2585	IrInstSrc fn_type, IrInstSrc arg_index, bool allow_var)
	2586	{
	2587	IrInstSrcArgType *instruction = heap::c_allocator.create<IrInstSrcArgType>();
	2588	instruction->base.id = allow_var ?
	2589	IrInstSrcIdArgTypeAllowVarTrue : IrInstSrcIdArgTypeAllowVarFalse;
	2590	instruction->base.base.scope = scope;
	2591	instruction->base.base.source_node = source_node;
	2592	instruction->base.base.debug_id = exec_next_debug_id(irb->exec);
	2593	instruction->base.owner_bb = irb->current_basic_block;
	2594	ir_instruction_append(irb->current_basic_block, &instruction->base);
	2595
	2596	instruction->fn_type = fn_type;
	2597	instruction->arg_index = arg_index;
	2598
	2599	ir_ref_instruction(fn_type, irb->current_basic_block);
	2600	ir_ref_instruction(arg_index, irb->current_basic_block);
	2601
	2602	return &instruction->base;
	2603	}
	2604
	2605	static IrInstSrc ir_build_error_return_trace_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2606	IrInstErrorReturnTraceOptional optional)
	2607	{
	2608	IrInstSrcErrorReturnTrace *inst = ir_build_instruction<IrInstSrcErrorReturnTrace>(irb, scope, source_node);
	2609	inst->optional = optional;
	2610
	2611	return &inst->base;
	2612	}
	2613
	2614	static IrInstSrc ir_build_error_union(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2615	IrInstSrc err_set, IrInstSrc payload)
	2616	{
	2617	IrInstSrcErrorUnion *instruction = ir_build_instruction<IrInstSrcErrorUnion>(irb, scope, source_node);
	2618	instruction->err_set = err_set;
	2619	instruction->payload = payload;
	2620
	2621	ir_ref_instruction(err_set, irb->current_basic_block);
	2622	ir_ref_instruction(payload, irb->current_basic_block);
	2623
	2624	return &instruction->base;
	2625	}
	2626
	2627	static IrInstSrc ir_build_atomic_rmw_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2628	IrInstSrc operand_type, IrInstSrc ptr, IrInstSrc op, IrInstSrc operand,
	2629	IrInstSrc *ordering)
	2630	{
	2631	IrInstSrcAtomicRmw *instruction = ir_build_instruction<IrInstSrcAtomicRmw>(irb, scope, source_node);
	2632	instruction->operand_type = operand_type;
	2633	instruction->ptr = ptr;
	2634	instruction->op = op;
	2635	instruction->operand = operand;
	2636	instruction->ordering = ordering;
	2637
	2638	ir_ref_instruction(operand_type, irb->current_basic_block);
	2639	ir_ref_instruction(ptr, irb->current_basic_block);
	2640	ir_ref_instruction(op, irb->current_basic_block);
	2641	ir_ref_instruction(operand, irb->current_basic_block);
	2642	ir_ref_instruction(ordering, irb->current_basic_block);
	2643
	2644	return &instruction->base;
	2645	}
	2646
	2647	static IrInstSrc ir_build_atomic_load_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2648	IrInstSrc operand_type, IrInstSrc ptr, IrInstSrc *ordering)
	2649	{
	2650	IrInstSrcAtomicLoad *instruction = ir_build_instruction<IrInstSrcAtomicLoad>(irb, scope, source_node);
	2651	instruction->operand_type = operand_type;
	2652	instruction->ptr = ptr;
	2653	instruction->ordering = ordering;
	2654
	2655	ir_ref_instruction(operand_type, irb->current_basic_block);
	2656	ir_ref_instruction(ptr, irb->current_basic_block);
	2657	ir_ref_instruction(ordering, irb->current_basic_block);
	2658
	2659	return &instruction->base;
	2660	}
	2661
	2662	static IrInstSrc ir_build_atomic_store_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2663	IrInstSrc operand_type, IrInstSrc ptr, IrInstSrc value, IrInstSrc ordering)
	2664	{
	2665	IrInstSrcAtomicStore *instruction = ir_build_instruction<IrInstSrcAtomicStore>(irb, scope, source_node);
	2666	instruction->operand_type = operand_type;
	2667	instruction->ptr = ptr;
	2668	instruction->value = value;
	2669	instruction->ordering = ordering;
	2670
	2671	ir_ref_instruction(operand_type, irb->current_basic_block);
	2672	ir_ref_instruction(ptr, irb->current_basic_block);
	2673	ir_ref_instruction(value, irb->current_basic_block);
	2674	ir_ref_instruction(ordering, irb->current_basic_block);
	2675
	2676	return &instruction->base;
	2677	}
	2678
	2679	static IrInstSrc ir_build_save_err_ret_addr_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	2680	IrInstSrcSaveErrRetAddr *inst = ir_build_instruction<IrInstSrcSaveErrRetAddr>(irb, scope, source_node);
	2681	return &inst->base;
	2682	}
	2683
	2684	static IrInstSrc ir_build_add_implicit_return_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2685	IrInstSrc value, ResultLocReturn result_loc_ret)
	2686	{
	2687	IrInstSrcAddImplicitReturnType *inst = ir_build_instruction<IrInstSrcAddImplicitReturnType>(irb, scope, source_node);
	2688	inst->value = value;
	2689	inst->result_loc_ret = result_loc_ret;
	2690
	2691	ir_ref_instruction(value, irb->current_basic_block);
	2692
	2693	return &inst->base;
	2694	}
	2695
	2696	static IrInstSrc ir_build_has_decl(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2697	IrInstSrc container, IrInstSrc name)
	2698	{
	2699	IrInstSrcHasDecl *instruction = ir_build_instruction<IrInstSrcHasDecl>(irb, scope, source_node);
	2700	instruction->container = container;
	2701	instruction->name = name;
	2702
	2703	ir_ref_instruction(container, irb->current_basic_block);
	2704	ir_ref_instruction(name, irb->current_basic_block);
	2705
	2706	return &instruction->base;
	2707	}
	2708
	2709	static IrInstSrc ir_build_undeclared_identifier(IrBuilderSrc irb, Scope scope, AstNode source_node, Buf *name) {
	2710	IrInstSrcUndeclaredIdent *instruction = ir_build_instruction<IrInstSrcUndeclaredIdent>(irb, scope, source_node);
	2711	instruction->name = name;
	2712
	2713	return &instruction->base;
	2714	}
	2715
	2716	static IrInstSrc ir_build_check_runtime_scope(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc scope_is_comptime, IrInstSrc is_comptime) {
	2717	IrInstSrcCheckRuntimeScope *instruction = ir_build_instruction<IrInstSrcCheckRuntimeScope>(irb, scope, source_node);
	2718	instruction->scope_is_comptime = scope_is_comptime;
	2719	instruction->is_comptime = is_comptime;
	2720
	2721	ir_ref_instruction(scope_is_comptime, irb->current_basic_block);
	2722	ir_ref_instruction(is_comptime, irb->current_basic_block);
	2723
	2724	return &instruction->base;
	2725	}
	2726
	2727	static IrInstSrc ir_build_union_init_named_field(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2728	IrInstSrc union_type, IrInstSrc field_name, IrInstSrc field_result_loc, IrInstSrc result_loc)
	2729	{
	2730	IrInstSrcUnionInitNamedField *instruction = ir_build_instruction<IrInstSrcUnionInitNamedField>(irb, scope, source_node);
	2731	instruction->union_type = union_type;
	2732	instruction->field_name = field_name;
	2733	instruction->field_result_loc = field_result_loc;
	2734	instruction->result_loc = result_loc;
	2735
	2736	ir_ref_instruction(union_type, irb->current_basic_block);
	2737	ir_ref_instruction(field_name, irb->current_basic_block);
	2738	ir_ref_instruction(field_result_loc, irb->current_basic_block);
	2739	if (result_loc != nullptr) ir_ref_instruction(result_loc, irb->current_basic_block);
	2740
	2741	return &instruction->base;
	2742	}
	2743
	2744	static IrInstSrc ir_build_alloca_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2745	IrInstSrc align, const char name_hint, IrInstSrc *is_comptime)
	2746	{
	2747	IrInstSrcAlloca *instruction = ir_build_instruction<IrInstSrcAlloca>(irb, scope, source_node);
	2748	instruction->base.is_gen = true;
	2749	instruction->align = align;
	2750	instruction->name_hint = name_hint;
	2751	instruction->is_comptime = is_comptime;
	2752
	2753	if (align != nullptr) ir_ref_instruction(align, irb->current_basic_block);
	2754	if (is_comptime != nullptr) ir_ref_instruction(is_comptime, irb->current_basic_block);
	2755
	2756	return &instruction->base;
	2757	}
	2758
	2759	static IrInstSrc ir_build_end_expr(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2760	IrInstSrc value, ResultLoc result_loc)
	2761	{
	2762	IrInstSrcEndExpr *instruction = ir_build_instruction<IrInstSrcEndExpr>(irb, scope, source_node);
	2763	instruction->base.is_gen = true;
	2764	instruction->value = value;
	2765	instruction->result_loc = result_loc;
	2766
	2767	ir_ref_instruction(value, irb->current_basic_block);
	2768
	2769	return &instruction->base;
	2770	}
	2771
	2772	static IrInstSrcSuspendBegin ir_build_suspend_begin_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	2773	return ir_build_instruction<IrInstSrcSuspendBegin>(irb, scope, source_node);
	2774	}
	2775
	2776	static IrInstSrc ir_build_suspend_finish_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2777	IrInstSrcSuspendBegin *begin)
	2778	{
	2779	IrInstSrcSuspendFinish *inst = ir_build_instruction<IrInstSrcSuspendFinish>(irb, scope, source_node);
	2780	inst->begin = begin;
	2781
	2782	ir_ref_instruction(&begin->base, irb->current_basic_block);
	2783
	2784	return &inst->base;
	2785	}
	2786
	2787	static IrInstSrc ir_build_await_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2788	IrInstSrc frame, ResultLoc result_loc, bool is_nosuspend)
	2789	{
	2790	IrInstSrcAwait *instruction = ir_build_instruction<IrInstSrcAwait>(irb, scope, source_node);
	2791	instruction->frame = frame;
	2792	instruction->result_loc = result_loc;
	2793	instruction->is_nosuspend = is_nosuspend;
	2794
	2795	ir_ref_instruction(frame, irb->current_basic_block);
	2796
	2797	return &instruction->base;
	2798	}
	2799
	2800	static IrInstSrc ir_build_resume_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *frame) {
	2801	IrInstSrcResume *instruction = ir_build_instruction<IrInstSrcResume>(irb, scope, source_node);
	2802	instruction->frame = frame;
	2803
	2804	ir_ref_instruction(frame, irb->current_basic_block);
	2805
	2806	return &instruction->base;
	2807	}
	2808
	2809	static IrInstSrcSpillBegin ir_build_spill_begin_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2810	IrInstSrc *operand, SpillId spill_id)
	2811	{
	2812	IrInstSrcSpillBegin *instruction = ir_build_instruction<IrInstSrcSpillBegin>(irb, scope, source_node);
	2813	instruction->operand = operand;
	2814	instruction->spill_id = spill_id;
	2815
	2816	ir_ref_instruction(operand, irb->current_basic_block);
	2817
	2818	return instruction;
	2819	}
	2820
	2821	static IrInstSrc ir_build_spill_end_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
	2822	IrInstSrcSpillBegin *begin)
	2823	{
	2824	IrInstSrcSpillEnd *instruction = ir_build_instruction<IrInstSrcSpillEnd>(irb, scope, source_node);
	2825	instruction->begin = begin;
	2826
	2827	ir_ref_instruction(&begin->base, irb->current_basic_block);
	2828
	2829	return &instruction->base;
	2830	}
	2831
	2832	static IrInstSrc ir_build_wasm_memory_size_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *index) {
	2833	IrInstSrcWasmMemorySize *instruction = ir_build_instruction<IrInstSrcWasmMemorySize>(irb, scope, source_node);
	2834	instruction->index = index;
	2835
	2836	ir_ref_instruction(index, irb->current_basic_block);
	2837
	2838	return &instruction->base;
	2839	}
	2840
	2841	static IrInstSrc ir_build_wasm_memory_grow_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc index, IrInstSrc delta) {
	2842	IrInstSrcWasmMemoryGrow *instruction = ir_build_instruction<IrInstSrcWasmMemoryGrow>(irb, scope, source_node);
	2843	instruction->index = index;
	2844	instruction->delta = delta;
	2845
	2846	ir_ref_instruction(index, irb->current_basic_block);
	2847	ir_ref_instruction(delta, irb->current_basic_block);
	2848
	2849	return &instruction->base;
	2850	}
	2851
	2852	static IrInstSrc ir_build_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
	2853	IrInstSrcSrc *instruction = ir_build_instruction<IrInstSrcSrc>(irb, scope, source_node);
	2854
	2855	return &instruction->base;
	2856	}
	2857
	2858	static void ir_count_defers(IrBuilderSrc irb, Scope inner_scope, Scope outer_scope, size_t results) {
	2859	results[ReturnKindUnconditional] = 0;
	2860	results[ReturnKindError] = 0;
	2861
	2862	Scope *scope = inner_scope;
	2863
	2864	while (scope != outer_scope) {
	2865	assert(scope);
	2866	switch (scope->id) {
	2867	case ScopeIdDefer: {
	2868	AstNode *defer_node = scope->source_node;
	2869	assert(defer_node->type == NodeTypeDefer);
	2870	ReturnKind defer_kind = defer_node->data.defer.kind;
	2871	results[defer_kind] += 1;
	2872	scope = scope->parent;
	2873	continue;
	2874	}
	2875	case ScopeIdDecls:
	2876	case ScopeIdFnDef:
	2877	return;
	2878	case ScopeIdBlock:
	2879	case ScopeIdVarDecl:
	2880	case ScopeIdLoop:
	2881	case ScopeIdSuspend:
	2882	case ScopeIdCompTime:
	2883	case ScopeIdNoSuspend:
	2884	case ScopeIdRuntime:
	2885	case ScopeIdTypeOf:
	2886	case ScopeIdExpr:
	2887	scope = scope->parent;
	2888	continue;
	2889	case ScopeIdDeferExpr:
	2890	case ScopeIdCImport:
	2891	zig_unreachable();
	2892	}
	2893	}
	2894	}
	2895
	2896	static IrInstSrc ir_mark_gen(IrInstSrc instruction) {
	2897	instruction->is_gen = true;
	2898	return instruction;
	2899	}
	2900
	2901	static bool ir_gen_defers_for_block(IrBuilderSrc irb, Scope inner_scope, Scope outer_scope, bool is_noreturn, IrInstSrc *err_value) {
	2902	Scope *scope = inner_scope;
	2903	if (is_noreturn != nullptr) *is_noreturn = false;
	2904	while (scope != outer_scope) {
	2905	if (!scope)
	2906	return true;
	2907
	2908	switch (scope->id) {
	2909	case ScopeIdDefer: {
	2910	AstNode *defer_node = scope->source_node;
	2911	assert(defer_node->type == NodeTypeDefer);
	2912	ReturnKind defer_kind = defer_node->data.defer.kind;
	2913	AstNode *defer_expr_node = defer_node->data.defer.expr;
	2914	AstNode *defer_var_node = defer_node->data.defer.err_payload;
	2915
	2916	if (defer_kind == ReturnKindError && err_value == nullptr) {
	2917	// This is an `errdefer` but we're generating code for a
	2918	// `return` that doesn't return an error, skip it
	2919	scope = scope->parent;
	2920	continue;
	2921	}
	2922
	2923	Scope *defer_expr_scope = defer_node->data.defer.expr_scope;
	2924	if (defer_var_node != nullptr) {
	2925	assert(defer_kind == ReturnKindError);
	2926	assert(defer_var_node->type == NodeTypeIdentifier);
	2927	Buf *var_name = node_identifier_buf(defer_var_node);
	2928
	2929	if (defer_expr_node->type == NodeTypeUnreachable) {
	2930	add_node_error(irb->codegen, defer_var_node,
	2931	buf_sprintf("unused variable: '%s'", buf_ptr(var_name)));
	2932	return false;
	2933	}
	2934
	2935	IrInstSrc *is_comptime;
	2936	if (ir_should_inline(irb->exec, defer_expr_scope)) {
	2937	is_comptime = ir_build_const_bool(irb, defer_expr_scope,
	2938	defer_expr_node, true);
	2939	} else {
	2940	is_comptime = ir_build_test_comptime(irb, defer_expr_scope,
	2941	defer_expr_node, err_value);
	2942	}
	2943
	2944	ZigVar *err_var = ir_create_var(irb, defer_var_node, defer_expr_scope,
	2945	var_name, true, true, false, is_comptime);
	2946	build_decl_var_and_init(irb, defer_expr_scope, defer_var_node, err_var, err_value,
	2947	buf_ptr(var_name), is_comptime);
	2948
	2949	defer_expr_scope = err_var->child_scope;
	2950	}
	2951
	2952	IrInstSrc *defer_expr_value = ir_gen_node(irb, defer_expr_node, defer_expr_scope);
	2953	if (defer_expr_value == irb->codegen->invalid_inst_src)
	2954	return irb->codegen->invalid_inst_src;
	2955
	2956	if (defer_expr_value->is_noreturn) {
	2957	if (is_noreturn != nullptr) *is_noreturn = true;
	2958	} else {
	2959	ir_mark_gen(ir_build_check_statement_is_void(irb, defer_expr_scope, defer_expr_node,
	2960	defer_expr_value));
	2961	}
	2962	scope = scope->parent;
	2963	continue;
	2964	}
	2965	case ScopeIdDecls:
	2966	case ScopeIdFnDef:
	2967	return true;
	2968	case ScopeIdBlock:
	2969	case ScopeIdVarDecl:
	2970	case ScopeIdLoop:
	2971	case ScopeIdSuspend:
	2972	case ScopeIdCompTime:
	2973	case ScopeIdNoSuspend:
	2974	case ScopeIdRuntime:
	2975	case ScopeIdTypeOf:
	2976	case ScopeIdExpr:
	2977	scope = scope->parent;
	2978	continue;
	2979	case ScopeIdDeferExpr:
	2980	case ScopeIdCImport:
	2981	zig_unreachable();
	2982	}
	2983	}
	2984	return true;
	2985	}
	2986
	2987	static void ir_set_cursor_at_end(IrBuilderSrc irb, IrBasicBlockSrc basic_block) {
	2988	assert(basic_block);
	2989	irb->current_basic_block = basic_block;
	2990	}
	2991
	2992	static void ir_set_cursor_at_end_and_append_block(IrBuilderSrc irb, IrBasicBlockSrc basic_block) {
	2993	basic_block->index = irb->exec->basic_block_list.length;
	2994	irb->exec->basic_block_list.append(basic_block);
	2995	ir_set_cursor_at_end(irb, basic_block);
	2996	}
	2997
	2998	static ScopeSuspend get_scope_suspend(Scope scope) {
	2999	while (scope) {
	3000	if (scope->id == ScopeIdSuspend)
	3001	return (ScopeSuspend *)scope;
	3002	if (scope->id == ScopeIdFnDef)
	3003	return nullptr;
	3004
	3005	scope = scope->parent;
	3006	}
	3007	return nullptr;
	3008	}
	3009
	3010	static ScopeDeferExpr get_scope_defer_expr(Scope scope) {
	3011	while (scope) {
	3012	if (scope->id == ScopeIdDeferExpr)
	3013	return (ScopeDeferExpr *)scope;
	3014	if (scope->id == ScopeIdFnDef)
	3015	return nullptr;
	3016
	3017	scope = scope->parent;
	3018	}
	3019	return nullptr;
	3020	}
	3021
	3022	static IrInstSrc ir_gen_return(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval, ResultLoc *result_loc) {
	3023	assert(node->type == NodeTypeReturnExpr);
	3024
	3025	ScopeDeferExpr *scope_defer_expr = get_scope_defer_expr(scope);
	3026	if (scope_defer_expr) {
	3027	if (!scope_defer_expr->reported_err) {
	3028	add_node_error(irb->codegen, node, buf_sprintf("cannot return from defer expression"));
	3029	scope_defer_expr->reported_err = true;
	3030	}
	3031	return irb->codegen->invalid_inst_src;
	3032	}
	3033
	3034	Scope *outer_scope = irb->exec->begin_scope;
	3035
	3036	AstNode *expr_node = node->data.return_expr.expr;
	3037	switch (node->data.return_expr.kind) {
	3038	case ReturnKindUnconditional:
	3039	{
	3040	ResultLocReturn *result_loc_ret = heap::c_allocator.create<ResultLocReturn>();
	3041	result_loc_ret->base.id = ResultLocIdReturn;
	3042	ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
	3043
	3044	IrInstSrc *return_value;
	3045	if (expr_node) {
	3046	// Temporarily set this so that if we return a type it gets the name of the function
	3047	ZigFn *prev_name_fn = irb->exec->name_fn;
	3048	irb->exec->name_fn = exec_fn_entry(irb->exec);
	3049	return_value = ir_gen_node_extra(irb, expr_node, scope, LValNone, &result_loc_ret->base);
	3050	irb->exec->name_fn = prev_name_fn;
	3051	if (return_value == irb->codegen->invalid_inst_src)
	3052	return irb->codegen->invalid_inst_src;
	3053	} else {
	3054	return_value = ir_build_const_void(irb, scope, node);
	3055	ir_build_end_expr(irb, scope, node, return_value, &result_loc_ret->base);
	3056	}
	3057
	3058	ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, return_value, result_loc_ret));
	3059
	3060	size_t defer_counts[2];
	3061	ir_count_defers(irb, scope, outer_scope, defer_counts);
	3062	bool have_err_defers = defer_counts[ReturnKindError] > 0;
	3063	if (!have_err_defers && !irb->codegen->have_err_ret_tracing) {
	3064	// only generate unconditional defers
	3065	if (!ir_gen_defers_for_block(irb, scope, outer_scope, nullptr, nullptr))
	3066	return irb->codegen->invalid_inst_src;
	3067	IrInstSrc *result = ir_build_return_src(irb, scope, node, nullptr);
	3068	result_loc_ret->base.source_instruction = result;
	3069	return result;
	3070	}
	3071	bool should_inline = ir_should_inline(irb->exec, scope);
	3072
	3073	IrBasicBlockSrc *err_block = ir_create_basic_block(irb, scope, "ErrRetErr");
	3074	IrBasicBlockSrc *ok_block = ir_create_basic_block(irb, scope, "ErrRetOk");
	3075
	3076	IrInstSrc *is_err = ir_build_test_err_src(irb, scope, node, return_value, false, true);
	3077
	3078	IrInstSrc *is_comptime;
	3079	if (should_inline) {
	3080	is_comptime = ir_build_const_bool(irb, scope, node, should_inline);
	3081	} else {
	3082	is_comptime = ir_build_test_comptime(irb, scope, node, is_err);
	3083	}
	3084
	3085	ir_mark_gen(ir_build_cond_br(irb, scope, node, is_err, err_block, ok_block, is_comptime));
	3086	IrBasicBlockSrc *ret_stmt_block = ir_create_basic_block(irb, scope, "RetStmt");
	3087
	3088	ir_set_cursor_at_end_and_append_block(irb, err_block);
	3089	if (!ir_gen_defers_for_block(irb, scope, outer_scope, nullptr, return_value))
	3090	return irb->codegen->invalid_inst_src;
	3091	if (irb->codegen->have_err_ret_tracing && !should_inline) {
	3092	ir_build_save_err_ret_addr_src(irb, scope, node);
	3093	}
	3094	ir_build_br(irb, scope, node, ret_stmt_block, is_comptime);
	3095
	3096	ir_set_cursor_at_end_and_append_block(irb, ok_block);
	3097	if (!ir_gen_defers_for_block(irb, scope, outer_scope, nullptr, nullptr))
	3098	return irb->codegen->invalid_inst_src;
	3099	ir_build_br(irb, scope, node, ret_stmt_block, is_comptime);
	3100
	3101	ir_set_cursor_at_end_and_append_block(irb, ret_stmt_block);
	3102	IrInstSrc *result = ir_build_return_src(irb, scope, node, nullptr);
	3103	result_loc_ret->base.source_instruction = result;
	3104	return result;
	3105	}
	3106	case ReturnKindError:
	3107	{
	3108	assert(expr_node);
	3109	IrInstSrc *err_union_ptr = ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr);
	3110	if (err_union_ptr == irb->codegen->invalid_inst_src)
	3111	return irb->codegen->invalid_inst_src;
	3112	IrInstSrc *is_err_val = ir_build_test_err_src(irb, scope, node, err_union_ptr, true, false);
	3113
	3114	IrBasicBlockSrc *return_block = ir_create_basic_block(irb, scope, "ErrRetReturn");
	3115	IrBasicBlockSrc *continue_block = ir_create_basic_block(irb, scope, "ErrRetContinue");
	3116	IrInstSrc *is_comptime;
	3117	bool should_inline = ir_should_inline(irb->exec, scope);
	3118	if (should_inline) {
	3119	is_comptime = ir_build_const_bool(irb, scope, node, true);
	3120	} else {
	3121	is_comptime = ir_build_test_comptime(irb, scope, node, is_err_val);
	3122	}
	3123	ir_mark_gen(ir_build_cond_br(irb, scope, node, is_err_val, return_block, continue_block, is_comptime));
	3124
	3125	ir_set_cursor_at_end_and_append_block(irb, return_block);
	3126	IrInstSrc *err_val_ptr = ir_build_unwrap_err_code_src(irb, scope, node, err_union_ptr);
	3127	IrInstSrc *err_val = ir_build_load_ptr(irb, scope, node, err_val_ptr);
	3128	ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, err_val, nullptr));
	3129	IrInstSrcSpillBegin *spill_begin = ir_build_spill_begin_src(irb, scope, node, err_val,
	3130	SpillIdRetErrCode);
	3131	ResultLocReturn *result_loc_ret = heap::c_allocator.create<ResultLocReturn>();
	3132	result_loc_ret->base.id = ResultLocIdReturn;
	3133	ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
	3134	ir_build_end_expr(irb, scope, node, err_val, &result_loc_ret->base);
	3135
	3136	bool is_noreturn = false;
	3137	if (!ir_gen_defers_for_block(irb, scope, outer_scope, &is_noreturn, err_val)) {
	3138	return irb->codegen->invalid_inst_src;
	3139	}
	3140	if (!is_noreturn) {
	3141	if (irb->codegen->have_err_ret_tracing && !should_inline) {
	3142	ir_build_save_err_ret_addr_src(irb, scope, node);
	3143	}
	3144	err_val = ir_build_spill_end_src(irb, scope, node, spill_begin);
	3145	IrInstSrc *ret_inst = ir_build_return_src(irb, scope, node, err_val);
	3146	result_loc_ret->base.source_instruction = ret_inst;
	3147	}
	3148
	3149	ir_set_cursor_at_end_and_append_block(irb, continue_block);
	3150	IrInstSrc *unwrapped_ptr = ir_build_unwrap_err_payload_src(irb, scope, node, err_union_ptr, false, false);
	3151	if (lval == LValPtr)
	3152	return unwrapped_ptr;
	3153	else
	3154	return ir_expr_wrap(irb, scope, ir_build_load_ptr(irb, scope, node, unwrapped_ptr), result_loc);
	3155	}
	3156	}
	3157	zig_unreachable();
	3158	}
	3159
	3160	ZigVar create_local_var(CodeGen codegen, AstNode node, Scope parent_scope,
	3161	Buf name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstSrc is_comptime,
	3162	bool skip_name_check)
	3163	{
	3164	ZigVar *variable_entry = heap::c_allocator.create<ZigVar>();
	3165	variable_entry->parent_scope = parent_scope;
	3166	variable_entry->shadowable = is_shadowable;
	3167	variable_entry->is_comptime = is_comptime;
	3168	variable_entry->src_arg_index = SIZE_MAX;
	3169	variable_entry->const_value = codegen->pass1_arena->create<ZigValue>();
	3170
	3171	if (is_comptime != nullptr) {
	3172	is_comptime->base.ref_count += 1;
	3173	}
	3174
	3175	if (name) {
	3176	variable_entry->name = strdup(buf_ptr(name));
	3177
	3178	if (!skip_name_check) {
	3179	ZigVar *existing_var = find_variable(codegen, parent_scope, name, nullptr);
	3180	if (existing_var && !existing_var->shadowable) {
	3181	if (existing_var->var_type == nullptr \|\| !type_is_invalid(existing_var->var_type)) {
	3182	ErrorMsg *msg = add_node_error(codegen, node,
	3183	buf_sprintf("redeclaration of variable '%s'", buf_ptr(name)));
	3184	add_error_note(codegen, msg, existing_var->decl_node, buf_sprintf("previous declaration is here"));
	3185	}
	3186	variable_entry->var_type = codegen->builtin_types.entry_invalid;
	3187	} else {
	3188	ZigType *type;
	3189	if (get_primitive_type(codegen, name, &type) != ErrorPrimitiveTypeNotFound) {
	3190	add_node_error(codegen, node,
	3191	buf_sprintf("variable shadows primitive type '%s'", buf_ptr(name)));
	3192	variable_entry->var_type = codegen->builtin_types.entry_invalid;
	3193	} else {
	3194	Tld *tld = find_decl(codegen, parent_scope, name);
	3195	if (tld != nullptr) {
	3196	bool want_err_msg = true;
	3197	if (tld->id == TldIdVar) {
	3198	ZigVar var = reinterpret_cast<TldVar >(tld)->var;
	3199	if (var != nullptr && var->var_type != nullptr && type_is_invalid(var->var_type)) {
	3200	want_err_msg = false;
	3201	}
	3202	}
	3203	if (want_err_msg) {
	3204	ErrorMsg *msg = add_node_error(codegen, node,
	3205	buf_sprintf("redefinition of '%s'", buf_ptr(name)));
	3206	add_error_note(codegen, msg, tld->source_node, buf_sprintf("previous definition is here"));
	3207	}
	3208	variable_entry->var_type = codegen->builtin_types.entry_invalid;
	3209	}
	3210	}
	3211	}
	3212	}
	3213	} else {
	3214	assert(is_shadowable);
	3215	// TODO make this name not actually be in scope. user should be able to make a variable called "_anon"
	3216	// might already be solved, let's just make sure it has test coverage
	3217	// maybe we put a prefix on this so the debug info doesn't clobber user debug info for same named variables
	3218	variable_entry->name = "_anon";
	3219	}
	3220
	3221	variable_entry->src_is_const = src_is_const;
	3222	variable_entry->gen_is_const = gen_is_const;
	3223	variable_entry->decl_node = node;
	3224	variable_entry->child_scope = create_var_scope(codegen, node, parent_scope, variable_entry);
	3225
	3226	return variable_entry;
	3227	}
	3228
	3229
	3230	// Set name to nullptr to make the variable anonymous (not visible to programmer).
	3231	// After you call this function var->child_scope has the variable in scope
	3232	static ZigVar ir_create_var(IrBuilderSrc irb, AstNode node, Scope scope, Buf *name,
	3233	bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstSrc *is_comptime)
	3234	{
	3235	bool is_underscored = name ? buf_eql_str(name, "_") : false;
	3236	ZigVar *var = create_local_var(irb->codegen, node, scope,
	3237	(is_underscored ? nullptr : name), src_is_const, gen_is_const,
	3238	(is_underscored ? true : is_shadowable), is_comptime, false);
	3239	assert(var->child_scope);
	3240	return var;
	3241	}
	3242
	3243	static ResultLocPeer create_peer_result(ResultLocPeerParent peer_parent) {
	3244	ResultLocPeer *result = heap::c_allocator.create<ResultLocPeer>();
	3245	result->base.id = ResultLocIdPeer;
	3246	result->base.source_instruction = peer_parent->base.source_instruction;
	3247	result->parent = peer_parent;
	3248	result->base.allow_write_through_const = peer_parent->parent->allow_write_through_const;
	3249	return result;
	3250	}
	3251
	3252	static bool is_duplicate_label(CodeGen g, Scope scope, AstNode node, Buf name) {
	3253	if (name == nullptr) return false;
	3254
	3255	for (;;) {
	3256	if (scope == nullptr \|\| scope->id == ScopeIdFnDef) {
	3257	break;
	3258	} else if (scope->id == ScopeIdBlock \|\| scope->id == ScopeIdLoop) {
	3259	Buf this_block_name = scope->id == ScopeIdBlock ? ((ScopeBlock )scope)->name : ((ScopeLoop *)scope)->name;
	3260	if (this_block_name != nullptr && buf_eql_buf(name, this_block_name)) {
	3261	ErrorMsg *msg = add_node_error(g, node, buf_sprintf("redeclaration of label '%s'", buf_ptr(name)));
	3262	add_error_note(g, msg, scope->source_node, buf_sprintf("previous declaration is here"));
	3263	return true;
	3264	}
	3265	}
	3266	scope = scope->parent;
	3267	}
	3268	return false;
	3269	}
	3270
	3271	static IrInstSrc ir_gen_block(IrBuilderSrc irb, Scope parent_scope, AstNode block_node, LVal lval,
	3272	ResultLoc *result_loc)
	3273	{
	3274	assert(block_node->type == NodeTypeBlock);
	3275
	3276	ZigList<IrInstSrc *> incoming_values = {0};
	3277	ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
	3278
	3279	if (is_duplicate_label(irb->codegen, parent_scope, block_node, block_node->data.block.name))
	3280	return irb->codegen->invalid_inst_src;
	3281
	3282	ScopeBlock *scope_block = create_block_scope(irb->codegen, block_node, parent_scope);
	3283
	3284	Scope *outer_block_scope = &scope_block->base;
	3285	Scope *child_scope = outer_block_scope;
	3286
	3287	ZigFn *fn_entry = scope_fn_entry(parent_scope);
	3288	if (fn_entry && fn_entry->child_scope == parent_scope) {
	3289	fn_entry->def_scope = scope_block;
	3290	}
	3291
	3292	if (block_node->data.block.statements.length == 0) {
	3293	if (scope_block->name != nullptr) {
	3294	add_node_error(irb->codegen, block_node, buf_sprintf("unused block label"));
	3295	}
	3296	// {}
	3297	return ir_lval_wrap(irb, parent_scope, ir_build_const_void(irb, child_scope, block_node), lval, result_loc);
	3298	}
	3299
	3300	if (block_node->data.block.name != nullptr) {
	3301	scope_block->lval = lval;
	3302	scope_block->incoming_blocks = &incoming_blocks;
	3303	scope_block->incoming_values = &incoming_values;
	3304	scope_block->end_block = ir_create_basic_block(irb, parent_scope, "BlockEnd");
	3305	scope_block->is_comptime = ir_build_const_bool(irb, parent_scope, block_node,
	3306	ir_should_inline(irb->exec, parent_scope));
	3307
	3308	scope_block->peer_parent = heap::c_allocator.create<ResultLocPeerParent>();
	3309	scope_block->peer_parent->base.id = ResultLocIdPeerParent;
	3310	scope_block->peer_parent->base.source_instruction = scope_block->is_comptime;
	3311	scope_block->peer_parent->base.allow_write_through_const = result_loc->allow_write_through_const;
	3312	scope_block->peer_parent->end_bb = scope_block->end_block;
	3313	scope_block->peer_parent->is_comptime = scope_block->is_comptime;
	3314	scope_block->peer_parent->parent = result_loc;
	3315	ir_build_reset_result(irb, parent_scope, block_node, &scope_block->peer_parent->base);
	3316	}
	3317
	3318	bool is_continuation_unreachable = false;
	3319	bool found_invalid_inst = false;
	3320	IrInstSrc *noreturn_return_value = nullptr;
	3321	for (size_t i = 0; i < block_node->data.block.statements.length; i += 1) {
	3322	AstNode *statement_node = block_node->data.block.statements.at(i);
	3323
	3324	IrInstSrc *statement_value = ir_gen_node(irb, statement_node, child_scope);
	3325	if (statement_value == irb->codegen->invalid_inst_src) {
	3326	// keep generating all the elements of the block in case of error,
	3327	// we want to collect other compile errors
	3328	found_invalid_inst = true;
	3329	continue;
	3330	}
	3331
	3332	is_continuation_unreachable = instr_is_unreachable(statement_value);
	3333	if (is_continuation_unreachable) {
	3334	// keep the last noreturn statement value around in case we need to return it
	3335	noreturn_return_value = statement_value;
	3336	}
	3337	// This logic must be kept in sync with
	3338	// [STMT_EXPR_TEST_THING] <--- (search this token)
	3339	if (statement_node->type == NodeTypeDefer) {
	3340	// defer starts a new scope
	3341	child_scope = statement_node->data.defer.child_scope;
	3342	assert(child_scope);
	3343	} else if (statement_value->id == IrInstSrcIdDeclVar) {
	3344	// variable declarations start a new scope
	3345	IrInstSrcDeclVar decl_var_instruction = (IrInstSrcDeclVar )statement_value;
	3346	child_scope = decl_var_instruction->var->child_scope;
	3347	} else if (!is_continuation_unreachable) {
	3348	// this statement's value must be void
	3349	ir_mark_gen(ir_build_check_statement_is_void(irb, child_scope, statement_node, statement_value));
	3350	}
	3351	}
	3352
	3353	if (scope_block->name != nullptr && scope_block->name_used == false) {
	3354	add_node_error(irb->codegen, block_node, buf_sprintf("unused block label"));
	3355	}
	3356
	3357	if (found_invalid_inst)
	3358	return irb->codegen->invalid_inst_src;
	3359
	3360	if (is_continuation_unreachable) {
	3361	assert(noreturn_return_value != nullptr);
	3362	if (block_node->data.block.name == nullptr \|\| incoming_blocks.length == 0) {
	3363	return noreturn_return_value;
	3364	}
	3365
	3366	if (scope_block->peer_parent != nullptr && scope_block->peer_parent->peers.length != 0) {
	3367	scope_block->peer_parent->peers.last()->next_bb = scope_block->end_block;
	3368	}
	3369	ir_set_cursor_at_end_and_append_block(irb, scope_block->end_block);
	3370	IrInstSrc *phi = ir_build_phi(irb, parent_scope, block_node, incoming_blocks.length,
	3371	incoming_blocks.items, incoming_values.items, scope_block->peer_parent);
	3372	return ir_expr_wrap(irb, parent_scope, phi, result_loc);
	3373	} else {
	3374	incoming_blocks.append(irb->current_basic_block);
	3375	IrInstSrc *else_expr_result = ir_mark_gen(ir_build_const_void(irb, parent_scope, block_node));
	3376
	3377	if (scope_block->peer_parent != nullptr) {
	3378	ResultLocPeer *peer_result = create_peer_result(scope_block->peer_parent);
	3379	scope_block->peer_parent->peers.append(peer_result);
	3380	ir_build_end_expr(irb, parent_scope, block_node, else_expr_result, &peer_result->base);
	3381
	3382	if (scope_block->peer_parent->peers.length != 0) {
	3383	scope_block->peer_parent->peers.last()->next_bb = scope_block->end_block;
	3384	}
	3385	}
	3386
	3387	incoming_values.append(else_expr_result);
	3388	}
	3389
	3390	bool is_return_from_fn = block_node == irb->main_block_node;
	3391	if (!is_return_from_fn) {
	3392	if (!ir_gen_defers_for_block(irb, child_scope, outer_block_scope, nullptr, nullptr))
	3393	return irb->codegen->invalid_inst_src;
	3394	}
	3395
	3396	IrInstSrc *result;
	3397	if (block_node->data.block.name != nullptr) {
	3398	ir_mark_gen(ir_build_br(irb, parent_scope, block_node, scope_block->end_block, scope_block->is_comptime));
	3399	ir_set_cursor_at_end_and_append_block(irb, scope_block->end_block);
	3400	IrInstSrc *phi = ir_build_phi(irb, parent_scope, block_node, incoming_blocks.length,
	3401	incoming_blocks.items, incoming_values.items, scope_block->peer_parent);
	3402	result = ir_expr_wrap(irb, parent_scope, phi, result_loc);
	3403	} else {
	3404	IrInstSrc *void_inst = ir_mark_gen(ir_build_const_void(irb, child_scope, block_node));
	3405	result = ir_lval_wrap(irb, parent_scope, void_inst, lval, result_loc);
	3406	}
	3407	if (!is_return_from_fn)
	3408	return result;
	3409
	3410	// no need for save_err_ret_addr because this cannot return error
	3411	// only generate unconditional defers
	3412
	3413	ir_mark_gen(ir_build_add_implicit_return_type(irb, child_scope, block_node, result, nullptr));
	3414	ResultLocReturn *result_loc_ret = heap::c_allocator.create<ResultLocReturn>();
	3415	result_loc_ret->base.id = ResultLocIdReturn;
	3416	ir_build_reset_result(irb, parent_scope, block_node, &result_loc_ret->base);
	3417	ir_mark_gen(ir_build_end_expr(irb, parent_scope, block_node, result, &result_loc_ret->base));
	3418	if (!ir_gen_defers_for_block(irb, child_scope, outer_block_scope, nullptr, nullptr))
	3419	return irb->codegen->invalid_inst_src;
	3420	return ir_mark_gen(ir_build_return_src(irb, child_scope, result->base.source_node, result));
	3421	}
	3422
	3423	static IrInstSrc ir_gen_bin_op_id(IrBuilderSrc irb, Scope scope, AstNode node, IrBinOp op_id) {
	3424	Scope *inner_scope = scope;
	3425	if (op_id == IrBinOpArrayCat \|\| op_id == IrBinOpArrayMult) {
	3426	inner_scope = create_comptime_scope(irb->codegen, node, scope);
	3427	}
	3428
	3429	IrInstSrc *op1 = ir_gen_node(irb, node->data.bin_op_expr.op1, inner_scope);
	3430	IrInstSrc *op2 = ir_gen_node(irb, node->data.bin_op_expr.op2, inner_scope);
	3431
	3432	if (op1 == irb->codegen->invalid_inst_src \|\| op2 == irb->codegen->invalid_inst_src)
	3433	return irb->codegen->invalid_inst_src;
	3434
	3435	return ir_build_bin_op(irb, scope, node, op_id, op1, op2, true);
	3436	}
	3437
	3438	static IrInstSrc ir_gen_merge_err_sets(IrBuilderSrc irb, Scope scope, AstNode node) {
	3439	IrInstSrc *op1 = ir_gen_node(irb, node->data.bin_op_expr.op1, scope);
	3440	IrInstSrc *op2 = ir_gen_node(irb, node->data.bin_op_expr.op2, scope);
	3441
	3442	if (op1 == irb->codegen->invalid_inst_src \|\| op2 == irb->codegen->invalid_inst_src)
	3443	return irb->codegen->invalid_inst_src;
	3444
	3445	// TODO only pass type_name when the \|\| operator is the top level AST node in the var decl expr
	3446	Buf bare_name = BUF_INIT;
	3447	Buf *type_name = get_anon_type_name(irb->codegen, irb->exec, "error", scope, node, &bare_name);
	3448
	3449	return ir_build_merge_err_sets(irb, scope, node, op1, op2, type_name);
	3450	}
	3451
	3452	static IrInstSrc ir_gen_assign(IrBuilderSrc irb, Scope scope, AstNode node) {
	3453	IrInstSrc *lvalue = ir_gen_node_extra(irb, node->data.bin_op_expr.op1, scope, LValAssign, nullptr);
	3454	if (lvalue == irb->codegen->invalid_inst_src)
	3455	return irb->codegen->invalid_inst_src;
	3456
	3457	ResultLocInstruction *result_loc_inst = heap::c_allocator.create<ResultLocInstruction>();
	3458	result_loc_inst->base.id = ResultLocIdInstruction;
	3459	result_loc_inst->base.source_instruction = lvalue;
	3460	ir_ref_instruction(lvalue, irb->current_basic_block);
	3461	ir_build_reset_result(irb, scope, node, &result_loc_inst->base);
	3462
	3463	IrInstSrc *rvalue = ir_gen_node_extra(irb, node->data.bin_op_expr.op2, scope, LValNone,
	3464	&result_loc_inst->base);
	3465	if (rvalue == irb->codegen->invalid_inst_src)
	3466	return irb->codegen->invalid_inst_src;
	3467
	3468	return ir_build_const_void(irb, scope, node);
	3469	}
	3470
	3471	static IrInstSrc ir_gen_assign_op(IrBuilderSrc irb, Scope scope, AstNode node, IrBinOp op_id) {
	3472	IrInstSrc *lvalue = ir_gen_node_extra(irb, node->data.bin_op_expr.op1, scope, LValAssign, nullptr);
	3473	if (lvalue == irb->codegen->invalid_inst_src)
	3474	return lvalue;
	3475	IrInstSrc *op1 = ir_build_load_ptr(irb, scope, node->data.bin_op_expr.op1, lvalue);
	3476	IrInstSrc *op2 = ir_gen_node(irb, node->data.bin_op_expr.op2, scope);
	3477	if (op2 == irb->codegen->invalid_inst_src)
	3478	return op2;
	3479	IrInstSrc *result = ir_build_bin_op(irb, scope, node, op_id, op1, op2, true);
	3480	ir_build_store_ptr(irb, scope, node, lvalue, result);
	3481	return ir_build_const_void(irb, scope, node);
	3482	}
	3483
	3484	static IrInstSrc ir_gen_bool_or(IrBuilderSrc irb, Scope scope, AstNode node) {
	3485	assert(node->type == NodeTypeBinOpExpr);
	3486
	3487	IrInstSrc *val1 = ir_gen_node(irb, node->data.bin_op_expr.op1, scope);
	3488	if (val1 == irb->codegen->invalid_inst_src)
	3489	return irb->codegen->invalid_inst_src;
	3490	IrBasicBlockSrc *post_val1_block = irb->current_basic_block;
	3491
	3492	IrInstSrc *is_comptime;
	3493	if (ir_should_inline(irb->exec, scope)) {
	3494	is_comptime = ir_build_const_bool(irb, scope, node, true);
	3495	} else {
	3496	is_comptime = ir_build_test_comptime(irb, scope, node, val1);
	3497	}
	3498
	3499	// block for when val1 == false
	3500	IrBasicBlockSrc *false_block = ir_create_basic_block(irb, scope, "BoolOrFalse");
	3501	// block for when val1 == true (don't even evaluate the second part)
	3502	IrBasicBlockSrc *true_block = ir_create_basic_block(irb, scope, "BoolOrTrue");
	3503
	3504	ir_build_cond_br(irb, scope, node, val1, true_block, false_block, is_comptime);
	3505
	3506	ir_set_cursor_at_end_and_append_block(irb, false_block);
	3507	IrInstSrc *val2 = ir_gen_node(irb, node->data.bin_op_expr.op2, scope);
	3508	if (val2 == irb->codegen->invalid_inst_src)
	3509	return irb->codegen->invalid_inst_src;
	3510	IrBasicBlockSrc *post_val2_block = irb->current_basic_block;
	3511
	3512	ir_build_br(irb, scope, node, true_block, is_comptime);
	3513
	3514	ir_set_cursor_at_end_and_append_block(irb, true_block);
	3515
	3516	IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
	3517	incoming_values[0] = val1;
	3518	incoming_values[1] = val2;
	3519	IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
	3520	incoming_blocks[0] = post_val1_block;
	3521	incoming_blocks[1] = post_val2_block;
	3522
	3523	return ir_build_phi(irb, scope, node, 2, incoming_blocks, incoming_values, nullptr);
	3524	}
	3525
	3526	static IrInstSrc ir_gen_bool_and(IrBuilderSrc irb, Scope scope, AstNode node) {
	3527	assert(node->type == NodeTypeBinOpExpr);
	3528
	3529	IrInstSrc *val1 = ir_gen_node(irb, node->data.bin_op_expr.op1, scope);
	3530	if (val1 == irb->codegen->invalid_inst_src)
	3531	return irb->codegen->invalid_inst_src;
	3532	IrBasicBlockSrc *post_val1_block = irb->current_basic_block;
	3533
	3534	IrInstSrc *is_comptime;
	3535	if (ir_should_inline(irb->exec, scope)) {
	3536	is_comptime = ir_build_const_bool(irb, scope, node, true);
	3537	} else {
	3538	is_comptime = ir_build_test_comptime(irb, scope, node, val1);
	3539	}
	3540
	3541	// block for when val1 == true
	3542	IrBasicBlockSrc *true_block = ir_create_basic_block(irb, scope, "BoolAndTrue");
	3543	// block for when val1 == false (don't even evaluate the second part)
	3544	IrBasicBlockSrc *false_block = ir_create_basic_block(irb, scope, "BoolAndFalse");
	3545
	3546	ir_build_cond_br(irb, scope, node, val1, true_block, false_block, is_comptime);
	3547
	3548	ir_set_cursor_at_end_and_append_block(irb, true_block);
	3549	IrInstSrc *val2 = ir_gen_node(irb, node->data.bin_op_expr.op2, scope);
	3550	if (val2 == irb->codegen->invalid_inst_src)
	3551	return irb->codegen->invalid_inst_src;
	3552	IrBasicBlockSrc *post_val2_block = irb->current_basic_block;
	3553
	3554	ir_build_br(irb, scope, node, false_block, is_comptime);
	3555
	3556	ir_set_cursor_at_end_and_append_block(irb, false_block);
	3557
	3558	IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
	3559	incoming_values[0] = val1;
	3560	incoming_values[1] = val2;
	3561	IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
	3562	incoming_blocks[0] = post_val1_block;
	3563	incoming_blocks[1] = post_val2_block;
	3564
	3565	return ir_build_phi(irb, scope, node, 2, incoming_blocks, incoming_values, nullptr);
	3566	}
	3567
	3568	static ResultLocPeerParent ir_build_result_peers(IrBuilderSrc irb, IrInstSrc *cond_br_inst,
	3569	IrBasicBlockSrc end_block, ResultLoc parent, IrInstSrc *is_comptime)
	3570	{
	3571	ResultLocPeerParent *peer_parent = heap::c_allocator.create<ResultLocPeerParent>();
	3572	peer_parent->base.id = ResultLocIdPeerParent;
	3573	peer_parent->base.source_instruction = cond_br_inst;
	3574	peer_parent->base.allow_write_through_const = parent->allow_write_through_const;
	3575	peer_parent->end_bb = end_block;
	3576	peer_parent->is_comptime = is_comptime;
	3577	peer_parent->parent = parent;
	3578
	3579	IrInstSrc *popped_inst = irb->current_basic_block->instruction_list.pop();
	3580	ir_assert(popped_inst == cond_br_inst, &cond_br_inst->base);
	3581
	3582	ir_build_reset_result(irb, cond_br_inst->base.scope, cond_br_inst->base.source_node, &peer_parent->base);
	3583	irb->current_basic_block->instruction_list.append(popped_inst);
	3584
	3585	return peer_parent;
	3586	}
	3587
	3588	static ResultLocPeerParent ir_build_binary_result_peers(IrBuilderSrc irb, IrInstSrc *cond_br_inst,
	3589	IrBasicBlockSrc else_block, IrBasicBlockSrc end_block, ResultLoc parent, IrInstSrc is_comptime)
	3590	{
	3591	ResultLocPeerParent *peer_parent = ir_build_result_peers(irb, cond_br_inst, end_block, parent, is_comptime);
	3592
	3593	peer_parent->peers.append(create_peer_result(peer_parent));
	3594	peer_parent->peers.last()->next_bb = else_block;
	3595
	3596	peer_parent->peers.append(create_peer_result(peer_parent));
	3597	peer_parent->peers.last()->next_bb = end_block;
	3598
	3599	return peer_parent;
	3600	}
	3601
	3602	static IrInstSrc ir_gen_orelse(IrBuilderSrc irb, Scope parent_scope, AstNode node, LVal lval,
	3603	ResultLoc *result_loc)
	3604	{
	3605	assert(node->type == NodeTypeBinOpExpr);
	3606
	3607	AstNode *op1_node = node->data.bin_op_expr.op1;
	3608	AstNode *op2_node = node->data.bin_op_expr.op2;
	3609
	3610	IrInstSrc *maybe_ptr = ir_gen_node_extra(irb, op1_node, parent_scope, LValPtr, nullptr);
	3611	if (maybe_ptr == irb->codegen->invalid_inst_src)
	3612	return irb->codegen->invalid_inst_src;
	3613
	3614	IrInstSrc *maybe_val = ir_build_load_ptr(irb, parent_scope, node, maybe_ptr);
	3615	IrInstSrc *is_non_null = ir_build_test_non_null_src(irb, parent_scope, node, maybe_val);
	3616
	3617	IrInstSrc *is_comptime;
	3618	if (ir_should_inline(irb->exec, parent_scope)) {
	3619	is_comptime = ir_build_const_bool(irb, parent_scope, node, true);
	3620	} else {
	3621	is_comptime = ir_build_test_comptime(irb, parent_scope, node, is_non_null);
	3622	}
	3623
	3624	IrBasicBlockSrc *ok_block = ir_create_basic_block(irb, parent_scope, "OptionalNonNull");
	3625	IrBasicBlockSrc *null_block = ir_create_basic_block(irb, parent_scope, "OptionalNull");
	3626	IrBasicBlockSrc *end_block = ir_create_basic_block(irb, parent_scope, "OptionalEnd");
	3627	IrInstSrc *cond_br_inst = ir_build_cond_br(irb, parent_scope, node, is_non_null, ok_block, null_block, is_comptime);
	3628
	3629	ResultLocPeerParent *peer_parent = ir_build_binary_result_peers(irb, cond_br_inst, ok_block, end_block,
	3630	result_loc, is_comptime);
	3631
	3632	ir_set_cursor_at_end_and_append_block(irb, null_block);
	3633	IrInstSrc *null_result = ir_gen_node_extra(irb, op2_node, parent_scope, LValNone,
	3634	&peer_parent->peers.at(0)->base);
	3635	if (null_result == irb->codegen->invalid_inst_src)
	3636	return irb->codegen->invalid_inst_src;
	3637	IrBasicBlockSrc *after_null_block = irb->current_basic_block;
	3638	if (!instr_is_unreachable(null_result))
	3639	ir_mark_gen(ir_build_br(irb, parent_scope, node, end_block, is_comptime));
	3640
	3641	ir_set_cursor_at_end_and_append_block(irb, ok_block);
	3642	IrInstSrc *unwrapped_ptr = ir_build_optional_unwrap_ptr(irb, parent_scope, node, maybe_ptr, false);
	3643	IrInstSrc *unwrapped_payload = ir_build_load_ptr(irb, parent_scope, node, unwrapped_ptr);
	3644	ir_build_end_expr(irb, parent_scope, node, unwrapped_payload, &peer_parent->peers.at(1)->base);
	3645	IrBasicBlockSrc *after_ok_block = irb->current_basic_block;
	3646	ir_build_br(irb, parent_scope, node, end_block, is_comptime);
	3647
	3648	ir_set_cursor_at_end_and_append_block(irb, end_block);
	3649	IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
	3650	incoming_values[0] = null_result;
	3651	incoming_values[1] = unwrapped_payload;
	3652	IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
	3653	incoming_blocks[0] = after_null_block;
	3654	incoming_blocks[1] = after_ok_block;
	3655	IrInstSrc *phi = ir_build_phi(irb, parent_scope, node, 2, incoming_blocks, incoming_values, peer_parent);
	3656	return ir_lval_wrap(irb, parent_scope, phi, lval, result_loc);
	3657	}
	3658
	3659	static IrInstSrc ir_gen_error_union(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
	3660	assert(node->type == NodeTypeBinOpExpr);
	3661
	3662	AstNode *op1_node = node->data.bin_op_expr.op1;
	3663	AstNode *op2_node = node->data.bin_op_expr.op2;
	3664
	3665	IrInstSrc *err_set = ir_gen_node(irb, op1_node, parent_scope);
	3666	if (err_set == irb->codegen->invalid_inst_src)
	3667	return irb->codegen->invalid_inst_src;
	3668
	3669	IrInstSrc *payload = ir_gen_node(irb, op2_node, parent_scope);
	3670	if (payload == irb->codegen->invalid_inst_src)
	3671	return irb->codegen->invalid_inst_src;
	3672
	3673	return ir_build_error_union(irb, parent_scope, node, err_set, payload);
	3674	}
	3675
	3676	static IrInstSrc ir_gen_bin_op(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval, ResultLoc *result_loc) {
	3677	assert(node->type == NodeTypeBinOpExpr);
	3678
	3679	BinOpType bin_op_type = node->data.bin_op_expr.bin_op;
	3680	switch (bin_op_type) {
	3681	case BinOpTypeInvalid:
	3682	zig_unreachable();
	3683	case BinOpTypeAssign:
	3684	return ir_lval_wrap(irb, scope, ir_gen_assign(irb, scope, node), lval, result_loc);
	3685	case BinOpTypeAssignTimes:
	3686	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpMult), lval, result_loc);
	3687	case BinOpTypeAssignTimesWrap:
	3688	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpMultWrap), lval, result_loc);
	3689	case BinOpTypeAssignDiv:
	3690	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpDivUnspecified), lval, result_loc);
	3691	case BinOpTypeAssignMod:
	3692	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpRemUnspecified), lval, result_loc);
	3693	case BinOpTypeAssignPlus:
	3694	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpAdd), lval, result_loc);
	3695	case BinOpTypeAssignPlusWrap:
	3696	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpAddWrap), lval, result_loc);
	3697	case BinOpTypeAssignMinus:
	3698	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpSub), lval, result_loc);
	3699	case BinOpTypeAssignMinusWrap:
	3700	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpSubWrap), lval, result_loc);
	3701	case BinOpTypeAssignBitShiftLeft:
	3702	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpBitShiftLeftLossy), lval, result_loc);
	3703	case BinOpTypeAssignBitShiftRight:
	3704	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpBitShiftRightLossy), lval, result_loc);
	3705	case BinOpTypeAssignBitAnd:
	3706	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpBinAnd), lval, result_loc);
	3707	case BinOpTypeAssignBitXor:
	3708	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpBinXor), lval, result_loc);
	3709	case BinOpTypeAssignBitOr:
	3710	return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpBinOr), lval, result_loc);
	3711	case BinOpTypeBoolOr:
	3712	return ir_lval_wrap(irb, scope, ir_gen_bool_or(irb, scope, node), lval, result_loc);
	3713	case BinOpTypeBoolAnd:
	3714	return ir_lval_wrap(irb, scope, ir_gen_bool_and(irb, scope, node), lval, result_loc);
	3715	case BinOpTypeCmpEq:
	3716	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpEq), lval, result_loc);
	3717	case BinOpTypeCmpNotEq:
	3718	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpNotEq), lval, result_loc);
	3719	case BinOpTypeCmpLessThan:
	3720	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpLessThan), lval, result_loc);
	3721	case BinOpTypeCmpGreaterThan:
	3722	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpGreaterThan), lval, result_loc);
	3723	case BinOpTypeCmpLessOrEq:
	3724	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpLessOrEq), lval, result_loc);
	3725	case BinOpTypeCmpGreaterOrEq:
	3726	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpGreaterOrEq), lval, result_loc);
	3727	case BinOpTypeBinOr:
	3728	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpBinOr), lval, result_loc);
	3729	case BinOpTypeBinXor:
	3730	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpBinXor), lval, result_loc);
	3731	case BinOpTypeBinAnd:
	3732	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpBinAnd), lval, result_loc);
	3733	case BinOpTypeBitShiftLeft:
	3734	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpBitShiftLeftLossy), lval, result_loc);
	3735	case BinOpTypeBitShiftRight:
	3736	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpBitShiftRightLossy), lval, result_loc);
	3737	case BinOpTypeAdd:
	3738	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpAdd), lval, result_loc);
	3739	case BinOpTypeAddWrap:
	3740	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpAddWrap), lval, result_loc);
	3741	case BinOpTypeSub:
	3742	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpSub), lval, result_loc);
	3743	case BinOpTypeSubWrap:
	3744	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpSubWrap), lval, result_loc);
	3745	case BinOpTypeMult:
	3746	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpMult), lval, result_loc);
	3747	case BinOpTypeMultWrap:
	3748	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpMultWrap), lval, result_loc);
	3749	case BinOpTypeDiv:
	3750	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpDivUnspecified), lval, result_loc);
	3751	case BinOpTypeMod:
	3752	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpRemUnspecified), lval, result_loc);
	3753	case BinOpTypeArrayCat:
	3754	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpArrayCat), lval, result_loc);
	3755	case BinOpTypeArrayMult:
	3756	return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpArrayMult), lval, result_loc);
	3757	case BinOpTypeMergeErrorSets:
	3758	return ir_lval_wrap(irb, scope, ir_gen_merge_err_sets(irb, scope, node), lval, result_loc);
	3759	case BinOpTypeUnwrapOptional:
	3760	return ir_gen_orelse(irb, scope, node, lval, result_loc);
	3761	case BinOpTypeErrorUnion:
	3762	return ir_lval_wrap(irb, scope, ir_gen_error_union(irb, scope, node), lval, result_loc);
	3763	}
	3764	zig_unreachable();
	3765	}
	3766
	3767	static IrInstSrc ir_gen_int_lit(IrBuilderSrc irb, Scope scope, AstNode node) {
	3768	assert(node->type == NodeTypeIntLiteral);
	3769
	3770	RootStruct *root_struct = node->owner->data.structure.root_struct;
	3771	BigInt bigint;
	3772	token_number_literal_bigint(root_struct, &bigint, node->main_token);
	3773	return ir_build_const_bigint(irb, scope, node, bigint);
	3774	}
	3775
	3776	static IrInstSrc ir_gen_float_lit(IrBuilderSrc irb, Scope scope, AstNode node) {
	3777	Error err;
	3778	assert(node->type == NodeTypeFloatLiteral);
	3779
	3780	RootStruct *root_struct = node->owner->data.structure.root_struct;
	3781	const char *source = buf_ptr(root_struct->source_code);
	3782	uint32_t byte_offset = root_struct->token_locs[node->main_token].offset;
	3783
	3784	BigFloat bigfloat;
	3785	if ((err = bigfloat_init_buf(&bigfloat, (const uint8_t *)source + byte_offset))) {
	3786	add_node_error(irb->codegen, node, buf_sprintf("float literal out of range of any type"));
	3787	return irb->codegen->invalid_inst_src;
	3788	}
	3789
	3790	return ir_build_const_bigfloat(irb, scope, node, bigfloat);
	3791	}
	3792
	3793	static IrInstSrc ir_gen_char_lit(IrBuilderSrc irb, Scope scope, AstNode node) {
	3794	Error err;
	3795	assert(node->type == NodeTypeCharLiteral);
	3796
	3797	RootStruct *root_struct = node->owner->data.structure.root_struct;
	3798	const char *source = buf_ptr(root_struct->source_code);
	3799	uint32_t byte_offset = root_struct->token_locs[node->main_token].offset;
	3800
	3801	src_assert(source[byte_offset] == '\'', node);
	3802	byte_offset += 1;
	3803
	3804	uint32_t codepoint;
	3805	size_t bad_index;
	3806	if ((err = source_char_literal(source + byte_offset, &codepoint, &bad_index))) {
	3807	add_node_error(irb->codegen, node, buf_sprintf("invalid character"));
	3808	return irb->codegen->invalid_inst_src;
	3809	}
	3810	return ir_build_const_uint(irb, scope, node, codepoint);
	3811	}
	3812
	3813	static IrInstSrc ir_gen_null_literal(IrBuilderSrc irb, Scope scope, AstNode node) {
	3814	assert(node->type == NodeTypeNullLiteral);
	3815
	3816	return ir_build_const_null(irb, scope, node);
	3817	}
	3818
	3819	static IrInstSrc ir_gen_symbol(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval, ResultLoc *result_loc) {
	3820	Error err;
	3821	assert(node->type == NodeTypeIdentifier);
	3822
	3823	Buf *variable_name = node_identifier_buf(node);
	3824
	3825	if (buf_eql_str(variable_name, "_")) {
	3826	if (lval == LValAssign) {
	3827	IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, node);
	3828	const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
	3829	const_instruction->value->type = get_pointer_to_type(irb->codegen,
	3830	irb->codegen->builtin_types.entry_void, false);
	3831	const_instruction->value->special = ConstValSpecialStatic;
	3832	const_instruction->value->data.x_ptr.special = ConstPtrSpecialDiscard;
	3833	return &const_instruction->base;
	3834	} else {
	3835	add_node_error(irb->codegen, node, buf_sprintf("`_` may only be used to assign things to"));
	3836	return irb->codegen->invalid_inst_src;
	3837	}
	3838	}
	3839
	3840	ZigType *primitive_type;
	3841	if ((err = get_primitive_type(irb->codegen, variable_name, &primitive_type))) {
	3842	if (err == ErrorOverflow) {
	3843	add_node_error(irb->codegen, node,
	3844	buf_sprintf("primitive integer type '%s' exceeds maximum bit width of 65535",
	3845	buf_ptr(variable_name)));
	3846	return irb->codegen->invalid_inst_src;
	3847	}
	3848	assert(err == ErrorPrimitiveTypeNotFound);
	3849	} else {
	3850	IrInstSrc *value = ir_build_const_type(irb, scope, node, primitive_type);
	3851	if (lval == LValPtr \|\| lval == LValAssign) {
	3852	return ir_build_ref_src(irb, scope, node, value);
	3853	} else {
	3854	return ir_expr_wrap(irb, scope, value, result_loc);
	3855	}
	3856	}
	3857
	3858	ScopeFnDef *crossed_fndef_scope;
	3859	ZigVar *var = find_variable(irb->codegen, scope, variable_name, &crossed_fndef_scope);
	3860	if (var) {
	3861	IrInstSrc *var_ptr = ir_build_var_ptr_x(irb, scope, node, var, crossed_fndef_scope);
	3862	if (lval == LValPtr \|\| lval == LValAssign) {
	3863	return var_ptr;
	3864	} else {
	3865	return ir_expr_wrap(irb, scope, ir_build_load_ptr(irb, scope, node, var_ptr), result_loc);
	3866	}
	3867	}
	3868
	3869	Tld *tld = find_decl(irb->codegen, scope, variable_name);
	3870	if (tld) {
	3871	IrInstSrc *decl_ref = ir_build_decl_ref(irb, scope, node, tld, lval);
	3872	if (lval == LValPtr \|\| lval == LValAssign) {
	3873	return decl_ref;
	3874	} else {
	3875	return ir_expr_wrap(irb, scope, decl_ref, result_loc);
	3876	}
	3877	}
	3878
	3879	if (get_container_scope(node->owner)->any_imports_failed) {
	3880	// skip the error message since we had a failing import in this file
	3881	// if an import breaks we don't need redundant undeclared identifier errors
	3882	return irb->codegen->invalid_inst_src;
	3883	}
	3884
	3885	return ir_build_undeclared_identifier(irb, scope, node, variable_name);
	3886	}
	3887
	3888	static IrInstSrc ir_gen_array_access(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
	3889	ResultLoc *result_loc)
	3890	{
	3891	assert(node->type == NodeTypeArrayAccessExpr);
	3892
	3893	AstNode *array_ref_node = node->data.array_access_expr.array_ref_expr;
	3894	IrInstSrc *array_ref_instruction = ir_gen_node_extra(irb, array_ref_node, scope, LValPtr, nullptr);
	3895	if (array_ref_instruction == irb->codegen->invalid_inst_src)
	3896	return array_ref_instruction;
	3897
	3898	// Create an usize-typed result location to hold the subscript value, this
	3899	// makes it possible for the compiler to infer the subscript expression type
	3900	// if needed
	3901	IrInstSrc *usize_type_inst = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize);
	3902	ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, usize_type_inst, no_result_loc());
	3903
	3904	AstNode *subscript_node = node->data.array_access_expr.subscript;
	3905	IrInstSrc *subscript_value = ir_gen_node_extra(irb, subscript_node, scope, LValNone, &result_loc_cast->base);
	3906	if (subscript_value == irb->codegen->invalid_inst_src)
	3907	return irb->codegen->invalid_inst_src;
	3908
	3909	IrInstSrc *subscript_instruction = ir_build_implicit_cast(irb, scope, subscript_node, subscript_value, result_loc_cast);
	3910
	3911	IrInstSrc *ptr_instruction = ir_build_elem_ptr(irb, scope, node, array_ref_instruction,
	3912	subscript_instruction, true, PtrLenSingle, nullptr);
	3913	if (lval == LValPtr \|\| lval == LValAssign)
	3914	return ptr_instruction;
	3915
	3916	IrInstSrc *load_ptr = ir_build_load_ptr(irb, scope, node, ptr_instruction);
	3917	return ir_expr_wrap(irb, scope, load_ptr, result_loc);
	3918	}
	3919
	3920	static IrInstSrc ir_gen_field_access(IrBuilderSrc irb, Scope scope, AstNode node) {
	3921	assert(node->type == NodeTypeFieldAccessExpr);
	3922
	3923	AstNode *container_ref_node = node->data.field_access_expr.struct_expr;
	3924	Buf *field_name = node->data.field_access_expr.field_name;
	3925
	3926	IrInstSrc *container_ref_instruction = ir_gen_node_extra(irb, container_ref_node, scope, LValPtr, nullptr);
	3927	if (container_ref_instruction == irb->codegen->invalid_inst_src)
	3928	return container_ref_instruction;
	3929
	3930	return ir_build_field_ptr(irb, scope, node, container_ref_instruction, field_name, false);
	3931	}
	3932
	3933	static IrInstSrc ir_gen_overflow_op(IrBuilderSrc irb, Scope scope, AstNode node, IrOverflowOp op) {
	3934	assert(node->type == NodeTypeFnCallExpr);
	3935
	3936	AstNode *type_node = node->data.fn_call_expr.params.at(0);
	3937	AstNode *op1_node = node->data.fn_call_expr.params.at(1);
	3938	AstNode *op2_node = node->data.fn_call_expr.params.at(2);
	3939	AstNode *result_ptr_node = node->data.fn_call_expr.params.at(3);
	3940
	3941
	3942	IrInstSrc *type_value = ir_gen_node(irb, type_node, scope);
	3943	if (type_value == irb->codegen->invalid_inst_src)
	3944	return irb->codegen->invalid_inst_src;
	3945
	3946	IrInstSrc *op1 = ir_gen_node(irb, op1_node, scope);
	3947	if (op1 == irb->codegen->invalid_inst_src)
	3948	return irb->codegen->invalid_inst_src;
	3949
	3950	IrInstSrc *op2 = ir_gen_node(irb, op2_node, scope);
	3951	if (op2 == irb->codegen->invalid_inst_src)
	3952	return irb->codegen->invalid_inst_src;
	3953
	3954	IrInstSrc *result_ptr = ir_gen_node(irb, result_ptr_node, scope);
	3955	if (result_ptr == irb->codegen->invalid_inst_src)
	3956	return irb->codegen->invalid_inst_src;
	3957
	3958	return ir_build_overflow_op_src(irb, scope, node, op, type_value, op1, op2, result_ptr);
	3959	}
	3960
	3961	static IrInstSrc ir_gen_mul_add(IrBuilderSrc irb, Scope scope, AstNode node) {
	3962	assert(node->type == NodeTypeFnCallExpr);
	3963
	3964	AstNode *type_node = node->data.fn_call_expr.params.at(0);
	3965	AstNode *op1_node = node->data.fn_call_expr.params.at(1);
	3966	AstNode *op2_node = node->data.fn_call_expr.params.at(2);
	3967	AstNode *op3_node = node->data.fn_call_expr.params.at(3);
	3968
	3969	IrInstSrc *type_value = ir_gen_node(irb, type_node, scope);
	3970	if (type_value == irb->codegen->invalid_inst_src)
	3971	return irb->codegen->invalid_inst_src;
	3972
	3973	IrInstSrc *op1 = ir_gen_node(irb, op1_node, scope);
	3974	if (op1 == irb->codegen->invalid_inst_src)
	3975	return irb->codegen->invalid_inst_src;
	3976
	3977	IrInstSrc *op2 = ir_gen_node(irb, op2_node, scope);
	3978	if (op2 == irb->codegen->invalid_inst_src)
	3979	return irb->codegen->invalid_inst_src;
	3980
	3981	IrInstSrc *op3 = ir_gen_node(irb, op3_node, scope);
	3982	if (op3 == irb->codegen->invalid_inst_src)
	3983	return irb->codegen->invalid_inst_src;
	3984
	3985	return ir_build_mul_add_src(irb, scope, node, type_value, op1, op2, op3);
	3986	}
	3987
	3988	static IrInstSrc ir_gen_this(IrBuilderSrc irb, Scope orig_scope, AstNode node) {
	3989	for (Scope *it_scope = orig_scope; it_scope != nullptr; it_scope = it_scope->parent) {
	3990	if (it_scope->id == ScopeIdDecls) {
	3991	ScopeDecls decls_scope = (ScopeDecls )it_scope;
	3992	ZigType *container_type = decls_scope->container_type;
	3993	if (container_type != nullptr) {
	3994	return ir_build_const_type(irb, orig_scope, node, container_type);
	3995	} else {
	3996	return ir_build_const_import(irb, orig_scope, node, decls_scope->import);
	3997	}
	3998	}
	3999	}
	4000	zig_unreachable();
	4001	}
	4002
	4003	static IrInstSrc ir_gen_async_call(IrBuilderSrc irb, Scope scope, AstNode await_node, AstNode *call_node,
	4004	LVal lval, ResultLoc *result_loc)
	4005	{
	4006	if (call_node->data.fn_call_expr.params.length != 4) {
	4007	add_node_error(irb->codegen, call_node,
	4008	buf_sprintf("expected 4 arguments, found %" ZIG_PRI_usize,
	4009	call_node->data.fn_call_expr.params.length));
	4010	return irb->codegen->invalid_inst_src;
	4011	}
	4012
	4013	AstNode *bytes_node = call_node->data.fn_call_expr.params.at(0);
	4014	IrInstSrc *bytes = ir_gen_node(irb, bytes_node, scope);
	4015	if (bytes == irb->codegen->invalid_inst_src)
	4016	return bytes;
	4017
	4018	AstNode *ret_ptr_node = call_node->data.fn_call_expr.params.at(1);
	4019	IrInstSrc *ret_ptr = ir_gen_node(irb, ret_ptr_node, scope);
	4020	if (ret_ptr == irb->codegen->invalid_inst_src)
	4021	return ret_ptr;
	4022
	4023	AstNode *fn_ref_node = call_node->data.fn_call_expr.params.at(2);
	4024	IrInstSrc *fn_ref = ir_gen_node(irb, fn_ref_node, scope);
	4025	if (fn_ref == irb->codegen->invalid_inst_src)
	4026	return fn_ref;
	4027
	4028	CallModifier modifier = (await_node == nullptr) ? CallModifierAsync : CallModifierNone;
	4029	bool is_async_call_builtin = true;
	4030	AstNode *args_node = call_node->data.fn_call_expr.params.at(3);
	4031	if (args_node->type == NodeTypeContainerInitExpr) {
	4032	if (args_node->data.container_init_expr.kind == ContainerInitKindArray \|\|
	4033	args_node->data.container_init_expr.entries.length == 0)
	4034	{
	4035	size_t arg_count = args_node->data.container_init_expr.entries.length;
	4036	IrInstSrc *args = heap::c_allocator.allocate<IrInstSrc>(arg_count);
	4037	for (size_t i = 0; i < arg_count; i += 1) {
	4038	AstNode *arg_node = args_node->data.container_init_expr.entries.at(i);
	4039	IrInstSrc *arg = ir_gen_node(irb, arg_node, scope);
	4040	if (arg == irb->codegen->invalid_inst_src)
	4041	return arg;
	4042	args[i] = arg;
	4043	}
	4044
	4045	IrInstSrc *call = ir_build_call_src(irb, scope, call_node, nullptr, fn_ref, arg_count, args,
	4046	ret_ptr, modifier, is_async_call_builtin, bytes, result_loc);
	4047	return ir_lval_wrap(irb, scope, call, lval, result_loc);
	4048	} else {
	4049	exec_add_error_node(irb->codegen, irb->exec, args_node,
	4050	buf_sprintf("TODO: @asyncCall with anon struct literal"));
	4051	return irb->codegen->invalid_inst_src;
	4052	}
	4053	}
	4054	IrInstSrc *args = ir_gen_node(irb, args_node, scope);
	4055	if (args == irb->codegen->invalid_inst_src)
	4056	return args;
	4057
	4058	IrInstSrc *call = ir_build_async_call_extra(irb, scope, call_node, modifier, fn_ref, ret_ptr, bytes, args, result_loc);
	4059	return ir_lval_wrap(irb, scope, call, lval, result_loc);
	4060	}
	4061
	4062	static IrInstSrc ir_gen_fn_call_with_args(IrBuilderSrc irb, Scope scope, AstNode source_node,
	4063	AstNode fn_ref_node, CallModifier modifier, IrInstSrc options,
	4064	AstNode *args_ptr, size_t args_len, LVal lval, ResultLoc result_loc)
	4065	{
	4066	IrInstSrc *fn_ref = ir_gen_node(irb, fn_ref_node, scope);
	4067	if (fn_ref == irb->codegen->invalid_inst_src)
	4068	return fn_ref;
	4069
	4070	IrInstSrc *fn_type = ir_build_typeof_1(irb, scope, source_node, fn_ref);
	4071
	4072	IrInstSrc *args = heap::c_allocator.allocate<IrInstSrc>(args_len);
	4073	for (size_t i = 0; i < args_len; i += 1) {
	4074	AstNode *arg_node = args_ptr[i];
	4075
	4076	IrInstSrc *arg_index = ir_build_const_usize(irb, scope, arg_node, i);
	4077	IrInstSrc *arg_type = ir_build_arg_type(irb, scope, source_node, fn_type, arg_index, true);
	4078	ResultLoc *no_result = no_result_loc();
	4079	ir_build_reset_result(irb, scope, source_node, no_result);
	4080	ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, arg_type, no_result);
	4081
	4082	IrInstSrc *arg = ir_gen_node_extra(irb, arg_node, scope, LValNone, &result_loc_cast->base);
	4083	if (arg == irb->codegen->invalid_inst_src)
	4084	return arg;
	4085
	4086	args[i] = ir_build_implicit_cast(irb, scope, arg_node, arg, result_loc_cast);
	4087	}
	4088
	4089	IrInstSrc *fn_call;
	4090	if (options != nullptr) {
	4091	fn_call = ir_build_call_args(irb, scope, source_node, options, fn_ref, args, args_len, result_loc);
	4092	} else {
	4093	fn_call = ir_build_call_src(irb, scope, source_node, nullptr, fn_ref, args_len, args, nullptr,
	4094	modifier, false, nullptr, result_loc);
	4095	}
	4096	return ir_lval_wrap(irb, scope, fn_call, lval, result_loc);
	4097	}
	4098
	4099	static IrInstSrc ir_gen_builtin_fn_call(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
	4100	ResultLoc *result_loc)
	4101	{
	4102	assert(node->type == NodeTypeFnCallExpr);
	4103
	4104	AstNode *fn_ref_expr = node->data.fn_call_expr.fn_ref_expr;
	4105	Buf *name = node_identifier_buf(fn_ref_expr);
	4106	auto entry = irb->codegen->builtin_fn_table.maybe_get(name);
	4107
	4108	if (!entry) {
	4109	add_node_error(irb->codegen, node,
	4110	buf_sprintf("invalid builtin function: '%s'", buf_ptr(name)));
	4111	return irb->codegen->invalid_inst_src;
	4112	}
	4113
	4114	BuiltinFnEntry *builtin_fn = entry->value;
	4115	size_t actual_param_count = node->data.fn_call_expr.params.length;
	4116
	4117	if (builtin_fn->param_count != SIZE_MAX && builtin_fn->param_count != actual_param_count) {
	4118	add_node_error(irb->codegen, node,
	4119	buf_sprintf("expected %" ZIG_PRI_usize " argument(s), found %" ZIG_PRI_usize,
	4120	builtin_fn->param_count, actual_param_count));
	4121	return irb->codegen->invalid_inst_src;
	4122	}
	4123
	4124	switch (builtin_fn->id) {
	4125	case BuiltinFnIdInvalid:
	4126	zig_unreachable();
	4127	case BuiltinFnIdTypeof:
	4128	{
	4129	Scope *sub_scope = create_typeof_scope(irb->codegen, node, scope);
	4130
	4131	size_t arg_count = node->data.fn_call_expr.params.length;
	4132
	4133	IrInstSrc *type_of;
	4134
	4135	if (arg_count == 0) {
	4136	add_node_error(irb->codegen, node,
	4137	buf_sprintf("expected at least 1 argument, found 0"));
	4138	return irb->codegen->invalid_inst_src;
	4139	} else if (arg_count == 1) {
	4140	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4141	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, sub_scope);
	4142	if (arg0_value == irb->codegen->invalid_inst_src)
	4143	return arg0_value;
	4144
	4145	type_of = ir_build_typeof_1(irb, scope, node, arg0_value);
	4146	} else {
	4147	IrInstSrc *args = heap::c_allocator.allocate<IrInstSrc>(arg_count);
	4148	for (size_t i = 0; i < arg_count; i += 1) {
	4149	AstNode *arg_node = node->data.fn_call_expr.params.at(i);
	4150	IrInstSrc *arg = ir_gen_node(irb, arg_node, sub_scope);
	4151	if (arg == irb->codegen->invalid_inst_src)
	4152	return irb->codegen->invalid_inst_src;
	4153	args[i] = arg;
	4154	}
	4155
	4156	type_of = ir_build_typeof_n(irb, scope, node, args, arg_count);
	4157	}
	4158	return ir_lval_wrap(irb, scope, type_of, lval, result_loc);
	4159	}
	4160	case BuiltinFnIdSetCold:
	4161	{
	4162	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4163	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4164	if (arg0_value == irb->codegen->invalid_inst_src)
	4165	return arg0_value;
	4166
	4167	IrInstSrc *set_cold = ir_build_set_cold(irb, scope, node, arg0_value);
	4168	return ir_lval_wrap(irb, scope, set_cold, lval, result_loc);
	4169	}
	4170	case BuiltinFnIdSetRuntimeSafety:
	4171	{
	4172	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4173	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4174	if (arg0_value == irb->codegen->invalid_inst_src)
	4175	return arg0_value;
	4176
	4177	IrInstSrc *set_safety = ir_build_set_runtime_safety(irb, scope, node, arg0_value);
	4178	return ir_lval_wrap(irb, scope, set_safety, lval, result_loc);
	4179	}
	4180	case BuiltinFnIdSetFloatMode:
	4181	{
	4182	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4183	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4184	if (arg0_value == irb->codegen->invalid_inst_src)
	4185	return arg0_value;
	4186
	4187	IrInstSrc *set_float_mode = ir_build_set_float_mode(irb, scope, node, arg0_value);
	4188	return ir_lval_wrap(irb, scope, set_float_mode, lval, result_loc);
	4189	}
	4190	case BuiltinFnIdSizeof:
	4191	case BuiltinFnIdBitSizeof:
	4192	{
	4193	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4194	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4195	if (arg0_value == irb->codegen->invalid_inst_src)
	4196	return arg0_value;
	4197
	4198	IrInstSrc *size_of = ir_build_size_of(irb, scope, node, arg0_value, builtin_fn->id == BuiltinFnIdBitSizeof);
	4199	return ir_lval_wrap(irb, scope, size_of, lval, result_loc);
	4200	}
	4201	case BuiltinFnIdImport:
	4202	{
	4203	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4204	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4205	if (arg0_value == irb->codegen->invalid_inst_src)
	4206	return arg0_value;
	4207
	4208	IrInstSrc *import = ir_build_import(irb, scope, node, arg0_value);
	4209	return ir_lval_wrap(irb, scope, import, lval, result_loc);
	4210	}
	4211	case BuiltinFnIdCImport:
	4212	{
	4213	IrInstSrc *c_import = ir_build_c_import(irb, scope, node);
	4214	return ir_lval_wrap(irb, scope, c_import, lval, result_loc);
	4215	}
	4216	case BuiltinFnIdCInclude:
	4217	{
	4218	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4219	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4220	if (arg0_value == irb->codegen->invalid_inst_src)
	4221	return arg0_value;
	4222
	4223	if (!exec_c_import_buf(irb->exec)) {
	4224	add_node_error(irb->codegen, node, buf_sprintf("C include valid only inside C import block"));
	4225	return irb->codegen->invalid_inst_src;
	4226	}
	4227
	4228	IrInstSrc *c_include = ir_build_c_include(irb, scope, node, arg0_value);
	4229	return ir_lval_wrap(irb, scope, c_include, lval, result_loc);
	4230	}
	4231	case BuiltinFnIdCDefine:
	4232	{
	4233	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4234	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4235	if (arg0_value == irb->codegen->invalid_inst_src)
	4236	return arg0_value;
	4237
	4238	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4239	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4240	if (arg1_value == irb->codegen->invalid_inst_src)
	4241	return arg1_value;
	4242
	4243	if (!exec_c_import_buf(irb->exec)) {
	4244	add_node_error(irb->codegen, node, buf_sprintf("C define valid only inside C import block"));
	4245	return irb->codegen->invalid_inst_src;
	4246	}
	4247
	4248	IrInstSrc *c_define = ir_build_c_define(irb, scope, node, arg0_value, arg1_value);
	4249	return ir_lval_wrap(irb, scope, c_define, lval, result_loc);
	4250	}
	4251	case BuiltinFnIdCUndef:
	4252	{
	4253	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4254	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4255	if (arg0_value == irb->codegen->invalid_inst_src)
	4256	return arg0_value;
	4257
	4258	if (!exec_c_import_buf(irb->exec)) {
	4259	add_node_error(irb->codegen, node, buf_sprintf("C undef valid only inside C import block"));
	4260	return irb->codegen->invalid_inst_src;
	4261	}
	4262
	4263	IrInstSrc *c_undef = ir_build_c_undef(irb, scope, node, arg0_value);
	4264	return ir_lval_wrap(irb, scope, c_undef, lval, result_loc);
	4265	}
	4266	case BuiltinFnIdCompileErr:
	4267	{
	4268	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4269	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4270	if (arg0_value == irb->codegen->invalid_inst_src)
	4271	return arg0_value;
	4272
	4273	IrInstSrc *compile_err = ir_build_compile_err(irb, scope, node, arg0_value);
	4274	return ir_lval_wrap(irb, scope, compile_err, lval, result_loc);
	4275	}
	4276	case BuiltinFnIdCompileLog:
	4277	{
	4278	IrInstSrc *args = heap::c_allocator.allocate<IrInstSrc>(actual_param_count);
	4279
	4280	for (size_t i = 0; i < actual_param_count; i += 1) {
	4281	AstNode *arg_node = node->data.fn_call_expr.params.at(i);
	4282	args[i] = ir_gen_node(irb, arg_node, scope);
	4283	if (args[i] == irb->codegen->invalid_inst_src)
	4284	return irb->codegen->invalid_inst_src;
	4285	}
	4286
	4287	IrInstSrc *compile_log = ir_build_compile_log(irb, scope, node, actual_param_count, args);
	4288	return ir_lval_wrap(irb, scope, compile_log, lval, result_loc);
	4289	}
	4290	case BuiltinFnIdErrName:
	4291	{
	4292	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4293	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4294	if (arg0_value == irb->codegen->invalid_inst_src)
	4295	return arg0_value;
	4296
	4297	IrInstSrc *err_name = ir_build_err_name(irb, scope, node, arg0_value);
	4298	return ir_lval_wrap(irb, scope, err_name, lval, result_loc);
	4299	}
	4300	case BuiltinFnIdEmbedFile:
	4301	{
	4302	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4303	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4304	if (arg0_value == irb->codegen->invalid_inst_src)
	4305	return arg0_value;
	4306
	4307	IrInstSrc *embed_file = ir_build_embed_file(irb, scope, node, arg0_value);
	4308	return ir_lval_wrap(irb, scope, embed_file, lval, result_loc);
	4309	}
	4310	case BuiltinFnIdCmpxchgWeak:
	4311	case BuiltinFnIdCmpxchgStrong:
	4312	{
	4313	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4314	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4315	if (arg0_value == irb->codegen->invalid_inst_src)
	4316	return arg0_value;
	4317
	4318	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4319	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4320	if (arg1_value == irb->codegen->invalid_inst_src)
	4321	return arg1_value;
	4322
	4323	AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
	4324	IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
	4325	if (arg2_value == irb->codegen->invalid_inst_src)
	4326	return arg2_value;
	4327
	4328	AstNode *arg3_node = node->data.fn_call_expr.params.at(3);
	4329	IrInstSrc *arg3_value = ir_gen_node(irb, arg3_node, scope);
	4330	if (arg3_value == irb->codegen->invalid_inst_src)
	4331	return arg3_value;
	4332
	4333	AstNode *arg4_node = node->data.fn_call_expr.params.at(4);
	4334	IrInstSrc *arg4_value = ir_gen_node(irb, arg4_node, scope);
	4335	if (arg4_value == irb->codegen->invalid_inst_src)
	4336	return arg4_value;
	4337
	4338	AstNode *arg5_node = node->data.fn_call_expr.params.at(5);
	4339	IrInstSrc *arg5_value = ir_gen_node(irb, arg5_node, scope);
	4340	if (arg5_value == irb->codegen->invalid_inst_src)
	4341	return arg5_value;
	4342
	4343	IrInstSrc *cmpxchg = ir_build_cmpxchg_src(irb, scope, node, arg0_value, arg1_value,
	4344	arg2_value, arg3_value, arg4_value, arg5_value, (builtin_fn->id == BuiltinFnIdCmpxchgWeak),
	4345	result_loc);
	4346	return ir_lval_wrap(irb, scope, cmpxchg, lval, result_loc);
	4347	}
	4348	case BuiltinFnIdFence:
	4349	{
	4350	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4351	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4352	if (arg0_value == irb->codegen->invalid_inst_src)
	4353	return arg0_value;
	4354
	4355	IrInstSrc *fence = ir_build_fence(irb, scope, node, arg0_value);
	4356	return ir_lval_wrap(irb, scope, fence, lval, result_loc);
	4357	}
	4358	case BuiltinFnIdReduce:
	4359	{
	4360	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4361	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4362	if (arg0_value == irb->codegen->invalid_inst_src)
	4363	return arg0_value;
	4364
	4365	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4366	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4367	if (arg1_value == irb->codegen->invalid_inst_src)
	4368	return arg1_value;
	4369
	4370	IrInstSrc *reduce = ir_build_reduce(irb, scope, node, arg0_value, arg1_value);
	4371	return ir_lval_wrap(irb, scope, reduce, lval, result_loc);
	4372	}
	4373	case BuiltinFnIdDivExact:
	4374	{
	4375	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4376	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4377	if (arg0_value == irb->codegen->invalid_inst_src)
	4378	return arg0_value;
	4379
	4380	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4381	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4382	if (arg1_value == irb->codegen->invalid_inst_src)
	4383	return arg1_value;
	4384
	4385	IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpDivExact, arg0_value, arg1_value, true);
	4386	return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
	4387	}
	4388	case BuiltinFnIdDivTrunc:
	4389	{
	4390	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4391	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4392	if (arg0_value == irb->codegen->invalid_inst_src)
	4393	return arg0_value;
	4394
	4395	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4396	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4397	if (arg1_value == irb->codegen->invalid_inst_src)
	4398	return arg1_value;
	4399
	4400	IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpDivTrunc, arg0_value, arg1_value, true);
	4401	return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
	4402	}
	4403	case BuiltinFnIdDivFloor:
	4404	{
	4405	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4406	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4407	if (arg0_value == irb->codegen->invalid_inst_src)
	4408	return arg0_value;
	4409
	4410	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4411	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4412	if (arg1_value == irb->codegen->invalid_inst_src)
	4413	return arg1_value;
	4414
	4415	IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpDivFloor, arg0_value, arg1_value, true);
	4416	return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
	4417	}
	4418	case BuiltinFnIdRem:
	4419	{
	4420	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4421	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4422	if (arg0_value == irb->codegen->invalid_inst_src)
	4423	return arg0_value;
	4424
	4425	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4426	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4427	if (arg1_value == irb->codegen->invalid_inst_src)
	4428	return arg1_value;
	4429
	4430	IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpRemRem, arg0_value, arg1_value, true);
	4431	return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
	4432	}
	4433	case BuiltinFnIdMod:
	4434	{
	4435	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4436	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4437	if (arg0_value == irb->codegen->invalid_inst_src)
	4438	return arg0_value;
	4439
	4440	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4441	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4442	if (arg1_value == irb->codegen->invalid_inst_src)
	4443	return arg1_value;
	4444
	4445	IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpRemMod, arg0_value, arg1_value, true);
	4446	return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
	4447	}
	4448	case BuiltinFnIdSqrt:
	4449	case BuiltinFnIdSin:
	4450	case BuiltinFnIdCos:
	4451	case BuiltinFnIdExp:
	4452	case BuiltinFnIdExp2:
	4453	case BuiltinFnIdLog:
	4454	case BuiltinFnIdLog2:
	4455	case BuiltinFnIdLog10:
	4456	case BuiltinFnIdFabs:
	4457	case BuiltinFnIdFloor:
	4458	case BuiltinFnIdCeil:
	4459	case BuiltinFnIdTrunc:
	4460	case BuiltinFnIdNearbyInt:
	4461	case BuiltinFnIdRound:
	4462	{
	4463	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4464	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4465	if (arg0_value == irb->codegen->invalid_inst_src)
	4466	return arg0_value;
	4467
	4468	IrInstSrc *inst = ir_build_float_op_src(irb, scope, node, arg0_value, builtin_fn->id);
	4469	return ir_lval_wrap(irb, scope, inst, lval, result_loc);
	4470	}
	4471	case BuiltinFnIdTruncate:
	4472	{
	4473	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4474	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4475	if (arg0_value == irb->codegen->invalid_inst_src)
	4476	return arg0_value;
	4477
	4478	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4479	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4480	if (arg1_value == irb->codegen->invalid_inst_src)
	4481	return arg1_value;
	4482
	4483	IrInstSrc *truncate = ir_build_truncate(irb, scope, node, arg0_value, arg1_value);
	4484	return ir_lval_wrap(irb, scope, truncate, lval, result_loc);
	4485	}
	4486	case BuiltinFnIdIntCast:
	4487	{
	4488	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4489	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4490	if (arg0_value == irb->codegen->invalid_inst_src)
	4491	return arg0_value;
	4492
	4493	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4494	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4495	if (arg1_value == irb->codegen->invalid_inst_src)
	4496	return arg1_value;
	4497
	4498	IrInstSrc *result = ir_build_int_cast(irb, scope, node, arg0_value, arg1_value);
	4499	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	4500	}
	4501	case BuiltinFnIdFloatCast:
	4502	{
	4503	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4504	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4505	if (arg0_value == irb->codegen->invalid_inst_src)
	4506	return arg0_value;
	4507
	4508	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4509	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4510	if (arg1_value == irb->codegen->invalid_inst_src)
	4511	return arg1_value;
	4512
	4513	IrInstSrc *result = ir_build_float_cast(irb, scope, node, arg0_value, arg1_value);
	4514	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	4515	}
	4516	case BuiltinFnIdErrSetCast:
	4517	{
	4518	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4519	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4520	if (arg0_value == irb->codegen->invalid_inst_src)
	4521	return arg0_value;
	4522
	4523	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4524	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4525	if (arg1_value == irb->codegen->invalid_inst_src)
	4526	return arg1_value;
	4527
	4528	IrInstSrc *result = ir_build_err_set_cast(irb, scope, node, arg0_value, arg1_value);
	4529	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	4530	}
	4531	case BuiltinFnIdIntToFloat:
	4532	{
	4533	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4534	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4535	if (arg0_value == irb->codegen->invalid_inst_src)
	4536	return arg0_value;
	4537
	4538	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4539	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4540	if (arg1_value == irb->codegen->invalid_inst_src)
	4541	return arg1_value;
	4542
	4543	IrInstSrc *result = ir_build_int_to_float(irb, scope, node, arg0_value, arg1_value);
	4544	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	4545	}
	4546	case BuiltinFnIdFloatToInt:
	4547	{
	4548	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4549	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4550	if (arg0_value == irb->codegen->invalid_inst_src)
	4551	return arg0_value;
	4552
	4553	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4554	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4555	if (arg1_value == irb->codegen->invalid_inst_src)
	4556	return arg1_value;
	4557
	4558	IrInstSrc *result = ir_build_float_to_int(irb, scope, node, arg0_value, arg1_value);
	4559	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	4560	}
	4561	case BuiltinFnIdErrToInt:
	4562	{
	4563	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4564	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4565	if (arg0_value == irb->codegen->invalid_inst_src)
	4566	return arg0_value;
	4567
	4568	IrInstSrc *result = ir_build_err_to_int_src(irb, scope, node, arg0_value);
	4569	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	4570	}
	4571	case BuiltinFnIdIntToErr:
	4572	{
	4573	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4574	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4575	if (arg0_value == irb->codegen->invalid_inst_src)
	4576	return arg0_value;
	4577
	4578	IrInstSrc *result = ir_build_int_to_err_src(irb, scope, node, arg0_value);
	4579	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	4580	}
	4581	case BuiltinFnIdBoolToInt:
	4582	{
	4583	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4584	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4585	if (arg0_value == irb->codegen->invalid_inst_src)
	4586	return arg0_value;
	4587
	4588	IrInstSrc *result = ir_build_bool_to_int(irb, scope, node, arg0_value);
	4589	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	4590	}
	4591	case BuiltinFnIdVectorType:
	4592	{
	4593	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4594	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4595	if (arg0_value == irb->codegen->invalid_inst_src)
	4596	return arg0_value;
	4597
	4598	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4599	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4600	if (arg1_value == irb->codegen->invalid_inst_src)
	4601	return arg1_value;
	4602
	4603	IrInstSrc *vector_type = ir_build_vector_type(irb, scope, node, arg0_value, arg1_value);
	4604	return ir_lval_wrap(irb, scope, vector_type, lval, result_loc);
	4605	}
	4606	case BuiltinFnIdShuffle:
	4607	{
	4608	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4609	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4610	if (arg0_value == irb->codegen->invalid_inst_src)
	4611	return arg0_value;
	4612
	4613	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4614	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4615	if (arg1_value == irb->codegen->invalid_inst_src)
	4616	return arg1_value;
	4617
	4618	AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
	4619	IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
	4620	if (arg2_value == irb->codegen->invalid_inst_src)
	4621	return arg2_value;
	4622
	4623	AstNode *arg3_node = node->data.fn_call_expr.params.at(3);
	4624	IrInstSrc *arg3_value = ir_gen_node(irb, arg3_node, scope);
	4625	if (arg3_value == irb->codegen->invalid_inst_src)
	4626	return arg3_value;
	4627
	4628	IrInstSrc *shuffle_vector = ir_build_shuffle_vector(irb, scope, node,
	4629	arg0_value, arg1_value, arg2_value, arg3_value);
	4630	return ir_lval_wrap(irb, scope, shuffle_vector, lval, result_loc);
	4631	}
	4632	case BuiltinFnIdSplat:
	4633	{
	4634	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4635	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4636	if (arg0_value == irb->codegen->invalid_inst_src)
	4637	return arg0_value;
	4638
	4639	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4640	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4641	if (arg1_value == irb->codegen->invalid_inst_src)
	4642	return arg1_value;
	4643
	4644	IrInstSrc *splat = ir_build_splat_src(irb, scope, node,
	4645	arg0_value, arg1_value);
	4646	return ir_lval_wrap(irb, scope, splat, lval, result_loc);
	4647	}
	4648	case BuiltinFnIdMemcpy:
	4649	{
	4650	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4651	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4652	if (arg0_value == irb->codegen->invalid_inst_src)
	4653	return arg0_value;
	4654
	4655	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4656	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4657	if (arg1_value == irb->codegen->invalid_inst_src)
	4658	return arg1_value;
	4659
	4660	AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
	4661	IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
	4662	if (arg2_value == irb->codegen->invalid_inst_src)
	4663	return arg2_value;
	4664
	4665	IrInstSrc *ir_memcpy = ir_build_memcpy_src(irb, scope, node, arg0_value, arg1_value, arg2_value);
	4666	return ir_lval_wrap(irb, scope, ir_memcpy, lval, result_loc);
	4667	}
	4668	case BuiltinFnIdMemset:
	4669	{
	4670	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4671	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4672	if (arg0_value == irb->codegen->invalid_inst_src)
	4673	return arg0_value;
	4674
	4675	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4676	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4677	if (arg1_value == irb->codegen->invalid_inst_src)
	4678	return arg1_value;
	4679
	4680	AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
	4681	IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
	4682	if (arg2_value == irb->codegen->invalid_inst_src)
	4683	return arg2_value;
	4684
	4685	IrInstSrc *ir_memset = ir_build_memset_src(irb, scope, node, arg0_value, arg1_value, arg2_value);
	4686	return ir_lval_wrap(irb, scope, ir_memset, lval, result_loc);
	4687	}
	4688	case BuiltinFnIdWasmMemorySize:
	4689	{
	4690	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4691	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4692	if (arg0_value == irb->codegen->invalid_inst_src)
	4693	return arg0_value;
	4694
	4695	IrInstSrc *ir_wasm_memory_size = ir_build_wasm_memory_size_src(irb, scope, node, arg0_value);
	4696	return ir_lval_wrap(irb, scope, ir_wasm_memory_size, lval, result_loc);
	4697	}
	4698	case BuiltinFnIdWasmMemoryGrow:
	4699	{
	4700	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4701	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4702	if (arg0_value == irb->codegen->invalid_inst_src)
	4703	return arg0_value;
	4704
	4705	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4706	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4707	if (arg1_value == irb->codegen->invalid_inst_src)
	4708	return arg1_value;
	4709
	4710	IrInstSrc *ir_wasm_memory_grow = ir_build_wasm_memory_grow_src(irb, scope, node, arg0_value, arg1_value);
	4711	return ir_lval_wrap(irb, scope, ir_wasm_memory_grow, lval, result_loc);
	4712	}
	4713	case BuiltinFnIdField:
	4714	{
	4715	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4716	IrInstSrc *arg0_value = ir_gen_node_extra(irb, arg0_node, scope, LValPtr, nullptr);
	4717	if (arg0_value == irb->codegen->invalid_inst_src)
	4718	return arg0_value;
	4719
	4720	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4721	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4722	if (arg1_value == irb->codegen->invalid_inst_src)
	4723	return arg1_value;
	4724
	4725	IrInstSrc *ptr_instruction = ir_build_field_ptr_instruction(irb, scope, node,
	4726	arg0_value, arg1_value, false);
	4727
	4728	if (lval == LValPtr \|\| lval == LValAssign)
	4729	return ptr_instruction;
	4730
	4731	IrInstSrc *load_ptr = ir_build_load_ptr(irb, scope, node, ptr_instruction);
	4732	return ir_expr_wrap(irb, scope, load_ptr, result_loc);
	4733	}
	4734	case BuiltinFnIdHasField:
	4735	{
	4736	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4737	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4738	if (arg0_value == irb->codegen->invalid_inst_src)
	4739	return arg0_value;
	4740
	4741	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4742	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4743	if (arg1_value == irb->codegen->invalid_inst_src)
	4744	return arg1_value;
	4745
	4746	IrInstSrc *type_info = ir_build_has_field(irb, scope, node, arg0_value, arg1_value);
	4747	return ir_lval_wrap(irb, scope, type_info, lval, result_loc);
	4748	}
	4749	case BuiltinFnIdTypeInfo:
	4750	{
	4751	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4752	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4753	if (arg0_value == irb->codegen->invalid_inst_src)
	4754	return arg0_value;
	4755
	4756	IrInstSrc *type_info = ir_build_type_info(irb, scope, node, arg0_value);
	4757	return ir_lval_wrap(irb, scope, type_info, lval, result_loc);
	4758	}
	4759	case BuiltinFnIdType:
	4760	{
	4761	AstNode *arg_node = node->data.fn_call_expr.params.at(0);
	4762	IrInstSrc *arg = ir_gen_node(irb, arg_node, scope);
	4763	if (arg == irb->codegen->invalid_inst_src)
	4764	return arg;
	4765
	4766	IrInstSrc *type = ir_build_type(irb, scope, node, arg);
	4767	return ir_lval_wrap(irb, scope, type, lval, result_loc);
	4768	}
	4769	case BuiltinFnIdBreakpoint:
	4770	return ir_lval_wrap(irb, scope, ir_build_breakpoint(irb, scope, node), lval, result_loc);
	4771	case BuiltinFnIdReturnAddress:
	4772	return ir_lval_wrap(irb, scope, ir_build_return_address_src(irb, scope, node), lval, result_loc);
	4773	case BuiltinFnIdFrameAddress:
	4774	return ir_lval_wrap(irb, scope, ir_build_frame_address_src(irb, scope, node), lval, result_loc);
	4775	case BuiltinFnIdFrameHandle:
	4776	if (!irb->exec->fn_entry) {
	4777	add_node_error(irb->codegen, node, buf_sprintf("@frame() called outside of function definition"));
	4778	return irb->codegen->invalid_inst_src;
	4779	}
	4780	return ir_lval_wrap(irb, scope, ir_build_handle_src(irb, scope, node), lval, result_loc);
	4781	case BuiltinFnIdFrameType: {
	4782	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4783	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4784	if (arg0_value == irb->codegen->invalid_inst_src)
	4785	return arg0_value;
	4786
	4787	IrInstSrc *frame_type = ir_build_frame_type(irb, scope, node, arg0_value);
	4788	return ir_lval_wrap(irb, scope, frame_type, lval, result_loc);
	4789	}
	4790	case BuiltinFnIdFrameSize: {
	4791	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4792	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4793	if (arg0_value == irb->codegen->invalid_inst_src)
	4794	return arg0_value;
	4795
	4796	IrInstSrc *frame_size = ir_build_frame_size_src(irb, scope, node, arg0_value);
	4797	return ir_lval_wrap(irb, scope, frame_size, lval, result_loc);
	4798	}
	4799	case BuiltinFnIdAlignOf:
	4800	{
	4801	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4802	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4803	if (arg0_value == irb->codegen->invalid_inst_src)
	4804	return arg0_value;
	4805
	4806	IrInstSrc *align_of = ir_build_align_of(irb, scope, node, arg0_value);
	4807	return ir_lval_wrap(irb, scope, align_of, lval, result_loc);
	4808	}
	4809	case BuiltinFnIdAddWithOverflow:
	4810	return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpAdd), lval, result_loc);
	4811	case BuiltinFnIdSubWithOverflow:
	4812	return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpSub), lval, result_loc);
	4813	case BuiltinFnIdMulWithOverflow:
	4814	return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpMul), lval, result_loc);
	4815	case BuiltinFnIdShlWithOverflow:
	4816	return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpShl), lval, result_loc);
	4817	case BuiltinFnIdMulAdd:
	4818	return ir_lval_wrap(irb, scope, ir_gen_mul_add(irb, scope, node), lval, result_loc);
	4819	case BuiltinFnIdTypeName:
	4820	{
	4821	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4822	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4823	if (arg0_value == irb->codegen->invalid_inst_src)
	4824	return arg0_value;
	4825
	4826	IrInstSrc *type_name = ir_build_type_name(irb, scope, node, arg0_value);
	4827	return ir_lval_wrap(irb, scope, type_name, lval, result_loc);
	4828	}
	4829	case BuiltinFnIdPanic:
	4830	{
	4831	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4832	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4833	if (arg0_value == irb->codegen->invalid_inst_src)
	4834	return arg0_value;
	4835
	4836	IrInstSrc *panic = ir_build_panic_src(irb, scope, node, arg0_value);
	4837	return ir_lval_wrap(irb, scope, panic, lval, result_loc);
	4838	}
	4839	case BuiltinFnIdPtrCast:
	4840	{
	4841	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4842	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4843	if (arg0_value == irb->codegen->invalid_inst_src)
	4844	return arg0_value;
	4845
	4846	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4847	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4848	if (arg1_value == irb->codegen->invalid_inst_src)
	4849	return arg1_value;
	4850
	4851	IrInstSrc *ptr_cast = ir_build_ptr_cast_src(irb, scope, node, arg0_value, arg1_value, true);
	4852	return ir_lval_wrap(irb, scope, ptr_cast, lval, result_loc);
	4853	}
	4854	case BuiltinFnIdBitCast:
	4855	{
	4856	AstNode *dest_type_node = node->data.fn_call_expr.params.at(0);
	4857	IrInstSrc *dest_type = ir_gen_node(irb, dest_type_node, scope);
	4858	if (dest_type == irb->codegen->invalid_inst_src)
	4859	return dest_type;
	4860
	4861	ResultLocBitCast *result_loc_bit_cast = heap::c_allocator.create<ResultLocBitCast>();
	4862	result_loc_bit_cast->base.id = ResultLocIdBitCast;
	4863	result_loc_bit_cast->base.source_instruction = dest_type;
	4864	result_loc_bit_cast->base.allow_write_through_const = result_loc->allow_write_through_const;
	4865	ir_ref_instruction(dest_type, irb->current_basic_block);
	4866	result_loc_bit_cast->parent = result_loc;
	4867
	4868	ir_build_reset_result(irb, scope, node, &result_loc_bit_cast->base);
	4869
	4870	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4871	IrInstSrc *arg1_value = ir_gen_node_extra(irb, arg1_node, scope, LValNone,
	4872	&result_loc_bit_cast->base);
	4873	if (arg1_value == irb->codegen->invalid_inst_src)
	4874	return arg1_value;
	4875
	4876	IrInstSrc *bitcast = ir_build_bit_cast_src(irb, scope, arg1_node, arg1_value, result_loc_bit_cast);
	4877	return ir_lval_wrap(irb, scope, bitcast, lval, result_loc);
	4878	}
	4879	case BuiltinFnIdAs:
	4880	{
	4881	AstNode *dest_type_node = node->data.fn_call_expr.params.at(0);
	4882	IrInstSrc *dest_type = ir_gen_node(irb, dest_type_node, scope);
	4883	if (dest_type == irb->codegen->invalid_inst_src)
	4884	return dest_type;
	4885
	4886	ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, dest_type, result_loc);
	4887
	4888	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4889	IrInstSrc *arg1_value = ir_gen_node_extra(irb, arg1_node, scope, LValNone,
	4890	&result_loc_cast->base);
	4891	if (arg1_value == irb->codegen->invalid_inst_src)
	4892	return arg1_value;
	4893
	4894	IrInstSrc *result = ir_build_implicit_cast(irb, scope, node, arg1_value, result_loc_cast);
	4895	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	4896	}
	4897	case BuiltinFnIdIntToPtr:
	4898	{
	4899	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4900	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4901	if (arg0_value == irb->codegen->invalid_inst_src)
	4902	return arg0_value;
	4903
	4904	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4905	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4906	if (arg1_value == irb->codegen->invalid_inst_src)
	4907	return arg1_value;
	4908
	4909	IrInstSrc *int_to_ptr = ir_build_int_to_ptr_src(irb, scope, node, arg0_value, arg1_value);
	4910	return ir_lval_wrap(irb, scope, int_to_ptr, lval, result_loc);
	4911	}
	4912	case BuiltinFnIdPtrToInt:
	4913	{
	4914	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4915	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4916	if (arg0_value == irb->codegen->invalid_inst_src)
	4917	return arg0_value;
	4918
	4919	IrInstSrc *ptr_to_int = ir_build_ptr_to_int_src(irb, scope, node, arg0_value);
	4920	return ir_lval_wrap(irb, scope, ptr_to_int, lval, result_loc);
	4921	}
	4922	case BuiltinFnIdTagName:
	4923	{
	4924	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4925	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4926	if (arg0_value == irb->codegen->invalid_inst_src)
	4927	return arg0_value;
	4928
	4929	IrInstSrc *tag_name = ir_build_tag_name_src(irb, scope, node, arg0_value);
	4930	return ir_lval_wrap(irb, scope, tag_name, lval, result_loc);
	4931	}
	4932	case BuiltinFnIdFieldParentPtr:
	4933	{
	4934	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4935	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4936	if (arg0_value == irb->codegen->invalid_inst_src)
	4937	return arg0_value;
	4938
	4939	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4940	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4941	if (arg1_value == irb->codegen->invalid_inst_src)
	4942	return arg1_value;
	4943
	4944	AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
	4945	IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
	4946	if (arg2_value == irb->codegen->invalid_inst_src)
	4947	return arg2_value;
	4948
	4949	IrInstSrc *field_parent_ptr = ir_build_field_parent_ptr_src(irb, scope, node,
	4950	arg0_value, arg1_value, arg2_value);
	4951	return ir_lval_wrap(irb, scope, field_parent_ptr, lval, result_loc);
	4952	}
	4953	case BuiltinFnIdByteOffsetOf:
	4954	{
	4955	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4956	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4957	if (arg0_value == irb->codegen->invalid_inst_src)
	4958	return arg0_value;
	4959
	4960	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4961	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4962	if (arg1_value == irb->codegen->invalid_inst_src)
	4963	return arg1_value;
	4964
	4965	IrInstSrc *offset_of = ir_build_byte_offset_of(irb, scope, node, arg0_value, arg1_value);
	4966	return ir_lval_wrap(irb, scope, offset_of, lval, result_loc);
	4967	}
	4968	case BuiltinFnIdBitOffsetOf:
	4969	{
	4970	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	4971	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	4972	if (arg0_value == irb->codegen->invalid_inst_src)
	4973	return arg0_value;
	4974
	4975	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	4976	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	4977	if (arg1_value == irb->codegen->invalid_inst_src)
	4978	return arg1_value;
	4979
	4980	IrInstSrc *offset_of = ir_build_bit_offset_of(irb, scope, node, arg0_value, arg1_value);
	4981	return ir_lval_wrap(irb, scope, offset_of, lval, result_loc);
	4982	}
	4983	case BuiltinFnIdCall: {
	4984	// Cast the options parameter to the options type
	4985	ZigType *options_type = get_builtin_type(irb->codegen, "CallOptions");
	4986	IrInstSrc *options_type_inst = ir_build_const_type(irb, scope, node, options_type);
	4987	ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, options_type_inst, no_result_loc());
	4988
	4989	AstNode *options_node = node->data.fn_call_expr.params.at(0);
	4990	IrInstSrc *options_inner = ir_gen_node_extra(irb, options_node, scope,
	4991	LValNone, &result_loc_cast->base);
	4992	if (options_inner == irb->codegen->invalid_inst_src)
	4993	return options_inner;
	4994	IrInstSrc *options = ir_build_implicit_cast(irb, scope, options_node, options_inner, result_loc_cast);
	4995
	4996	AstNode *fn_ref_node = node->data.fn_call_expr.params.at(1);
	4997	AstNode *args_node = node->data.fn_call_expr.params.at(2);
	4998	if (args_node->type == NodeTypeContainerInitExpr) {
	4999	if (args_node->data.container_init_expr.kind == ContainerInitKindArray \|\|
	5000	args_node->data.container_init_expr.entries.length == 0)
	5001	{
	5002	return ir_gen_fn_call_with_args(irb, scope, node,
	5003	fn_ref_node, CallModifierNone, options,
	5004	args_node->data.container_init_expr.entries.items,
	5005	args_node->data.container_init_expr.entries.length,
	5006	lval, result_loc);
	5007	} else {
	5008	exec_add_error_node(irb->codegen, irb->exec, args_node,
	5009	buf_sprintf("TODO: @call with anon struct literal"));
	5010	return irb->codegen->invalid_inst_src;
	5011	}
	5012	} else {
	5013	IrInstSrc *fn_ref = ir_gen_node(irb, fn_ref_node, scope);
	5014	if (fn_ref == irb->codegen->invalid_inst_src)
	5015	return fn_ref;
	5016
	5017	IrInstSrc *args = ir_gen_node(irb, args_node, scope);
	5018	if (args == irb->codegen->invalid_inst_src)
	5019	return args;
	5020
	5021	IrInstSrc *call = ir_build_call_extra(irb, scope, node, options, fn_ref, args, result_loc);
	5022	return ir_lval_wrap(irb, scope, call, lval, result_loc);
	5023	}
	5024	}
	5025	case BuiltinFnIdAsyncCall:
	5026	return ir_gen_async_call(irb, scope, nullptr, node, lval, result_loc);
	5027	case BuiltinFnIdShlExact:
	5028	{
	5029	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5030	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5031	if (arg0_value == irb->codegen->invalid_inst_src)
	5032	return arg0_value;
	5033
	5034	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	5035	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	5036	if (arg1_value == irb->codegen->invalid_inst_src)
	5037	return arg1_value;
	5038
	5039	IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpBitShiftLeftExact, arg0_value, arg1_value, true);
	5040	return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
	5041	}
	5042	case BuiltinFnIdShrExact:
	5043	{
	5044	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5045	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5046	if (arg0_value == irb->codegen->invalid_inst_src)
	5047	return arg0_value;
	5048
	5049	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	5050	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	5051	if (arg1_value == irb->codegen->invalid_inst_src)
	5052	return arg1_value;
	5053
	5054	IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpBitShiftRightExact, arg0_value, arg1_value, true);
	5055	return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
	5056	}
	5057	case BuiltinFnIdSetEvalBranchQuota:
	5058	{
	5059	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5060	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5061	if (arg0_value == irb->codegen->invalid_inst_src)
	5062	return arg0_value;
	5063
	5064	IrInstSrc *set_eval_branch_quota = ir_build_set_eval_branch_quota(irb, scope, node, arg0_value);
	5065	return ir_lval_wrap(irb, scope, set_eval_branch_quota, lval, result_loc);
	5066	}
	5067	case BuiltinFnIdAlignCast:
	5068	{
	5069	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5070	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5071	if (arg0_value == irb->codegen->invalid_inst_src)
	5072	return arg0_value;
	5073
	5074	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	5075	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	5076	if (arg1_value == irb->codegen->invalid_inst_src)
	5077	return arg1_value;
	5078
	5079	IrInstSrc *align_cast = ir_build_align_cast_src(irb, scope, node, arg0_value, arg1_value);
	5080	return ir_lval_wrap(irb, scope, align_cast, lval, result_loc);
	5081	}
	5082	case BuiltinFnIdThis:
	5083	{
	5084	IrInstSrc *this_inst = ir_gen_this(irb, scope, node);
	5085	return ir_lval_wrap(irb, scope, this_inst, lval, result_loc);
	5086	}
	5087	case BuiltinFnIdSetAlignStack:
	5088	{
	5089	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5090	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5091	if (arg0_value == irb->codegen->invalid_inst_src)
	5092	return arg0_value;
	5093
	5094	IrInstSrc *set_align_stack = ir_build_set_align_stack(irb, scope, node, arg0_value);
	5095	return ir_lval_wrap(irb, scope, set_align_stack, lval, result_loc);
	5096	}
	5097	case BuiltinFnIdExport:
	5098	{
	5099	// Cast the options parameter to the options type
	5100	ZigType *options_type = get_builtin_type(irb->codegen, "ExportOptions");
	5101	IrInstSrc *options_type_inst = ir_build_const_type(irb, scope, node, options_type);
	5102	ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, options_type_inst, no_result_loc());
	5103
	5104	AstNode *target_node = node->data.fn_call_expr.params.at(0);
	5105	IrInstSrc *target_value = ir_gen_node(irb, target_node, scope);
	5106	if (target_value == irb->codegen->invalid_inst_src)
	5107	return target_value;
	5108
	5109	AstNode *options_node = node->data.fn_call_expr.params.at(1);
	5110	IrInstSrc *options_value = ir_gen_node_extra(irb, options_node,
	5111	scope, LValNone, &result_loc_cast->base);
	5112	if (options_value == irb->codegen->invalid_inst_src)
	5113	return options_value;
	5114
	5115	IrInstSrc *casted_options_value = ir_build_implicit_cast(
	5116	irb, scope, options_node, options_value, result_loc_cast);
	5117
	5118	IrInstSrc *ir_export = ir_build_export(irb, scope, node, target_value, casted_options_value);
	5119	return ir_lval_wrap(irb, scope, ir_export, lval, result_loc);
	5120	}
	5121	case BuiltinFnIdExtern:
	5122	{
	5123	// Cast the options parameter to the options type
	5124	ZigType *options_type = get_builtin_type(irb->codegen, "ExternOptions");
	5125	IrInstSrc *options_type_inst = ir_build_const_type(irb, scope, node, options_type);
	5126	ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, options_type_inst, no_result_loc());
	5127
	5128	AstNode *type_node = node->data.fn_call_expr.params.at(0);
	5129	IrInstSrc *type_value = ir_gen_node(irb, type_node, scope);
	5130	if (type_value == irb->codegen->invalid_inst_src)
	5131	return type_value;
	5132
	5133	AstNode *options_node = node->data.fn_call_expr.params.at(1);
	5134	IrInstSrc *options_value = ir_gen_node_extra(irb, options_node,
	5135	scope, LValNone, &result_loc_cast->base);
	5136	if (options_value == irb->codegen->invalid_inst_src)
	5137	return options_value;
	5138
	5139	IrInstSrc *casted_options_value = ir_build_implicit_cast(
	5140	irb, scope, options_node, options_value, result_loc_cast);
	5141
	5142	IrInstSrc *ir_extern = ir_build_extern(irb, scope, node, type_value, casted_options_value);
	5143	return ir_lval_wrap(irb, scope, ir_extern, lval, result_loc);
	5144	}
	5145	case BuiltinFnIdErrorReturnTrace:
	5146	{
	5147	IrInstSrc *error_return_trace = ir_build_error_return_trace_src(irb, scope, node,
	5148	IrInstErrorReturnTraceNull);
	5149	return ir_lval_wrap(irb, scope, error_return_trace, lval, result_loc);
	5150	}
	5151	case BuiltinFnIdAtomicRmw:
	5152	{
	5153	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5154	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5155	if (arg0_value == irb->codegen->invalid_inst_src)
	5156	return arg0_value;
	5157
	5158	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	5159	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	5160	if (arg1_value == irb->codegen->invalid_inst_src)
	5161	return arg1_value;
	5162
	5163	AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
	5164	IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
	5165	if (arg2_value == irb->codegen->invalid_inst_src)
	5166	return arg2_value;
	5167
	5168	AstNode *arg3_node = node->data.fn_call_expr.params.at(3);
	5169	IrInstSrc *arg3_value = ir_gen_node(irb, arg3_node, scope);
	5170	if (arg3_value == irb->codegen->invalid_inst_src)
	5171	return arg3_value;
	5172
	5173	AstNode *arg4_node = node->data.fn_call_expr.params.at(4);
	5174	IrInstSrc *arg4_value = ir_gen_node(irb, arg4_node, scope);
	5175	if (arg4_value == irb->codegen->invalid_inst_src)
	5176	return arg4_value;
	5177
	5178	IrInstSrc *inst = ir_build_atomic_rmw_src(irb, scope, node,
	5179	arg0_value, arg1_value, arg2_value, arg3_value, arg4_value);
	5180	return ir_lval_wrap(irb, scope, inst, lval, result_loc);
	5181	}
	5182	case BuiltinFnIdAtomicLoad:
	5183	{
	5184	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5185	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5186	if (arg0_value == irb->codegen->invalid_inst_src)
	5187	return arg0_value;
	5188
	5189	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	5190	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	5191	if (arg1_value == irb->codegen->invalid_inst_src)
	5192	return arg1_value;
	5193
	5194	AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
	5195	IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
	5196	if (arg2_value == irb->codegen->invalid_inst_src)
	5197	return arg2_value;
	5198
	5199	IrInstSrc *inst = ir_build_atomic_load_src(irb, scope, node, arg0_value, arg1_value, arg2_value);
	5200	return ir_lval_wrap(irb, scope, inst, lval, result_loc);
	5201	}
	5202	case BuiltinFnIdAtomicStore:
	5203	{
	5204	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5205	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5206	if (arg0_value == irb->codegen->invalid_inst_src)
	5207	return arg0_value;
	5208
	5209	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	5210	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	5211	if (arg1_value == irb->codegen->invalid_inst_src)
	5212	return arg1_value;
	5213
	5214	AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
	5215	IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
	5216	if (arg2_value == irb->codegen->invalid_inst_src)
	5217	return arg2_value;
	5218
	5219	AstNode *arg3_node = node->data.fn_call_expr.params.at(3);
	5220	IrInstSrc *arg3_value = ir_gen_node(irb, arg3_node, scope);
	5221	if (arg3_value == irb->codegen->invalid_inst_src)
	5222	return arg3_value;
	5223
	5224	IrInstSrc *inst = ir_build_atomic_store_src(irb, scope, node, arg0_value, arg1_value,
	5225	arg2_value, arg3_value);
	5226	return ir_lval_wrap(irb, scope, inst, lval, result_loc);
	5227	}
	5228	case BuiltinFnIdIntToEnum:
	5229	{
	5230	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5231	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5232	if (arg0_value == irb->codegen->invalid_inst_src)
	5233	return arg0_value;
	5234
	5235	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	5236	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	5237	if (arg1_value == irb->codegen->invalid_inst_src)
	5238	return arg1_value;
	5239
	5240	IrInstSrc *result = ir_build_int_to_enum_src(irb, scope, node, arg0_value, arg1_value);
	5241	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	5242	}
	5243	case BuiltinFnIdEnumToInt:
	5244	{
	5245	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5246	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5247	if (arg0_value == irb->codegen->invalid_inst_src)
	5248	return arg0_value;
	5249
	5250	IrInstSrc *result = ir_build_enum_to_int(irb, scope, node, arg0_value);
	5251	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	5252	}
	5253	case BuiltinFnIdCtz:
	5254	case BuiltinFnIdPopCount:
	5255	case BuiltinFnIdClz:
	5256	case BuiltinFnIdBswap:
	5257	case BuiltinFnIdBitReverse:
	5258	{
	5259	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5260	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5261	if (arg0_value == irb->codegen->invalid_inst_src)
	5262	return arg0_value;
	5263
	5264	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	5265	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	5266	if (arg1_value == irb->codegen->invalid_inst_src)
	5267	return arg1_value;
	5268
	5269	IrInstSrc *result;
	5270	switch (builtin_fn->id) {
	5271	case BuiltinFnIdCtz:
	5272	result = ir_build_ctz(irb, scope, node, arg0_value, arg1_value);
	5273	break;
	5274	case BuiltinFnIdPopCount:
	5275	result = ir_build_pop_count(irb, scope, node, arg0_value, arg1_value);
	5276	break;
	5277	case BuiltinFnIdClz:
	5278	result = ir_build_clz(irb, scope, node, arg0_value, arg1_value);
	5279	break;
	5280	case BuiltinFnIdBswap:
	5281	result = ir_build_bswap(irb, scope, node, arg0_value, arg1_value);
	5282	break;
	5283	case BuiltinFnIdBitReverse:
	5284	result = ir_build_bit_reverse(irb, scope, node, arg0_value, arg1_value);
	5285	break;
	5286	default:
	5287	zig_unreachable();
	5288	}
	5289	return ir_lval_wrap(irb, scope, result, lval, result_loc);
	5290	}
	5291	case BuiltinFnIdHasDecl:
	5292	{
	5293	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5294	IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
	5295	if (arg0_value == irb->codegen->invalid_inst_src)
	5296	return arg0_value;
	5297
	5298	AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
	5299	IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
	5300	if (arg1_value == irb->codegen->invalid_inst_src)
	5301	return arg1_value;
	5302
	5303	IrInstSrc *has_decl = ir_build_has_decl(irb, scope, node, arg0_value, arg1_value);
	5304	return ir_lval_wrap(irb, scope, has_decl, lval, result_loc);
	5305	}
	5306	case BuiltinFnIdUnionInit:
	5307	{
	5308	AstNode *union_type_node = node->data.fn_call_expr.params.at(0);
	5309	IrInstSrc *union_type_inst = ir_gen_node(irb, union_type_node, scope);
	5310	if (union_type_inst == irb->codegen->invalid_inst_src)
	5311	return union_type_inst;
	5312
	5313	AstNode *name_node = node->data.fn_call_expr.params.at(1);
	5314	IrInstSrc *name_inst = ir_gen_node(irb, name_node, scope);
	5315	if (name_inst == irb->codegen->invalid_inst_src)
	5316	return name_inst;
	5317
	5318	AstNode *init_node = node->data.fn_call_expr.params.at(2);
	5319
	5320	return ir_gen_union_init_expr(irb, scope, node, union_type_inst, name_inst, init_node,
	5321	lval, result_loc);
	5322	}
	5323	case BuiltinFnIdSrc:
	5324	{
	5325	IrInstSrc *src_inst = ir_build_src(irb, scope, node);
	5326	return ir_lval_wrap(irb, scope, src_inst, lval, result_loc);
	5327	}
	5328	}
	5329	zig_unreachable();
	5330	}
	5331
	5332	static ScopeNoSuspend get_scope_nosuspend(Scope scope) {
	5333	while (scope) {
	5334	if (scope->id == ScopeIdNoSuspend)
	5335	return (ScopeNoSuspend *)scope;
	5336	if (scope->id == ScopeIdFnDef)
	5337	return nullptr;
	5338
	5339	scope = scope->parent;
	5340	}
	5341	return nullptr;
	5342	}
	5343
	5344	static IrInstSrc ir_gen_fn_call(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
	5345	ResultLoc *result_loc)
	5346	{
	5347	assert(node->type == NodeTypeFnCallExpr);
	5348
	5349	if (node->data.fn_call_expr.modifier == CallModifierBuiltin)
	5350	return ir_gen_builtin_fn_call(irb, scope, node, lval, result_loc);
	5351
	5352	bool is_nosuspend = get_scope_nosuspend(scope) != nullptr;
	5353	CallModifier modifier = node->data.fn_call_expr.modifier;
	5354	if (is_nosuspend && modifier != CallModifierAsync) {
	5355	modifier = CallModifierNoSuspend;
	5356	}
	5357
	5358	AstNode *fn_ref_node = node->data.fn_call_expr.fn_ref_expr;
	5359	return ir_gen_fn_call_with_args(irb, scope, node, fn_ref_node, modifier,
	5360	nullptr, node->data.fn_call_expr.params.items, node->data.fn_call_expr.params.length, lval, result_loc);
	5361	}
	5362
	5363	static IrInstSrc ir_gen_if_bool_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
	5364	ResultLoc *result_loc)
	5365	{
	5366	assert(node->type == NodeTypeIfBoolExpr);
	5367
	5368	IrInstSrc *condition = ir_gen_node(irb, node->data.if_bool_expr.condition, scope);
	5369	if (condition == irb->codegen->invalid_inst_src)
	5370	return irb->codegen->invalid_inst_src;
	5371
	5372	IrInstSrc *is_comptime;
	5373	if (ir_should_inline(irb->exec, scope)) {
	5374	is_comptime = ir_build_const_bool(irb, scope, node, true);
	5375	} else {
	5376	is_comptime = ir_build_test_comptime(irb, scope, node, condition);
	5377	}
	5378
	5379	AstNode *then_node = node->data.if_bool_expr.then_block;
	5380	AstNode *else_node = node->data.if_bool_expr.else_node;
	5381
	5382	IrBasicBlockSrc *then_block = ir_create_basic_block(irb, scope, "Then");
	5383	IrBasicBlockSrc *else_block = ir_create_basic_block(irb, scope, "Else");
	5384	IrBasicBlockSrc *endif_block = ir_create_basic_block(irb, scope, "EndIf");
	5385
	5386	IrInstSrc *cond_br_inst = ir_build_cond_br(irb, scope, node, condition,
	5387	then_block, else_block, is_comptime);
	5388	ResultLocPeerParent *peer_parent = ir_build_binary_result_peers(irb, cond_br_inst, else_block, endif_block,
	5389	result_loc, is_comptime);
	5390
	5391	ir_set_cursor_at_end_and_append_block(irb, then_block);
	5392
	5393	Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
	5394	IrInstSrc *then_expr_result = ir_gen_node_extra(irb, then_node, subexpr_scope, lval,
	5395	&peer_parent->peers.at(0)->base);
	5396	if (then_expr_result == irb->codegen->invalid_inst_src)
	5397	return irb->codegen->invalid_inst_src;
	5398	IrBasicBlockSrc *after_then_block = irb->current_basic_block;
	5399	if (!instr_is_unreachable(then_expr_result))
	5400	ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
	5401
	5402	ir_set_cursor_at_end_and_append_block(irb, else_block);
	5403	IrInstSrc *else_expr_result;
	5404	if (else_node) {
	5405	else_expr_result = ir_gen_node_extra(irb, else_node, subexpr_scope, lval, &peer_parent->peers.at(1)->base);
	5406	if (else_expr_result == irb->codegen->invalid_inst_src)
	5407	return irb->codegen->invalid_inst_src;
	5408	} else {
	5409	else_expr_result = ir_build_const_void(irb, scope, node);
	5410	ir_build_end_expr(irb, scope, node, else_expr_result, &peer_parent->peers.at(1)->base);
	5411	}
	5412	IrBasicBlockSrc *after_else_block = irb->current_basic_block;
	5413	if (!instr_is_unreachable(else_expr_result))
	5414	ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
	5415
	5416	ir_set_cursor_at_end_and_append_block(irb, endif_block);
	5417	IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
	5418	incoming_values[0] = then_expr_result;
	5419	incoming_values[1] = else_expr_result;
	5420	IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
	5421	incoming_blocks[0] = after_then_block;
	5422	incoming_blocks[1] = after_else_block;
	5423
	5424	IrInstSrc *phi = ir_build_phi(irb, scope, node, 2, incoming_blocks, incoming_values, peer_parent);
	5425	return ir_expr_wrap(irb, scope, phi, result_loc);
	5426	}
	5427
	5428	static IrInstSrc ir_gen_prefix_op_id_lval(IrBuilderSrc irb, Scope scope, AstNode node, IrUnOp op_id, LVal lval) {
	5429	assert(node->type == NodeTypePrefixOpExpr);
	5430	AstNode *expr_node = node->data.prefix_op_expr.primary_expr;
	5431
	5432	IrInstSrc *value = ir_gen_node_extra(irb, expr_node, scope, lval, nullptr);
	5433	if (value == irb->codegen->invalid_inst_src)
	5434	return value;
	5435
	5436	return ir_build_un_op(irb, scope, node, op_id, value);
	5437	}
	5438
	5439	static IrInstSrc ir_gen_prefix_op_id(IrBuilderSrc irb, Scope scope, AstNode node, IrUnOp op_id) {
	5440	return ir_gen_prefix_op_id_lval(irb, scope, node, op_id, LValNone);
	5441	}
	5442
	5443	static IrInstSrc ir_expr_wrap(IrBuilderSrc irb, Scope scope, IrInstSrc inst, ResultLoc *result_loc) {
	5444	if (inst == irb->codegen->invalid_inst_src) return inst;
	5445	ir_build_end_expr(irb, scope, inst->base.source_node, inst, result_loc);
	5446	return inst;
	5447	}
	5448
	5449	static IrInstSrc ir_lval_wrap(IrBuilderSrc irb, Scope scope, IrInstSrc value, LVal lval,
	5450	ResultLoc *result_loc)
	5451	{
	5452	// This logic must be kept in sync with
	5453	// [STMT_EXPR_TEST_THING] <--- (search this token)
	5454	if (value == irb->codegen->invalid_inst_src \|\|
	5455	instr_is_unreachable(value) \|\|
	5456	value->base.source_node->type == NodeTypeDefer \|\|
	5457	value->id == IrInstSrcIdDeclVar)
	5458	{
	5459	return value;
	5460	}
	5461
	5462	assert(lval != LValAssign);
	5463	if (lval == LValPtr) {
	5464	// We needed a pointer to a value, but we got a value. So we create
	5465	// an instruction which just makes a pointer of it.
	5466	return ir_build_ref_src(irb, scope, value->base.source_node, value);
	5467	} else if (result_loc != nullptr) {
	5468	return ir_expr_wrap(irb, scope, value, result_loc);
	5469	} else {
	5470	return value;
	5471	}
	5472
	5473	}
	5474
	5475	static PtrLen star_token_to_ptr_len(TokenId token_id) {
	5476	switch (token_id) {
	5477	case TokenIdStar:
	5478	case TokenIdStarStar:
	5479	return PtrLenSingle;
	5480	case TokenIdLBracket:
	5481	return PtrLenUnknown;
	5482	case TokenIdIdentifier:
	5483	return PtrLenC;
	5484	default:
	5485	zig_unreachable();
	5486	}
	5487	}
	5488
	5489	static Error token_number_literal_u32(IrBuilderSrc irb, AstNode source_node,
	5490	RootStruct root_struct, uint32_t result, TokenIndex token)
	5491	{
	5492	BigInt bigint;
	5493	token_number_literal_bigint(root_struct, &bigint, token);
	5494
	5495	if (!bigint_fits_in_bits(&bigint, 32, false)) {
	5496	Buf *val_buf = buf_alloc();
	5497	bigint_append_buf(val_buf, &bigint, 10);
	5498	exec_add_error_node(irb->codegen, irb->exec, source_node,
	5499	buf_sprintf("value %s too large for u32", buf_ptr(val_buf)));
	5500	bigint_deinit(&bigint);
	5501	return ErrorSemanticAnalyzeFail;
	5502	}
	5503	*result = bigint_as_u32(&bigint);
	5504	bigint_deinit(&bigint);
	5505	return ErrorNone;
	5506
	5507	}
	5508
	5509	static IrInstSrc ir_gen_pointer_type(IrBuilderSrc irb, Scope scope, AstNode node) {
	5510	Error err;
	5511	assert(node->type == NodeTypePointerType);
	5512
	5513	RootStruct *root_struct = node->owner->data.structure.root_struct;
	5514	TokenId star_tok_id = root_struct->token_ids[node->data.pointer_type.star_token];
	5515	PtrLen ptr_len = star_token_to_ptr_len(star_tok_id);
	5516
	5517	bool is_const = node->data.pointer_type.is_const;
	5518	bool is_volatile = node->data.pointer_type.is_volatile;
	5519	bool is_allow_zero = node->data.pointer_type.allow_zero_token != 0;
	5520	AstNode *sentinel_expr = node->data.pointer_type.sentinel;
	5521	AstNode *expr_node = node->data.pointer_type.op_expr;
	5522	AstNode *align_expr = node->data.pointer_type.align_expr;
	5523
	5524	IrInstSrc *sentinel;
	5525	if (sentinel_expr != nullptr) {
	5526	sentinel = ir_gen_node(irb, sentinel_expr, scope);
	5527	if (sentinel == irb->codegen->invalid_inst_src)
	5528	return sentinel;
	5529	} else {
	5530	sentinel = nullptr;
	5531	}
	5532
	5533	IrInstSrc *align_value;
	5534	if (align_expr != nullptr) {
	5535	align_value = ir_gen_node(irb, align_expr, scope);
	5536	if (align_value == irb->codegen->invalid_inst_src)
	5537	return align_value;
	5538	} else {
	5539	align_value = nullptr;
	5540	}
	5541
	5542	IrInstSrc *child_type = ir_gen_node(irb, expr_node, scope);
	5543	if (child_type == irb->codegen->invalid_inst_src)
	5544	return child_type;
	5545
	5546	uint32_t bit_offset_start = 0;
	5547	if (node->data.pointer_type.bit_offset_start != 0) {
	5548	if ((err = token_number_literal_u32(irb, node, root_struct, &bit_offset_start,
	5549	node->data.pointer_type.bit_offset_start)))
	5550	{
	5551	return irb->codegen->invalid_inst_src;
	5552	}
	5553	}
	5554
	5555	uint32_t host_int_bytes = 0;
	5556	if (node->data.pointer_type.host_int_bytes != 0) {
	5557	if ((err = token_number_literal_u32(irb, node, root_struct, &host_int_bytes,
	5558	node->data.pointer_type.host_int_bytes)))
	5559	{
	5560	return irb->codegen->invalid_inst_src;
	5561	}
	5562	}
	5563
	5564	if (host_int_bytes != 0 && bit_offset_start >= host_int_bytes * 8) {
	5565	exec_add_error_node(irb->codegen, irb->exec, node,
	5566	buf_sprintf("bit offset starts after end of host integer"));
	5567	return irb->codegen->invalid_inst_src;
	5568	}
	5569
	5570	return ir_build_ptr_type(irb, scope, node, child_type, is_const, is_volatile,
	5571	ptr_len, sentinel, align_value, bit_offset_start, host_int_bytes, is_allow_zero);
	5572	}
	5573
	5574	static IrInstSrc ir_gen_catch_unreachable(IrBuilderSrc irb, Scope scope, AstNode source_node,
	5575	AstNode expr_node, LVal lval, ResultLoc result_loc)
	5576	{
	5577	IrInstSrc *err_union_ptr = ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr);
	5578	if (err_union_ptr == irb->codegen->invalid_inst_src)
	5579	return irb->codegen->invalid_inst_src;
	5580
	5581	IrInstSrc *payload_ptr = ir_build_unwrap_err_payload_src(irb, scope, source_node, err_union_ptr, true, false);
	5582	if (payload_ptr == irb->codegen->invalid_inst_src)
	5583	return irb->codegen->invalid_inst_src;
	5584
	5585	if (lval == LValPtr)
	5586	return payload_ptr;
	5587
	5588	IrInstSrc *load_ptr = ir_build_load_ptr(irb, scope, source_node, payload_ptr);
	5589	return ir_expr_wrap(irb, scope, load_ptr, result_loc);
	5590	}
	5591
	5592	static IrInstSrc ir_gen_bool_not(IrBuilderSrc irb, Scope scope, AstNode node) {
	5593	assert(node->type == NodeTypePrefixOpExpr);
	5594	AstNode *expr_node = node->data.prefix_op_expr.primary_expr;
	5595
	5596	IrInstSrc *value = ir_gen_node(irb, expr_node, scope);
	5597	if (value == irb->codegen->invalid_inst_src)
	5598	return irb->codegen->invalid_inst_src;
	5599
	5600	return ir_build_bool_not(irb, scope, node, value);
	5601	}
	5602
	5603	static IrInstSrc ir_gen_prefix_op_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
	5604	ResultLoc *result_loc)
	5605	{
	5606	assert(node->type == NodeTypePrefixOpExpr);
	5607
	5608	PrefixOp prefix_op = node->data.prefix_op_expr.prefix_op;
	5609
	5610	switch (prefix_op) {
	5611	case PrefixOpInvalid:
	5612	zig_unreachable();
	5613	case PrefixOpBoolNot:
	5614	return ir_lval_wrap(irb, scope, ir_gen_bool_not(irb, scope, node), lval, result_loc);
	5615	case PrefixOpBinNot:
	5616	return ir_lval_wrap(irb, scope, ir_gen_prefix_op_id(irb, scope, node, IrUnOpBinNot), lval, result_loc);
	5617	case PrefixOpNegation:
	5618	return ir_lval_wrap(irb, scope, ir_gen_prefix_op_id(irb, scope, node, IrUnOpNegation), lval, result_loc);
	5619	case PrefixOpNegationWrap:
	5620	return ir_lval_wrap(irb, scope, ir_gen_prefix_op_id(irb, scope, node, IrUnOpNegationWrap), lval, result_loc);
	5621	case PrefixOpOptional:
	5622	return ir_lval_wrap(irb, scope, ir_gen_prefix_op_id(irb, scope, node, IrUnOpOptional), lval, result_loc);
	5623	case PrefixOpAddrOf: {
	5624	AstNode *expr_node = node->data.prefix_op_expr.primary_expr;
	5625	return ir_lval_wrap(irb, scope, ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr), lval, result_loc);
	5626	}
	5627	}
	5628	zig_unreachable();
	5629	}
	5630
	5631	static IrInstSrc ir_gen_union_init_expr(IrBuilderSrc irb, Scope scope, AstNode source_node,
	5632	IrInstSrc union_type, IrInstSrc field_name, AstNode *expr_node,
	5633	LVal lval, ResultLoc *parent_result_loc)
	5634	{
	5635	IrInstSrc *container_ptr = ir_build_resolve_result(irb, scope, source_node, parent_result_loc, union_type);
	5636	IrInstSrc *field_ptr = ir_build_field_ptr_instruction(irb, scope, source_node, container_ptr,
	5637	field_name, true);
	5638
	5639	ResultLocInstruction *result_loc_inst = heap::c_allocator.create<ResultLocInstruction>();
	5640	result_loc_inst->base.id = ResultLocIdInstruction;
	5641	result_loc_inst->base.source_instruction = field_ptr;
	5642	ir_ref_instruction(field_ptr, irb->current_basic_block);
	5643	ir_build_reset_result(irb, scope, expr_node, &result_loc_inst->base);
	5644
	5645	IrInstSrc *expr_value = ir_gen_node_extra(irb, expr_node, scope, LValNone,
	5646	&result_loc_inst->base);
	5647	if (expr_value == irb->codegen->invalid_inst_src)
	5648	return expr_value;
	5649
	5650	IrInstSrc *init_union = ir_build_union_init_named_field(irb, scope, source_node, union_type,
	5651	field_name, field_ptr, container_ptr);
	5652
	5653	return ir_lval_wrap(irb, scope, init_union, lval, parent_result_loc);
	5654	}
	5655
	5656	static IrInstSrc ir_gen_container_init_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
	5657	ResultLoc *parent_result_loc)
	5658	{
	5659	assert(node->type == NodeTypeContainerInitExpr);
	5660
	5661	AstNodeContainerInitExpr *container_init_expr = &node->data.container_init_expr;
	5662	ContainerInitKind kind = container_init_expr->kind;
	5663
	5664	ResultLocCast *result_loc_cast = nullptr;
	5665	ResultLoc *child_result_loc;
	5666	AstNode *init_array_type_source_node;
	5667	if (container_init_expr->type != nullptr) {
	5668	IrInstSrc *container_type;
	5669	if (container_init_expr->type->type == NodeTypeInferredArrayType) {
	5670	if (kind == ContainerInitKindStruct) {
	5671	add_node_error(irb->codegen, container_init_expr->type,
	5672	buf_sprintf("initializing array with struct syntax"));
	5673	return irb->codegen->invalid_inst_src;
	5674	}
	5675	IrInstSrc *sentinel;
	5676	if (container_init_expr->type->data.inferred_array_type.sentinel != nullptr) {
	5677	sentinel = ir_gen_node(irb, container_init_expr->type->data.inferred_array_type.sentinel, scope);
	5678	if (sentinel == irb->codegen->invalid_inst_src)
	5679	return sentinel;
	5680	} else {
	5681	sentinel = nullptr;
	5682	}
	5683
	5684	IrInstSrc *elem_type = ir_gen_node(irb,
	5685	container_init_expr->type->data.inferred_array_type.child_type, scope);
	5686	if (elem_type == irb->codegen->invalid_inst_src)
	5687	return elem_type;
	5688	size_t item_count = container_init_expr->entries.length;
	5689	IrInstSrc *item_count_inst = ir_build_const_usize(irb, scope, node, item_count);
	5690	container_type = ir_build_array_type(irb, scope, node, item_count_inst, sentinel, elem_type);
	5691	} else {
	5692	container_type = ir_gen_node(irb, container_init_expr->type, scope);
	5693	if (container_type == irb->codegen->invalid_inst_src)
	5694	return container_type;
	5695	}
	5696
	5697	result_loc_cast = ir_build_cast_result_loc(irb, container_type, parent_result_loc);
	5698	child_result_loc = &result_loc_cast->base;
	5699	init_array_type_source_node = container_type->base.source_node;
	5700	} else {
	5701	child_result_loc = parent_result_loc;
	5702	if (parent_result_loc->source_instruction != nullptr) {
	5703	init_array_type_source_node = parent_result_loc->source_instruction->base.source_node;
	5704	} else {
	5705	init_array_type_source_node = node;
	5706	}
	5707	}
	5708
	5709	switch (kind) {
	5710	case ContainerInitKindStruct: {
	5711	IrInstSrc *container_ptr = ir_build_resolve_result(irb, scope, node, child_result_loc,
	5712	nullptr);
	5713
	5714	size_t field_count = container_init_expr->entries.length;
	5715	IrInstSrcContainerInitFieldsField *fields = heap::c_allocator.allocate<IrInstSrcContainerInitFieldsField>(field_count);
	5716	for (size_t i = 0; i < field_count; i += 1) {
	5717	AstNode *entry_node = container_init_expr->entries.at(i);
	5718	assert(entry_node->type == NodeTypeStructValueField);
	5719
	5720	Buf *name = entry_node->data.struct_val_field.name;
	5721	AstNode *expr_node = entry_node->data.struct_val_field.expr;
	5722
	5723	IrInstSrc *field_ptr = ir_build_field_ptr(irb, scope, entry_node, container_ptr, name, true);
	5724	ResultLocInstruction *result_loc_inst = heap::c_allocator.create<ResultLocInstruction>();
	5725	result_loc_inst->base.id = ResultLocIdInstruction;
	5726	result_loc_inst->base.source_instruction = field_ptr;
	5727	result_loc_inst->base.allow_write_through_const = true;
	5728	ir_ref_instruction(field_ptr, irb->current_basic_block);
	5729	ir_build_reset_result(irb, scope, expr_node, &result_loc_inst->base);
	5730
	5731	IrInstSrc *expr_value = ir_gen_node_extra(irb, expr_node, scope, LValNone,
	5732	&result_loc_inst->base);
	5733	if (expr_value == irb->codegen->invalid_inst_src)
	5734	return expr_value;
	5735
	5736	fields[i].name = name;
	5737	fields[i].source_node = entry_node;
	5738	fields[i].result_loc = field_ptr;
	5739	}
	5740	IrInstSrc *result = ir_build_container_init_fields(irb, scope, node, field_count,
	5741	fields, container_ptr);
	5742
	5743	if (result_loc_cast != nullptr) {
	5744	result = ir_build_implicit_cast(irb, scope, node, result, result_loc_cast);
	5745	}
	5746	return ir_lval_wrap(irb, scope, result, lval, parent_result_loc);
	5747	}
	5748	case ContainerInitKindArray: {
	5749	size_t item_count = container_init_expr->entries.length;
	5750
	5751	IrInstSrc *container_ptr = ir_build_resolve_result(irb, scope, node, child_result_loc,
	5752	nullptr);
	5753
	5754	IrInstSrc *result_locs = heap::c_allocator.allocate<IrInstSrc >(item_count);
	5755	for (size_t i = 0; i < item_count; i += 1) {
	5756	AstNode *expr_node = container_init_expr->entries.at(i);
	5757
	5758	IrInstSrc *elem_index = ir_build_const_usize(irb, scope, expr_node, i);
	5759	IrInstSrc *elem_ptr = ir_build_elem_ptr(irb, scope, expr_node, container_ptr,
	5760	elem_index, false, PtrLenSingle, init_array_type_source_node);
	5761	ResultLocInstruction *result_loc_inst = heap::c_allocator.create<ResultLocInstruction>();
	5762	result_loc_inst->base.id = ResultLocIdInstruction;
	5763	result_loc_inst->base.source_instruction = elem_ptr;
	5764	result_loc_inst->base.allow_write_through_const = true;
	5765	ir_ref_instruction(elem_ptr, irb->current_basic_block);
	5766	ir_build_reset_result(irb, scope, expr_node, &result_loc_inst->base);
	5767
	5768	IrInstSrc *expr_value = ir_gen_node_extra(irb, expr_node, scope, LValNone,
	5769	&result_loc_inst->base);
	5770	if (expr_value == irb->codegen->invalid_inst_src)
	5771	return expr_value;
	5772
	5773	result_locs[i] = elem_ptr;
	5774	}
	5775	IrInstSrc *result = ir_build_container_init_list(irb, scope, node, item_count,
	5776	result_locs, container_ptr, init_array_type_source_node);
	5777	if (result_loc_cast != nullptr) {
	5778	result = ir_build_implicit_cast(irb, scope, node, result, result_loc_cast);
	5779	}
	5780	return ir_lval_wrap(irb, scope, result, lval, parent_result_loc);
	5781	}
	5782	}
	5783	zig_unreachable();
	5784	}
	5785
	5786	static ResultLocVar ir_build_var_result_loc(IrBuilderSrc irb, IrInstSrc alloca, ZigVar var) {
	5787	ResultLocVar *result_loc_var = heap::c_allocator.create<ResultLocVar>();
	5788	result_loc_var->base.id = ResultLocIdVar;
	5789	result_loc_var->base.source_instruction = alloca;
	5790	result_loc_var->base.allow_write_through_const = true;
	5791	result_loc_var->var = var;
	5792
	5793	ir_build_reset_result(irb, alloca->base.scope, alloca->base.source_node, &result_loc_var->base);
	5794
	5795	return result_loc_var;
	5796	}
	5797
	5798	static ResultLocCast ir_build_cast_result_loc(IrBuilderSrc irb, IrInstSrc *dest_type,
	5799	ResultLoc *parent_result_loc)
	5800	{
	5801	ResultLocCast *result_loc_cast = heap::c_allocator.create<ResultLocCast>();
	5802	result_loc_cast->base.id = ResultLocIdCast;
	5803	result_loc_cast->base.source_instruction = dest_type;
	5804	result_loc_cast->base.allow_write_through_const = parent_result_loc->allow_write_through_const;
	5805	ir_ref_instruction(dest_type, irb->current_basic_block);
	5806	result_loc_cast->parent = parent_result_loc;
	5807
	5808	ir_build_reset_result(irb, dest_type->base.scope, dest_type->base.source_node, &result_loc_cast->base);
	5809
	5810	return result_loc_cast;
	5811	}
	5812
	5813	static void build_decl_var_and_init(IrBuilderSrc irb, Scope scope, AstNode source_node, ZigVar var,
	5814	IrInstSrc init, const char name_hint, IrInstSrc *is_comptime)
	5815	{
	5816	IrInstSrc *alloca = ir_build_alloca_src(irb, scope, source_node, nullptr, name_hint, is_comptime);
	5817	ResultLocVar *var_result_loc = ir_build_var_result_loc(irb, alloca, var);
	5818	ir_build_end_expr(irb, scope, source_node, init, &var_result_loc->base);
	5819	ir_build_var_decl_src(irb, scope, source_node, var, nullptr, alloca);
	5820	}
	5821
	5822	static IrInstSrc ir_gen_var_decl(IrBuilderSrc irb, Scope scope, AstNode node) {
	5823	assert(node->type == NodeTypeVariableDeclaration);
	5824
	5825	AstNodeVariableDeclaration *variable_declaration = &node->data.variable_declaration;
	5826
	5827	if (buf_eql_str(variable_declaration->symbol, "_")) {
	5828	add_node_error(irb->codegen, node, buf_sprintf("`_` is not a declarable symbol"));
	5829	return irb->codegen->invalid_inst_src;
	5830	}
	5831
	5832	// Used for the type expr and the align expr
	5833	Scope *comptime_scope = create_comptime_scope(irb->codegen, node, scope);
	5834
	5835	IrInstSrc *type_instruction;
	5836	if (variable_declaration->type != nullptr) {
	5837	type_instruction = ir_gen_node(irb, variable_declaration->type, comptime_scope);
	5838	if (type_instruction == irb->codegen->invalid_inst_src)
	5839	return type_instruction;
	5840	} else {
	5841	type_instruction = nullptr;
	5842	}
	5843
	5844	bool is_shadowable = false;
	5845	bool is_const = variable_declaration->is_const;
	5846	bool is_extern = variable_declaration->is_extern;
	5847
	5848	bool is_comptime_scalar = ir_should_inline(irb->exec, scope) \|\| variable_declaration->is_comptime;
	5849	IrInstSrc *is_comptime = ir_build_const_bool(irb, scope, node, is_comptime_scalar);
	5850	ZigVar *var = ir_create_var(irb, node, scope, variable_declaration->symbol,
	5851	is_const, is_const, is_shadowable, is_comptime);
	5852	// we detect IrInstSrcDeclVar in gen_block to make sure the next node
	5853	// is inside var->child_scope
	5854
	5855	if (!is_extern && !variable_declaration->expr) {
	5856	var->var_type = irb->codegen->builtin_types.entry_invalid;
	5857	add_node_error(irb->codegen, node, buf_sprintf("variables must be initialized"));
	5858	return irb->codegen->invalid_inst_src;
	5859	}
	5860
	5861	IrInstSrc *align_value = nullptr;
	5862	if (variable_declaration->align_expr != nullptr) {
	5863	align_value = ir_gen_node(irb, variable_declaration->align_expr, comptime_scope);
	5864	if (align_value == irb->codegen->invalid_inst_src)
	5865	return align_value;
	5866	}
	5867
	5868	if (variable_declaration->section_expr != nullptr) {
	5869	add_node_error(irb->codegen, variable_declaration->section_expr,
	5870	buf_sprintf("cannot set section of local variable '%s'", buf_ptr(variable_declaration->symbol)));
	5871	}
	5872
	5873	// Parser should ensure that this never happens
	5874	assert(variable_declaration->threadlocal_tok == 0);
	5875
	5876	IrInstSrc *alloca = ir_build_alloca_src(irb, scope, node, align_value,
	5877	buf_ptr(variable_declaration->symbol), is_comptime);
	5878
	5879	// Create a result location for the initialization expression.
	5880	ResultLocVar *result_loc_var = ir_build_var_result_loc(irb, alloca, var);
	5881	ResultLoc *init_result_loc;
	5882	ResultLocCast *result_loc_cast;
	5883	if (type_instruction != nullptr) {
	5884	result_loc_cast = ir_build_cast_result_loc(irb, type_instruction, &result_loc_var->base);
	5885	init_result_loc = &result_loc_cast->base;
	5886	} else {
	5887	result_loc_cast = nullptr;
	5888	init_result_loc = &result_loc_var->base;
	5889	}
	5890
	5891	Scope *init_scope = is_comptime_scalar ?
	5892	create_comptime_scope(irb->codegen, variable_declaration->expr, scope) : scope;
	5893
	5894	// Temporarily set the name of the IrExecutableSrc to the VariableDeclaration
	5895	// so that the struct or enum from the init expression inherits the name.
	5896	Buf *old_exec_name = irb->exec->name;
	5897	irb->exec->name = variable_declaration->symbol;
	5898	IrInstSrc *init_value = ir_gen_node_extra(irb, variable_declaration->expr, init_scope,
	5899	LValNone, init_result_loc);
	5900	irb->exec->name = old_exec_name;
	5901
	5902	if (init_value == irb->codegen->invalid_inst_src)
	5903	return irb->codegen->invalid_inst_src;
	5904
	5905	if (result_loc_cast != nullptr) {
	5906	IrInstSrc *implicit_cast = ir_build_implicit_cast(irb, scope, init_value->base.source_node,
	5907	init_value, result_loc_cast);
	5908	ir_build_end_expr(irb, scope, node, implicit_cast, &result_loc_var->base);
	5909	}
	5910
	5911	return ir_build_var_decl_src(irb, scope, node, var, align_value, alloca);
	5912	}
	5913
	5914	static IrInstSrc ir_gen_while_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
	5915	ResultLoc *result_loc)
	5916	{
	5917	assert(node->type == NodeTypeWhileExpr);
	5918
	5919	AstNode *continue_expr_node = node->data.while_expr.continue_expr;
	5920	AstNode *else_node = node->data.while_expr.else_node;
	5921
	5922	IrBasicBlockSrc *cond_block = ir_create_basic_block(irb, scope, "WhileCond");
	5923	IrBasicBlockSrc *body_block = ir_create_basic_block(irb, scope, "WhileBody");
	5924	IrBasicBlockSrc *continue_block = continue_expr_node ?
	5925	ir_create_basic_block(irb, scope, "WhileContinue") : cond_block;
	5926	IrBasicBlockSrc *end_block = ir_create_basic_block(irb, scope, "WhileEnd");
	5927	IrBasicBlockSrc *else_block = else_node ?
	5928	ir_create_basic_block(irb, scope, "WhileElse") : end_block;
	5929
	5930	IrInstSrc *is_comptime = ir_build_const_bool(irb, scope, node,
	5931	ir_should_inline(irb->exec, scope) \|\| node->data.while_expr.is_inline);
	5932	ir_build_br(irb, scope, node, cond_block, is_comptime);
	5933
	5934	Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
	5935	Buf *var_symbol = node->data.while_expr.var_symbol;
	5936	Buf *err_symbol = node->data.while_expr.err_symbol;
	5937	if (err_symbol != nullptr) {
	5938	ir_set_cursor_at_end_and_append_block(irb, cond_block);
	5939
	5940	Scope *payload_scope;
	5941	AstNode *symbol_node = node; // TODO make more accurate
	5942	ZigVar *payload_var;
	5943	if (var_symbol) {
	5944	// TODO make it an error to write to payload variable
	5945	payload_var = ir_create_var(irb, symbol_node, subexpr_scope, var_symbol,
	5946	true, false, false, is_comptime);
	5947	payload_scope = payload_var->child_scope;
	5948	} else {
	5949	payload_scope = subexpr_scope;
	5950	}
	5951	ScopeExpr *spill_scope = create_expr_scope(irb->codegen, node, payload_scope);
	5952	IrInstSrc *err_val_ptr = ir_gen_node_extra(irb, node->data.while_expr.condition, subexpr_scope,
	5953	LValPtr, nullptr);
	5954	if (err_val_ptr == irb->codegen->invalid_inst_src)
	5955	return err_val_ptr;
	5956	IrInstSrc *is_err = ir_build_test_err_src(irb, scope, node->data.while_expr.condition, err_val_ptr,
	5957	true, false);
	5958	IrBasicBlockSrc *after_cond_block = irb->current_basic_block;
	5959	IrInstSrc *void_else_result = else_node ? nullptr : ir_mark_gen(ir_build_const_void(irb, scope, node));
	5960	IrInstSrc *cond_br_inst;
	5961	if (!instr_is_unreachable(is_err)) {
	5962	cond_br_inst = ir_build_cond_br(irb, scope, node->data.while_expr.condition, is_err,
	5963	else_block, body_block, is_comptime);
	5964	cond_br_inst->is_gen = true;
	5965	} else {
	5966	// for the purposes of the source instruction to ir_build_result_peers
	5967	cond_br_inst = irb->current_basic_block->instruction_list.last();
	5968	}
	5969
	5970	ResultLocPeerParent *peer_parent = ir_build_result_peers(irb, cond_br_inst, end_block, result_loc,
	5971	is_comptime);
	5972
	5973	ir_set_cursor_at_end_and_append_block(irb, body_block);
	5974	if (var_symbol) {
	5975	IrInstSrc *payload_ptr = ir_build_unwrap_err_payload_src(irb, &spill_scope->base, symbol_node,
	5976	err_val_ptr, false, false);
	5977	IrInstSrc *var_value = node->data.while_expr.var_is_ptr ?
	5978	payload_ptr : ir_build_load_ptr(irb, &spill_scope->base, symbol_node, payload_ptr);
	5979	build_decl_var_and_init(irb, payload_scope, symbol_node, payload_var, var_value, buf_ptr(var_symbol), is_comptime);
	5980	}
	5981
	5982	ZigList<IrInstSrc *> incoming_values = {0};
	5983	ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
	5984
	5985	if (is_duplicate_label(irb->codegen, payload_scope, node, node->data.while_expr.name))
	5986	return irb->codegen->invalid_inst_src;
	5987
	5988	ScopeLoop *loop_scope = create_loop_scope(irb->codegen, node, payload_scope);
	5989	loop_scope->break_block = end_block;
	5990	loop_scope->continue_block = continue_block;
	5991	loop_scope->is_comptime = is_comptime;
	5992	loop_scope->incoming_blocks = &incoming_blocks;
	5993	loop_scope->incoming_values = &incoming_values;
	5994	loop_scope->lval = lval;
	5995	loop_scope->peer_parent = peer_parent;
	5996	loop_scope->spill_scope = spill_scope;
	5997
	5998	// Note the body block of the loop is not the place that lval and result_loc are used -
	5999	// it's actually in break statements, handled similarly to return statements.
	6000	// That is why we set those values in loop_scope above and not in this ir_gen_node call.
	6001	IrInstSrc *body_result = ir_gen_node(irb, node->data.while_expr.body, &loop_scope->base);
	6002	if (body_result == irb->codegen->invalid_inst_src)
	6003	return body_result;
	6004
	6005	if (loop_scope->name != nullptr && loop_scope->name_used == false) {
	6006	add_node_error(irb->codegen, node, buf_sprintf("unused while label"));
	6007	}
	6008
	6009	if (!instr_is_unreachable(body_result)) {
	6010	ir_mark_gen(ir_build_check_statement_is_void(irb, payload_scope, node->data.while_expr.body, body_result));
	6011	ir_mark_gen(ir_build_br(irb, payload_scope, node, continue_block, is_comptime));
	6012	}
	6013
	6014	if (continue_expr_node) {
	6015	ir_set_cursor_at_end_and_append_block(irb, continue_block);
	6016	IrInstSrc *expr_result = ir_gen_node(irb, continue_expr_node, payload_scope);
	6017	if (expr_result == irb->codegen->invalid_inst_src)
	6018	return expr_result;
	6019	if (!instr_is_unreachable(expr_result)) {
	6020	ir_mark_gen(ir_build_check_statement_is_void(irb, payload_scope, continue_expr_node, expr_result));
	6021	ir_mark_gen(ir_build_br(irb, payload_scope, node, cond_block, is_comptime));
	6022	}
	6023	}
	6024
	6025	ir_set_cursor_at_end_and_append_block(irb, else_block);
	6026	assert(else_node != nullptr);
	6027
	6028	// TODO make it an error to write to error variable
	6029	AstNode *err_symbol_node = else_node; // TODO make more accurate
	6030	ZigVar *err_var = ir_create_var(irb, err_symbol_node, scope, err_symbol,
	6031	true, false, false, is_comptime);
	6032	Scope *err_scope = err_var->child_scope;
	6033	IrInstSrc *err_ptr = ir_build_unwrap_err_code_src(irb, err_scope, err_symbol_node, err_val_ptr);
	6034	IrInstSrc *err_value = ir_build_load_ptr(irb, err_scope, err_symbol_node, err_ptr);
	6035	build_decl_var_and_init(irb, err_scope, err_symbol_node, err_var, err_value, buf_ptr(err_symbol), is_comptime);
	6036
	6037	if (peer_parent->peers.length != 0) {
	6038	peer_parent->peers.last()->next_bb = else_block;
	6039	}
	6040	ResultLocPeer *peer_result = create_peer_result(peer_parent);
	6041	peer_parent->peers.append(peer_result);
	6042	IrInstSrc *else_result = ir_gen_node_extra(irb, else_node, err_scope, lval, &peer_result->base);
	6043	if (else_result == irb->codegen->invalid_inst_src)
	6044	return else_result;
	6045	if (!instr_is_unreachable(else_result))
	6046	ir_mark_gen(ir_build_br(irb, scope, node, end_block, is_comptime));
	6047	IrBasicBlockSrc *after_else_block = irb->current_basic_block;
	6048	ir_set_cursor_at_end_and_append_block(irb, end_block);
	6049	if (else_result) {
	6050	incoming_blocks.append(after_else_block);
	6051	incoming_values.append(else_result);
	6052	} else {
	6053	incoming_blocks.append(after_cond_block);
	6054	incoming_values.append(void_else_result);
	6055	}
	6056	if (peer_parent->peers.length != 0) {
	6057	peer_parent->peers.last()->next_bb = end_block;
	6058	}
	6059
	6060	IrInstSrc *phi = ir_build_phi(irb, scope, node, incoming_blocks.length,
	6061	incoming_blocks.items, incoming_values.items, peer_parent);
	6062	return ir_expr_wrap(irb, scope, phi, result_loc);
	6063	} else if (var_symbol != nullptr) {
	6064	ir_set_cursor_at_end_and_append_block(irb, cond_block);
	6065	Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
	6066	// TODO make it an error to write to payload variable
	6067	AstNode *symbol_node = node; // TODO make more accurate
	6068
	6069	ZigVar *payload_var = ir_create_var(irb, symbol_node, subexpr_scope, var_symbol,
	6070	true, false, false, is_comptime);
	6071	Scope *child_scope = payload_var->child_scope;
	6072	ScopeExpr *spill_scope = create_expr_scope(irb->codegen, node, child_scope);
	6073	IrInstSrc *maybe_val_ptr = ir_gen_node_extra(irb, node->data.while_expr.condition, subexpr_scope,
	6074	LValPtr, nullptr);
	6075	if (maybe_val_ptr == irb->codegen->invalid_inst_src)
	6076	return maybe_val_ptr;
	6077	IrInstSrc *maybe_val = ir_build_load_ptr(irb, scope, node->data.while_expr.condition, maybe_val_ptr);
	6078	IrInstSrc *is_non_null = ir_build_test_non_null_src(irb, scope, node->data.while_expr.condition, maybe_val);
	6079	IrBasicBlockSrc *after_cond_block = irb->current_basic_block;
	6080	IrInstSrc *void_else_result = else_node ? nullptr : ir_mark_gen(ir_build_const_void(irb, scope, node));
	6081	IrInstSrc *cond_br_inst;
	6082	if (!instr_is_unreachable(is_non_null)) {
	6083	cond_br_inst = ir_build_cond_br(irb, scope, node->data.while_expr.condition, is_non_null,
	6084	body_block, else_block, is_comptime);
	6085	cond_br_inst->is_gen = true;
	6086	} else {
	6087	// for the purposes of the source instruction to ir_build_result_peers
	6088	cond_br_inst = irb->current_basic_block->instruction_list.last();
	6089	}
	6090
	6091	ResultLocPeerParent *peer_parent = ir_build_result_peers(irb, cond_br_inst, end_block, result_loc,
	6092	is_comptime);
	6093
	6094	ir_set_cursor_at_end_and_append_block(irb, body_block);
	6095	IrInstSrc *payload_ptr = ir_build_optional_unwrap_ptr(irb, &spill_scope->base, symbol_node, maybe_val_ptr, false);
	6096	IrInstSrc *var_value = node->data.while_expr.var_is_ptr ?
	6097	payload_ptr : ir_build_load_ptr(irb, &spill_scope->base, symbol_node, payload_ptr);
	6098	build_decl_var_and_init(irb, child_scope, symbol_node, payload_var, var_value, buf_ptr(var_symbol), is_comptime);
	6099
	6100	ZigList<IrInstSrc *> incoming_values = {0};
	6101	ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
	6102
	6103	if (is_duplicate_label(irb->codegen, child_scope, node, node->data.while_expr.name))
	6104	return irb->codegen->invalid_inst_src;
	6105
	6106	ScopeLoop *loop_scope = create_loop_scope(irb->codegen, node, child_scope);
	6107	loop_scope->break_block = end_block;
	6108	loop_scope->continue_block = continue_block;
	6109	loop_scope->is_comptime = is_comptime;
	6110	loop_scope->incoming_blocks = &incoming_blocks;
	6111	loop_scope->incoming_values = &incoming_values;
	6112	loop_scope->lval = lval;
	6113	loop_scope->peer_parent = peer_parent;
	6114	loop_scope->spill_scope = spill_scope;
	6115
	6116	// Note the body block of the loop is not the place that lval and result_loc are used -
	6117	// it's actually in break statements, handled similarly to return statements.
	6118	// That is why we set those values in loop_scope above and not in this ir_gen_node call.
	6119	IrInstSrc *body_result = ir_gen_node(irb, node->data.while_expr.body, &loop_scope->base);
	6120	if (body_result == irb->codegen->invalid_inst_src)
	6121	return body_result;
	6122
	6123	if (loop_scope->name != nullptr && loop_scope->name_used == false) {
	6124	add_node_error(irb->codegen, node, buf_sprintf("unused while label"));
	6125	}
	6126
	6127	if (!instr_is_unreachable(body_result)) {
	6128	ir_mark_gen(ir_build_check_statement_is_void(irb, child_scope, node->data.while_expr.body, body_result));
	6129	ir_mark_gen(ir_build_br(irb, child_scope, node, continue_block, is_comptime));
	6130	}
	6131
	6132	if (continue_expr_node) {
	6133	ir_set_cursor_at_end_and_append_block(irb, continue_block);
	6134	IrInstSrc *expr_result = ir_gen_node(irb, continue_expr_node, child_scope);
	6135	if (expr_result == irb->codegen->invalid_inst_src)
	6136	return expr_result;
	6137	if (!instr_is_unreachable(expr_result)) {
	6138	ir_mark_gen(ir_build_check_statement_is_void(irb, child_scope, continue_expr_node, expr_result));
	6139	ir_mark_gen(ir_build_br(irb, child_scope, node, cond_block, is_comptime));
	6140	}
	6141	}
	6142
	6143	IrInstSrc *else_result = nullptr;
	6144	if (else_node) {
	6145	ir_set_cursor_at_end_and_append_block(irb, else_block);
	6146
	6147	if (peer_parent->peers.length != 0) {
	6148	peer_parent->peers.last()->next_bb = else_block;
	6149	}
	6150	ResultLocPeer *peer_result = create_peer_result(peer_parent);
	6151	peer_parent->peers.append(peer_result);
	6152	else_result = ir_gen_node_extra(irb, else_node, scope, lval, &peer_result->base);
	6153	if (else_result == irb->codegen->invalid_inst_src)
	6154	return else_result;
	6155	if (!instr_is_unreachable(else_result))
	6156	ir_mark_gen(ir_build_br(irb, scope, node, end_block, is_comptime));
	6157	}
	6158	IrBasicBlockSrc *after_else_block = irb->current_basic_block;
	6159	ir_set_cursor_at_end_and_append_block(irb, end_block);
	6160	if (else_result) {
	6161	incoming_blocks.append(after_else_block);
	6162	incoming_values.append(else_result);
	6163	} else {
	6164	incoming_blocks.append(after_cond_block);
	6165	incoming_values.append(void_else_result);
	6166	}
	6167	if (peer_parent->peers.length != 0) {
	6168	peer_parent->peers.last()->next_bb = end_block;
	6169	}
	6170
	6171	IrInstSrc *phi = ir_build_phi(irb, scope, node, incoming_blocks.length,
	6172	incoming_blocks.items, incoming_values.items, peer_parent);
	6173	return ir_expr_wrap(irb, scope, phi, result_loc);
	6174	} else {
	6175	ir_set_cursor_at_end_and_append_block(irb, cond_block);
	6176	IrInstSrc *cond_val = ir_gen_node(irb, node->data.while_expr.condition, scope);
	6177	if (cond_val == irb->codegen->invalid_inst_src)
	6178	return cond_val;
	6179	IrBasicBlockSrc *after_cond_block = irb->current_basic_block;
	6180	IrInstSrc *void_else_result = else_node ? nullptr : ir_mark_gen(ir_build_const_void(irb, scope, node));
	6181	IrInstSrc *cond_br_inst;
	6182	if (!instr_is_unreachable(cond_val)) {
	6183	cond_br_inst = ir_build_cond_br(irb, scope, node->data.while_expr.condition, cond_val,
	6184	body_block, else_block, is_comptime);
	6185	cond_br_inst->is_gen = true;
	6186	} else {
	6187	// for the purposes of the source instruction to ir_build_result_peers
	6188	cond_br_inst = irb->current_basic_block->instruction_list.last();
	6189	}
	6190
	6191	ResultLocPeerParent *peer_parent = ir_build_result_peers(irb, cond_br_inst, end_block, result_loc,
	6192	is_comptime);
	6193	ir_set_cursor_at_end_and_append_block(irb, body_block);
	6194
	6195	ZigList<IrInstSrc *> incoming_values = {0};
	6196	ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
	6197
	6198	Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
	6199
	6200	if (is_duplicate_label(irb->codegen, subexpr_scope, node, node->data.while_expr.name))
	6201	return irb->codegen->invalid_inst_src;
	6202
	6203	ScopeLoop *loop_scope = create_loop_scope(irb->codegen, node, subexpr_scope);
	6204	loop_scope->break_block = end_block;
	6205	loop_scope->continue_block = continue_block;
	6206	loop_scope->is_comptime = is_comptime;
	6207	loop_scope->incoming_blocks = &incoming_blocks;
	6208	loop_scope->incoming_values = &incoming_values;
	6209	loop_scope->lval = lval;
	6210	loop_scope->peer_parent = peer_parent;
	6211
	6212	// Note the body block of the loop is not the place that lval and result_loc are used -
	6213	// it's actually in break statements, handled similarly to return statements.
	6214	// That is why we set those values in loop_scope above and not in this ir_gen_node call.
	6215	IrInstSrc *body_result = ir_gen_node(irb, node->data.while_expr.body, &loop_scope->base);
	6216	if (body_result == irb->codegen->invalid_inst_src)
	6217	return body_result;
	6218
	6219	if (loop_scope->name != nullptr && loop_scope->name_used == false) {
	6220	add_node_error(irb->codegen, node, buf_sprintf("unused while label"));
	6221	}
	6222
	6223	if (!instr_is_unreachable(body_result)) {
	6224	ir_mark_gen(ir_build_check_statement_is_void(irb, scope, node->data.while_expr.body, body_result));
	6225	ir_mark_gen(ir_build_br(irb, scope, node, continue_block, is_comptime));
	6226	}
	6227
	6228	if (continue_expr_node) {
	6229	ir_set_cursor_at_end_and_append_block(irb, continue_block);
	6230	IrInstSrc *expr_result = ir_gen_node(irb, continue_expr_node, subexpr_scope);
	6231	if (expr_result == irb->codegen->invalid_inst_src)
	6232	return expr_result;
	6233	if (!instr_is_unreachable(expr_result)) {
	6234	ir_mark_gen(ir_build_check_statement_is_void(irb, scope, continue_expr_node, expr_result));
	6235	ir_mark_gen(ir_build_br(irb, scope, node, cond_block, is_comptime));
	6236	}
	6237	}
	6238
	6239	IrInstSrc *else_result = nullptr;
	6240	if (else_node) {
	6241	ir_set_cursor_at_end_and_append_block(irb, else_block);
	6242
	6243	if (peer_parent->peers.length != 0) {
	6244	peer_parent->peers.last()->next_bb = else_block;
	6245	}
	6246	ResultLocPeer *peer_result = create_peer_result(peer_parent);
	6247	peer_parent->peers.append(peer_result);
	6248
	6249	else_result = ir_gen_node_extra(irb, else_node, subexpr_scope, lval, &peer_result->base);
	6250	if (else_result == irb->codegen->invalid_inst_src)
	6251	return else_result;
	6252	if (!instr_is_unreachable(else_result))
	6253	ir_mark_gen(ir_build_br(irb, scope, node, end_block, is_comptime));
	6254	}
	6255	IrBasicBlockSrc *after_else_block = irb->current_basic_block;
	6256	ir_set_cursor_at_end_and_append_block(irb, end_block);
	6257	if (else_result) {
	6258	incoming_blocks.append(after_else_block);
	6259	incoming_values.append(else_result);
	6260	} else {
	6261	incoming_blocks.append(after_cond_block);
	6262	incoming_values.append(void_else_result);
	6263	}
	6264	if (peer_parent->peers.length != 0) {
	6265	peer_parent->peers.last()->next_bb = end_block;
	6266	}
	6267
	6268	IrInstSrc *phi = ir_build_phi(irb, scope, node, incoming_blocks.length,
	6269	incoming_blocks.items, incoming_values.items, peer_parent);
	6270	return ir_expr_wrap(irb, scope, phi, result_loc);
	6271	}
	6272	}
	6273
	6274	static IrInstSrc ir_gen_for_expr(IrBuilderSrc irb, Scope parent_scope, AstNode node, LVal lval,
	6275	ResultLoc *result_loc)
	6276	{
	6277	assert(node->type == NodeTypeForExpr);
	6278
	6279	AstNode *array_node = node->data.for_expr.array_expr;
	6280	AstNode *elem_node = node->data.for_expr.elem_node;
	6281	AstNode *index_node = node->data.for_expr.index_node;
	6282	AstNode *body_node = node->data.for_expr.body;
	6283	AstNode *else_node = node->data.for_expr.else_node;
	6284
	6285	if (!elem_node) {
	6286	add_node_error(irb->codegen, node, buf_sprintf("for loop expression missing element parameter"));
	6287	return irb->codegen->invalid_inst_src;
	6288	}
	6289	assert(elem_node->type == NodeTypeIdentifier);
	6290
	6291	ScopeExpr *spill_scope = create_expr_scope(irb->codegen, node, parent_scope);
	6292
	6293	IrInstSrc *array_val_ptr = ir_gen_node_extra(irb, array_node, &spill_scope->base, LValPtr, nullptr);
	6294	if (array_val_ptr == irb->codegen->invalid_inst_src)
	6295	return array_val_ptr;
	6296
	6297	IrInstSrc *is_comptime = ir_build_const_bool(irb, parent_scope, node,
	6298	ir_should_inline(irb->exec, parent_scope) \|\| node->data.for_expr.is_inline);
	6299
	6300	AstNode *index_var_source_node;
	6301	ZigVar *index_var;
	6302	const char *index_var_name;
	6303	if (index_node) {
	6304	index_var_source_node = index_node;
	6305	Buf *index_var_name_buf = node_identifier_buf(index_node);
	6306	index_var = ir_create_var(irb, index_node, parent_scope, index_var_name_buf, true, false, false, is_comptime);
	6307	index_var_name = buf_ptr(index_var_name_buf);
	6308	} else {
	6309	index_var_source_node = node;
	6310	index_var = ir_create_var(irb, node, parent_scope, nullptr, true, false, true, is_comptime);
	6311	index_var_name = "i";
	6312	}
	6313
	6314	IrInstSrc *zero = ir_build_const_usize(irb, parent_scope, node, 0);
	6315	build_decl_var_and_init(irb, parent_scope, index_var_source_node, index_var, zero, index_var_name, is_comptime);
	6316	parent_scope = index_var->child_scope;
	6317
	6318	IrInstSrc *one = ir_build_const_usize(irb, parent_scope, node, 1);
	6319	IrInstSrc *index_ptr = ir_build_var_ptr(irb, parent_scope, node, index_var);
	6320
	6321
	6322	IrBasicBlockSrc *cond_block = ir_create_basic_block(irb, parent_scope, "ForCond");
	6323	IrBasicBlockSrc *body_block = ir_create_basic_block(irb, parent_scope, "ForBody");
	6324	IrBasicBlockSrc *end_block = ir_create_basic_block(irb, parent_scope, "ForEnd");
	6325	IrBasicBlockSrc *else_block = else_node ? ir_create_basic_block(irb, parent_scope, "ForElse") : end_block;
	6326	IrBasicBlockSrc *continue_block = ir_create_basic_block(irb, parent_scope, "ForContinue");
	6327
	6328	Buf *len_field_name = buf_create_from_str("len");
	6329	IrInstSrc *len_ref = ir_build_field_ptr(irb, parent_scope, node, array_val_ptr, len_field_name, false);
	6330	IrInstSrc *len_val = ir_build_load_ptr(irb, &spill_scope->base, node, len_ref);
	6331	ir_build_br(irb, parent_scope, node, cond_block, is_comptime);
	6332
	6333	ir_set_cursor_at_end_and_append_block(irb, cond_block);
	6334	IrInstSrc *index_val = ir_build_load_ptr(irb, &spill_scope->base, node, index_ptr);
	6335	IrInstSrc *cond = ir_build_bin_op(irb, parent_scope, node, IrBinOpCmpLessThan, index_val, len_val, false);
	6336	IrBasicBlockSrc *after_cond_block = irb->current_basic_block;
	6337	IrInstSrc *void_else_value = else_node ? nullptr : ir_mark_gen(ir_build_const_void(irb, parent_scope, node));
	6338	IrInstSrc *cond_br_inst = ir_mark_gen(ir_build_cond_br(irb, parent_scope, node, cond,
	6339	body_block, else_block, is_comptime));
	6340
	6341	ResultLocPeerParent *peer_parent = ir_build_result_peers(irb, cond_br_inst, end_block, result_loc, is_comptime);
	6342
	6343	ir_set_cursor_at_end_and_append_block(irb, body_block);
	6344	IrInstSrc *elem_ptr = ir_build_elem_ptr(irb, &spill_scope->base, node, array_val_ptr, index_val,
	6345	false, PtrLenSingle, nullptr);
	6346	// TODO make it an error to write to element variable or i variable.
	6347	Buf *elem_var_name = node_identifier_buf(elem_node);
	6348	ZigVar *elem_var = ir_create_var(irb, elem_node, parent_scope, elem_var_name, true, false, false, is_comptime);
	6349	Scope *child_scope = elem_var->child_scope;
	6350
	6351	IrInstSrc *elem_value = node->data.for_expr.elem_is_ptr ?
	6352	elem_ptr : ir_build_load_ptr(irb, &spill_scope->base, elem_node, elem_ptr);
	6353	build_decl_var_and_init(irb, parent_scope, elem_node, elem_var, elem_value, buf_ptr(elem_var_name), is_comptime);
	6354
	6355	if (is_duplicate_label(irb->codegen, child_scope, node, node->data.for_expr.name))
	6356	return irb->codegen->invalid_inst_src;
	6357
	6358	ZigList<IrInstSrc *> incoming_values = {0};
	6359	ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
	6360	ScopeLoop *loop_scope = create_loop_scope(irb->codegen, node, child_scope);
	6361	loop_scope->break_block = end_block;
	6362	loop_scope->continue_block = continue_block;
	6363	loop_scope->is_comptime = is_comptime;
	6364	loop_scope->incoming_blocks = &incoming_blocks;
	6365	loop_scope->incoming_values = &incoming_values;
	6366	loop_scope->lval = LValNone;
	6367	loop_scope->peer_parent = peer_parent;
	6368	loop_scope->spill_scope = spill_scope;
	6369
	6370	// Note the body block of the loop is not the place that lval and result_loc are used -
	6371	// it's actually in break statements, handled similarly to return statements.
	6372	// That is why we set those values in loop_scope above and not in this ir_gen_node call.
	6373	IrInstSrc *body_result = ir_gen_node(irb, body_node, &loop_scope->base);
	6374	if (body_result == irb->codegen->invalid_inst_src)
	6375	return irb->codegen->invalid_inst_src;
	6376
	6377	if (loop_scope->name != nullptr && loop_scope->name_used == false) {
	6378	add_node_error(irb->codegen, node, buf_sprintf("unused for label"));
	6379	}
	6380
	6381	if (!instr_is_unreachable(body_result)) {
	6382	ir_mark_gen(ir_build_check_statement_is_void(irb, child_scope, node->data.for_expr.body, body_result));
	6383	ir_mark_gen(ir_build_br(irb, child_scope, node, continue_block, is_comptime));
	6384	}
	6385
	6386	ir_set_cursor_at_end_and_append_block(irb, continue_block);
	6387	IrInstSrc *new_index_val = ir_build_bin_op(irb, child_scope, node, IrBinOpAdd, index_val, one, false);
	6388	ir_build_store_ptr(irb, child_scope, node, index_ptr, new_index_val)->allow_write_through_const = true;
	6389	ir_build_br(irb, child_scope, node, cond_block, is_comptime);
	6390
	6391	IrInstSrc *else_result = nullptr;
	6392	if (else_node) {
	6393	ir_set_cursor_at_end_and_append_block(irb, else_block);
	6394
	6395	if (peer_parent->peers.length != 0) {
	6396	peer_parent->peers.last()->next_bb = else_block;
	6397	}
	6398	ResultLocPeer *peer_result = create_peer_result(peer_parent);
	6399	peer_parent->peers.append(peer_result);
	6400	else_result = ir_gen_node_extra(irb, else_node, parent_scope, LValNone, &peer_result->base);
	6401	if (else_result == irb->codegen->invalid_inst_src)
	6402	return else_result;
	6403	if (!instr_is_unreachable(else_result))
	6404	ir_mark_gen(ir_build_br(irb, parent_scope, node, end_block, is_comptime));
	6405	}
	6406	IrBasicBlockSrc *after_else_block = irb->current_basic_block;
	6407	ir_set_cursor_at_end_and_append_block(irb, end_block);
	6408
	6409	if (else_result) {
	6410	incoming_blocks.append(after_else_block);
	6411	incoming_values.append(else_result);
	6412	} else {
	6413	incoming_blocks.append(after_cond_block);
	6414	incoming_values.append(void_else_value);
	6415	}
	6416	if (peer_parent->peers.length != 0) {
	6417	peer_parent->peers.last()->next_bb = end_block;
	6418	}
	6419
	6420	IrInstSrc *phi = ir_build_phi(irb, parent_scope, node, incoming_blocks.length,
	6421	incoming_blocks.items, incoming_values.items, peer_parent);
	6422	return ir_lval_wrap(irb, parent_scope, phi, lval, result_loc);
	6423	}
	6424
	6425	static IrInstSrc ir_gen_bool_literal(IrBuilderSrc irb, Scope scope, AstNode node) {
	6426	assert(node->type == NodeTypeBoolLiteral);
	6427	return ir_build_const_bool(irb, scope, node, node->data.bool_literal.value);
	6428	}
	6429
	6430	static IrInstSrc ir_gen_enum_literal(IrBuilderSrc irb, Scope scope, AstNode node) {
	6431	assert(node->type == NodeTypeEnumLiteral);
	6432	RootStruct *root_struct = node->owner->data.structure.root_struct;
	6433	Buf *name = token_identifier_buf(root_struct, node->main_token + 1);
	6434	return ir_build_const_enum_literal(irb, scope, node, name);
	6435	}
	6436
	6437	static IrInstSrc ir_gen_string_literal(IrBuilderSrc irb, Scope scope, AstNode node) {
	6438	Error err;
	6439	assert(node->type == NodeTypeStringLiteral);
	6440
	6441	RootStruct *root_struct = node->owner->data.structure.root_struct;
	6442	const char *source = buf_ptr(root_struct->source_code);
	6443
	6444	TokenId *token_ids = root_struct->token_ids;
	6445
	6446	Buf *str = buf_alloc();
	6447	if (token_ids[node->main_token] == TokenIdStringLiteral) {
	6448	size_t byte_offset = root_struct->token_locs[node->main_token].offset;
	6449	size_t bad_index;
	6450	if ((err = source_string_literal_buf(source + byte_offset, str, &bad_index))) {
	6451	add_token_error_offset(irb->codegen, node->owner, node->main_token,
	6452	buf_create_from_str("invalid string literal character"), bad_index);
	6453	}
	6454	src_assert(source[byte_offset] == '"', node);
	6455	byte_offset += 1;
	6456	} else if (token_ids[node->main_token] == TokenIdMultilineStringLiteralLine) {
	6457	TokenIndex tok_index = node->main_token;
	6458	bool first = true;
	6459	for (;token_ids[tok_index] == TokenIdMultilineStringLiteralLine; tok_index += 1) {
	6460	size_t byte_offset = root_struct->token_locs[tok_index].offset;
	6461	size_t end = byte_offset;
	6462	while (source[end] != 0 && source[end] != '\n') {
	6463	end += 1;
	6464	}
	6465	if (!first) {
	6466	buf_append_char(str, '\n');
	6467	} else {
	6468	first = false;
	6469	}
	6470	buf_append_mem(str, source + byte_offset + 2, end - byte_offset - 2);
	6471	}
	6472	} else {
	6473	zig_unreachable();
	6474	}
	6475	return ir_build_const_str_lit(irb, scope, node, str);
	6476	}
	6477
	6478	static IrInstSrc ir_gen_array_type(IrBuilderSrc irb, Scope scope, AstNode node) {
	6479	assert(node->type == NodeTypeArrayType);
	6480
	6481	AstNode *size_node = node->data.array_type.size;
	6482	AstNode *child_type_node = node->data.array_type.child_type;
	6483	bool is_const = node->data.array_type.is_const;
	6484	bool is_volatile = node->data.array_type.is_volatile;
	6485	bool is_allow_zero = node->data.array_type.allow_zero_token != 0;
	6486	AstNode *sentinel_expr = node->data.array_type.sentinel;
	6487	AstNode *align_expr = node->data.array_type.align_expr;
	6488
	6489	Scope *comptime_scope = create_comptime_scope(irb->codegen, node, scope);
	6490
	6491	IrInstSrc *sentinel;
	6492	if (sentinel_expr != nullptr) {
	6493	sentinel = ir_gen_node(irb, sentinel_expr, comptime_scope);
	6494	if (sentinel == irb->codegen->invalid_inst_src)
	6495	return sentinel;
	6496	} else {
	6497	sentinel = nullptr;
	6498	}
	6499
	6500	if (size_node) {
	6501	if (is_const) {
	6502	add_node_error(irb->codegen, node, buf_create_from_str("const qualifier invalid on array type"));
	6503	return irb->codegen->invalid_inst_src;
	6504	}
	6505	if (is_volatile) {
	6506	add_node_error(irb->codegen, node, buf_create_from_str("volatile qualifier invalid on array type"));
	6507	return irb->codegen->invalid_inst_src;
	6508	}
	6509	if (is_allow_zero) {
	6510	add_node_error(irb->codegen, node, buf_create_from_str("allowzero qualifier invalid on array type"));
	6511	return irb->codegen->invalid_inst_src;
	6512	}
	6513	if (align_expr != nullptr) {
	6514	add_node_error(irb->codegen, node, buf_create_from_str("align qualifier invalid on array type"));
	6515	return irb->codegen->invalid_inst_src;
	6516	}
	6517
	6518	IrInstSrc *size_value = ir_gen_node(irb, size_node, comptime_scope);
	6519	if (size_value == irb->codegen->invalid_inst_src)
	6520	return size_value;
	6521
	6522	IrInstSrc *child_type = ir_gen_node(irb, child_type_node, comptime_scope);
	6523	if (child_type == irb->codegen->invalid_inst_src)
	6524	return child_type;
	6525
	6526	return ir_build_array_type(irb, scope, node, size_value, sentinel, child_type);
	6527	} else {
	6528	IrInstSrc *align_value;
	6529	if (align_expr != nullptr) {
	6530	align_value = ir_gen_node(irb, align_expr, comptime_scope);
	6531	if (align_value == irb->codegen->invalid_inst_src)
	6532	return align_value;
	6533	} else {
	6534	align_value = nullptr;
	6535	}
	6536
	6537	IrInstSrc *child_type = ir_gen_node(irb, child_type_node, comptime_scope);
	6538	if (child_type == irb->codegen->invalid_inst_src)
	6539	return child_type;
	6540
	6541	return ir_build_slice_type(irb, scope, node, child_type, is_const, is_volatile, sentinel,
	6542	align_value, is_allow_zero);
	6543	}
	6544	}
	6545
	6546	static IrInstSrc ir_gen_anyframe_type(IrBuilderSrc irb, Scope scope, AstNode node) {
	6547	assert(node->type == NodeTypeAnyFrameType);
	6548
	6549	AstNode *payload_type_node = node->data.anyframe_type.payload_type;
	6550	IrInstSrc *payload_type_value = nullptr;
	6551
	6552	if (payload_type_node != nullptr) {
	6553	payload_type_value = ir_gen_node(irb, payload_type_node, scope);
	6554	if (payload_type_value == irb->codegen->invalid_inst_src)
	6555	return payload_type_value;
	6556
	6557	}
	6558
	6559	return ir_build_anyframe_type(irb, scope, node, payload_type_value);
	6560	}
	6561
	6562	static IrInstSrc ir_gen_undefined_literal(IrBuilderSrc irb, Scope scope, AstNode node) {
	6563	assert(node->type == NodeTypeUndefinedLiteral);
	6564	return ir_build_const_undefined(irb, scope, node);
	6565	}
	6566
	6567	static IrInstSrc ir_gen_asm_expr(IrBuilderSrc irb, Scope scope, AstNode node) {
	6568	assert(node->type == NodeTypeAsmExpr);
	6569	AstNodeAsmExpr *asm_expr = &node->data.asm_expr;
	6570
	6571	IrInstSrc *asm_template = ir_gen_node(irb, asm_expr->asm_template, scope);
	6572	if (asm_template == irb->codegen->invalid_inst_src)
	6573	return irb->codegen->invalid_inst_src;
	6574
	6575	bool is_volatile = asm_expr->volatile_token != 0;
	6576	bool in_fn_scope = (scope_fn_entry(scope) != nullptr);
	6577
	6578	if (!in_fn_scope) {
	6579	if (is_volatile) {
	6580	add_token_error(irb->codegen, node->owner, asm_expr->volatile_token,
	6581	buf_sprintf("volatile is meaningless on global assembly"));
	6582	return irb->codegen->invalid_inst_src;
	6583	}
	6584
	6585	if (asm_expr->output_list.length != 0 \|\| asm_expr->input_list.length != 0 \|\|
	6586	asm_expr->clobber_list.length != 0)
	6587	{
	6588	add_node_error(irb->codegen, node,
	6589	buf_sprintf("global assembly cannot have inputs, outputs, or clobbers"));
	6590	return irb->codegen->invalid_inst_src;
	6591	}
	6592
	6593	return ir_build_asm_src(irb, scope, node, asm_template, nullptr, nullptr,
	6594	nullptr, 0, is_volatile, true);
	6595	}
	6596
	6597	IrInstSrc *input_list = heap::c_allocator.allocate<IrInstSrc >(asm_expr->input_list.length);
	6598	IrInstSrc *output_types = heap::c_allocator.allocate<IrInstSrc >(asm_expr->output_list.length);
	6599	ZigVar *output_vars = heap::c_allocator.allocate<ZigVar >(asm_expr->output_list.length);
	6600	size_t return_count = 0;
	6601	if (!is_volatile && asm_expr->output_list.length == 0) {
	6602	add_node_error(irb->codegen, node,
	6603	buf_sprintf("assembly expression with no output must be marked volatile"));
	6604	return irb->codegen->invalid_inst_src;
	6605	}
	6606	for (size_t i = 0; i < asm_expr->output_list.length; i += 1) {
	6607	AsmOutput *asm_output = asm_expr->output_list.at(i);
	6608	if (asm_output->return_type) {
	6609	return_count += 1;
	6610
	6611	IrInstSrc *return_type = ir_gen_node(irb, asm_output->return_type, scope);
	6612	if (return_type == irb->codegen->invalid_inst_src)
	6613	return irb->codegen->invalid_inst_src;
	6614	if (return_count > 1) {
	6615	add_node_error(irb->codegen, node,
	6616	buf_sprintf("inline assembly allows up to one output value"));
	6617	return irb->codegen->invalid_inst_src;
	6618	}
	6619	output_types[i] = return_type;
	6620	} else {
	6621	Buf *variable_name = asm_output->variable_name;
	6622	// TODO there is some duplication here with ir_gen_symbol. I need to do a full audit of how
	6623	// inline assembly works. https://github.com/ziglang/zig/issues/215
	6624	ZigVar *var = find_variable(irb->codegen, scope, variable_name, nullptr);
	6625	if (var) {
	6626	output_vars[i] = var;
	6627	} else {
	6628	add_node_error(irb->codegen, node,
	6629	buf_sprintf("use of undeclared identifier '%s'", buf_ptr(variable_name)));
	6630	return irb->codegen->invalid_inst_src;
	6631	}
	6632	}
	6633
	6634	const char modifier = *buf_ptr(asm_output->constraint);
	6635	if (modifier != '=') {
	6636	add_node_error(irb->codegen, node,
	6637	buf_sprintf("invalid modifier starting output constraint for '%s': '%c', only '=' is supported."
	6638	" Compiler TODO: see https://github.com/ziglang/zig/issues/215",
	6639	buf_ptr(asm_output->asm_symbolic_name), modifier));
	6640	return irb->codegen->invalid_inst_src;
	6641	}
	6642	}
	6643	for (size_t i = 0; i < asm_expr->input_list.length; i += 1) {
	6644	AsmInput *asm_input = asm_expr->input_list.at(i);
	6645	IrInstSrc *input_value = ir_gen_node(irb, asm_input->expr, scope);
	6646	if (input_value == irb->codegen->invalid_inst_src)
	6647	return irb->codegen->invalid_inst_src;
	6648
	6649	input_list[i] = input_value;
	6650	}
	6651
	6652	return ir_build_asm_src(irb, scope, node, asm_template, input_list, output_types,
	6653	output_vars, return_count, is_volatile, false);
	6654	}
	6655
	6656	static IrInstSrc ir_gen_if_optional_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
	6657	ResultLoc *result_loc)
	6658	{
	6659	assert(node->type == NodeTypeIfOptional);
	6660
	6661	Buf *var_symbol = node->data.test_expr.var_symbol;
	6662	AstNode *expr_node = node->data.test_expr.target_node;
	6663	AstNode *then_node = node->data.test_expr.then_node;
	6664	AstNode *else_node = node->data.test_expr.else_node;
	6665	bool var_is_ptr = node->data.test_expr.var_is_ptr;
	6666
	6667	ScopeExpr *spill_scope = create_expr_scope(irb->codegen, expr_node, scope);
	6668	spill_scope->spill_harder = true;
	6669
	6670	IrInstSrc *maybe_val_ptr = ir_gen_node_extra(irb, expr_node, &spill_scope->base, LValPtr, nullptr);
	6671	if (maybe_val_ptr == irb->codegen->invalid_inst_src)
	6672	return maybe_val_ptr;
	6673
	6674	IrInstSrc *maybe_val = ir_build_load_ptr(irb, scope, node, maybe_val_ptr);
	6675	IrInstSrc *is_non_null = ir_build_test_non_null_src(irb, scope, node, maybe_val);
	6676
	6677	IrBasicBlockSrc *then_block = ir_create_basic_block(irb, scope, "OptionalThen");
	6678	IrBasicBlockSrc *else_block = ir_create_basic_block(irb, scope, "OptionalElse");
	6679	IrBasicBlockSrc *endif_block = ir_create_basic_block(irb, scope, "OptionalEndIf");
	6680
	6681	IrInstSrc *is_comptime;
	6682	if (ir_should_inline(irb->exec, scope)) {
	6683	is_comptime = ir_build_const_bool(irb, scope, node, true);
	6684	} else {
	6685	is_comptime = ir_build_test_comptime(irb, scope, node, is_non_null);
	6686	}
	6687	IrInstSrc *cond_br_inst = ir_build_cond_br(irb, scope, node, is_non_null,
	6688	then_block, else_block, is_comptime);
	6689
	6690	ResultLocPeerParent *peer_parent = ir_build_binary_result_peers(irb, cond_br_inst, else_block, endif_block,
	6691	result_loc, is_comptime);
	6692
	6693	ir_set_cursor_at_end_and_append_block(irb, then_block);
	6694
	6695	Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, &spill_scope->base, is_comptime);
	6696	Scope *var_scope;
	6697	if (var_symbol) {
	6698	bool is_shadowable = false;
	6699	bool is_const = true;
	6700	ZigVar *var = ir_create_var(irb, node, subexpr_scope,
	6701	var_symbol, is_const, is_const, is_shadowable, is_comptime);
	6702
	6703	IrInstSrc *payload_ptr = ir_build_optional_unwrap_ptr(irb, subexpr_scope, node, maybe_val_ptr, false);
	6704	IrInstSrc *var_value = var_is_ptr ?
	6705	payload_ptr : ir_build_load_ptr(irb, &spill_scope->base, node, payload_ptr);
	6706	build_decl_var_and_init(irb, subexpr_scope, node, var, var_value, buf_ptr(var_symbol), is_comptime);
	6707	var_scope = var->child_scope;
	6708	} else {
	6709	var_scope = subexpr_scope;
	6710	}
	6711	IrInstSrc *then_expr_result = ir_gen_node_extra(irb, then_node, var_scope, lval,
	6712	&peer_parent->peers.at(0)->base);
	6713	if (then_expr_result == irb->codegen->invalid_inst_src)
	6714	return then_expr_result;
	6715	IrBasicBlockSrc *after_then_block = irb->current_basic_block;
	6716	if (!instr_is_unreachable(then_expr_result))
	6717	ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
	6718
	6719	ir_set_cursor_at_end_and_append_block(irb, else_block);
	6720	IrInstSrc *else_expr_result;
	6721	if (else_node) {
	6722	else_expr_result = ir_gen_node_extra(irb, else_node, subexpr_scope, lval, &peer_parent->peers.at(1)->base);
	6723	if (else_expr_result == irb->codegen->invalid_inst_src)
	6724	return else_expr_result;
	6725	} else {
	6726	else_expr_result = ir_build_const_void(irb, scope, node);
	6727	ir_build_end_expr(irb, scope, node, else_expr_result, &peer_parent->peers.at(1)->base);
	6728	}
	6729	IrBasicBlockSrc *after_else_block = irb->current_basic_block;
	6730	if (!instr_is_unreachable(else_expr_result))
	6731	ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
	6732
	6733	ir_set_cursor_at_end_and_append_block(irb, endif_block);
	6734	IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
	6735	incoming_values[0] = then_expr_result;
	6736	incoming_values[1] = else_expr_result;
	6737	IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
	6738	incoming_blocks[0] = after_then_block;
	6739	incoming_blocks[1] = after_else_block;
	6740
	6741	IrInstSrc *phi = ir_build_phi(irb, scope, node, 2, incoming_blocks, incoming_values, peer_parent);
	6742	return ir_expr_wrap(irb, scope, phi, result_loc);
	6743	}
	6744
	6745	static IrInstSrc ir_gen_if_err_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
	6746	ResultLoc *result_loc)
	6747	{
	6748	assert(node->type == NodeTypeIfErrorExpr);
	6749
	6750	AstNode *target_node = node->data.if_err_expr.target_node;
	6751	AstNode *then_node = node->data.if_err_expr.then_node;
	6752	AstNode *else_node = node->data.if_err_expr.else_node;
	6753	bool var_is_ptr = node->data.if_err_expr.var_is_ptr;
	6754	bool var_is_const = true;
	6755	Buf *var_symbol = node->data.if_err_expr.var_symbol;
	6756	Buf *err_symbol = node->data.if_err_expr.err_symbol;
	6757
	6758	IrInstSrc *err_val_ptr = ir_gen_node_extra(irb, target_node, scope, LValPtr, nullptr);
	6759	if (err_val_ptr == irb->codegen->invalid_inst_src)
	6760	return err_val_ptr;
	6761
	6762	IrInstSrc *err_val = ir_build_load_ptr(irb, scope, node, err_val_ptr);
	6763	IrInstSrc *is_err = ir_build_test_err_src(irb, scope, node, err_val_ptr, true, false);
	6764
	6765	IrBasicBlockSrc *ok_block = ir_create_basic_block(irb, scope, "TryOk");
	6766	IrBasicBlockSrc *else_block = ir_create_basic_block(irb, scope, "TryElse");
	6767	IrBasicBlockSrc *endif_block = ir_create_basic_block(irb, scope, "TryEnd");
	6768
	6769	bool force_comptime = ir_should_inline(irb->exec, scope);
	6770	IrInstSrc *is_comptime = force_comptime ? ir_build_const_bool(irb, scope, node, true) : ir_build_test_comptime(irb, scope, node, is_err);
	6771	IrInstSrc *cond_br_inst = ir_build_cond_br(irb, scope, node, is_err, else_block, ok_block, is_comptime);
	6772
	6773	ResultLocPeerParent *peer_parent = ir_build_binary_result_peers(irb, cond_br_inst, else_block, endif_block,
	6774	result_loc, is_comptime);
	6775
	6776	ir_set_cursor_at_end_and_append_block(irb, ok_block);
	6777
	6778	Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
	6779	Scope *var_scope;
	6780	if (var_symbol) {
	6781	bool is_shadowable = false;
	6782	IrInstSrc *var_is_comptime = force_comptime ? ir_build_const_bool(irb, subexpr_scope, node, true) : ir_build_test_comptime(irb, subexpr_scope, node, err_val);
	6783	ZigVar *var = ir_create_var(irb, node, subexpr_scope,
	6784	var_symbol, var_is_const, var_is_const, is_shadowable, var_is_comptime);
	6785
	6786	IrInstSrc *payload_ptr = ir_build_unwrap_err_payload_src(irb, subexpr_scope, node, err_val_ptr, false, false);
	6787	IrInstSrc *var_value = var_is_ptr ?
	6788	payload_ptr : ir_build_load_ptr(irb, subexpr_scope, node, payload_ptr);
	6789	build_decl_var_and_init(irb, subexpr_scope, node, var, var_value, buf_ptr(var_symbol), var_is_comptime);
	6790	var_scope = var->child_scope;
	6791	} else {
	6792	var_scope = subexpr_scope;
	6793	}
	6794	IrInstSrc *then_expr_result = ir_gen_node_extra(irb, then_node, var_scope, lval,
	6795	&peer_parent->peers.at(0)->base);
	6796	if (then_expr_result == irb->codegen->invalid_inst_src)
	6797	return then_expr_result;
	6798	IrBasicBlockSrc *after_then_block = irb->current_basic_block;
	6799	if (!instr_is_unreachable(then_expr_result))
	6800	ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
	6801
	6802	ir_set_cursor_at_end_and_append_block(irb, else_block);
	6803
	6804	IrInstSrc *else_expr_result;
	6805	if (else_node) {
	6806	Scope *err_var_scope;
	6807	if (err_symbol) {
	6808	bool is_shadowable = false;
	6809	bool is_const = true;
	6810	ZigVar *var = ir_create_var(irb, node, subexpr_scope,
	6811	err_symbol, is_const, is_const, is_shadowable, is_comptime);
	6812
	6813	IrInstSrc *err_ptr = ir_build_unwrap_err_code_src(irb, subexpr_scope, node, err_val_ptr);
	6814	IrInstSrc *err_value = ir_build_load_ptr(irb, subexpr_scope, node, err_ptr);
	6815	build_decl_var_and_init(irb, subexpr_scope, node, var, err_value, buf_ptr(err_symbol), is_comptime);
	6816	err_var_scope = var->child_scope;
	6817	} else {
	6818	err_var_scope = subexpr_scope;
	6819	}
	6820	else_expr_result = ir_gen_node_extra(irb, else_node, err_var_scope, lval, &peer_parent->peers.at(1)->base);
	6821	if (else_expr_result == irb->codegen->invalid_inst_src)
	6822	return else_expr_result;
	6823	} else {
	6824	else_expr_result = ir_build_const_void(irb, scope, node);
	6825	ir_build_end_expr(irb, scope, node, else_expr_result, &peer_parent->peers.at(1)->base);
	6826	}
	6827	IrBasicBlockSrc *after_else_block = irb->current_basic_block;
	6828	if (!instr_is_unreachable(else_expr_result))
	6829	ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
	6830
	6831	ir_set_cursor_at_end_and_append_block(irb, endif_block);
	6832	IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
	6833	incoming_values[0] = then_expr_result;
	6834	incoming_values[1] = else_expr_result;
	6835	IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
	6836	incoming_blocks[0] = after_then_block;
	6837	incoming_blocks[1] = after_else_block;
	6838
	6839	IrInstSrc *phi = ir_build_phi(irb, scope, node, 2, incoming_blocks, incoming_values, peer_parent);
	6840	return ir_expr_wrap(irb, scope, phi, result_loc);
	6841	}
	6842
	6843	static bool ir_gen_switch_prong_expr(IrBuilderSrc irb, Scope scope, AstNode switch_node, AstNode prong_node,
	6844	IrBasicBlockSrc end_block, IrInstSrc is_comptime, IrInstSrc *var_is_comptime,
	6845	IrInstSrc target_value_ptr, IrInstSrc *prong_values, size_t prong_values_len,
	6846	ZigList<IrBasicBlockSrc > incoming_blocks, ZigList<IrInstSrc > incoming_values,
	6847	IrInstSrcSwitchElseVar *out_switch_else_var, LVal lval, ResultLoc result_loc)
	6848	{
	6849	assert(switch_node->type == NodeTypeSwitchExpr);
	6850	assert(prong_node->type == NodeTypeSwitchProng);
	6851
	6852	AstNode *expr_node = prong_node->data.switch_prong.expr;
	6853	AstNode *var_symbol_node = prong_node->data.switch_prong.var_symbol;
	6854	Scope *child_scope;
	6855	if (var_symbol_node) {
	6856	assert(var_symbol_node->type == NodeTypeIdentifier);
	6857	Buf *var_name = node_identifier_buf(var_symbol_node);
	6858	bool var_is_ptr = prong_node->data.switch_prong.var_is_ptr;
	6859
	6860	bool is_shadowable = false;
	6861	bool is_const = true;
	6862	ZigVar *var = ir_create_var(irb, var_symbol_node, scope,
	6863	var_name, is_const, is_const, is_shadowable, var_is_comptime);
	6864	child_scope = var->child_scope;
	6865	IrInstSrc *var_value;
	6866	if (out_switch_else_var != nullptr) {
	6867	IrInstSrcSwitchElseVar *switch_else_var = ir_build_switch_else_var(irb, scope, var_symbol_node,
	6868	target_value_ptr);
	6869	*out_switch_else_var = switch_else_var;
	6870	IrInstSrc *payload_ptr = &switch_else_var->base;
	6871	var_value = var_is_ptr ?
	6872	payload_ptr : ir_build_load_ptr(irb, scope, var_symbol_node, payload_ptr);
	6873	} else if (prong_values != nullptr) {
	6874	IrInstSrc *payload_ptr = ir_build_switch_var(irb, scope, var_symbol_node, target_value_ptr,
	6875	prong_values, prong_values_len);
	6876	var_value = var_is_ptr ?
	6877	payload_ptr : ir_build_load_ptr(irb, scope, var_symbol_node, payload_ptr);
	6878	} else {
	6879	var_value = var_is_ptr ?
	6880	target_value_ptr : ir_build_load_ptr(irb, scope, var_symbol_node, target_value_ptr);
	6881	}
	6882	build_decl_var_and_init(irb, scope, var_symbol_node, var, var_value, buf_ptr(var_name), var_is_comptime);
	6883	} else {
	6884	child_scope = scope;
	6885	}
	6886
	6887	IrInstSrc *expr_result = ir_gen_node_extra(irb, expr_node, child_scope, lval, result_loc);
	6888	if (expr_result == irb->codegen->invalid_inst_src)
	6889	return false;
	6890	if (!instr_is_unreachable(expr_result))
	6891	ir_mark_gen(ir_build_br(irb, scope, switch_node, end_block, is_comptime));
	6892	incoming_blocks->append(irb->current_basic_block);
	6893	incoming_values->append(expr_result);
	6894	return true;
	6895	}
	6896
	6897	static IrInstSrc ir_gen_switch_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
	6898	ResultLoc *result_loc)
	6899	{
	6900	assert(node->type == NodeTypeSwitchExpr);
	6901
	6902	AstNode *target_node = node->data.switch_expr.expr;
	6903	IrInstSrc *target_value_ptr = ir_gen_node_extra(irb, target_node, scope, LValPtr, nullptr);
	6904	if (target_value_ptr == irb->codegen->invalid_inst_src)
	6905	return target_value_ptr;
	6906	IrInstSrc *target_value = ir_build_switch_target(irb, scope, node, target_value_ptr);
	6907
	6908	IrBasicBlockSrc *else_block = ir_create_basic_block(irb, scope, "SwitchElse");
	6909	IrBasicBlockSrc *end_block = ir_create_basic_block(irb, scope, "SwitchEnd");
	6910
	6911	size_t prong_count = node->data.switch_expr.prongs.length;
	6912	ZigList<IrInstSrcSwitchBrCase> cases = {0};
	6913
	6914	IrInstSrc *is_comptime;
	6915	IrInstSrc *var_is_comptime;
	6916	if (ir_should_inline(irb->exec, scope)) {
	6917	is_comptime = ir_build_const_bool(irb, scope, node, true);
	6918	var_is_comptime = is_comptime;
	6919	} else {
	6920	is_comptime = ir_build_test_comptime(irb, scope, node, target_value);
	6921	var_is_comptime = ir_build_test_comptime(irb, scope, node, target_value_ptr);
	6922	}
	6923
	6924	ZigList<IrInstSrc *> incoming_values = {0};
	6925	ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
	6926	ZigList<IrInstSrcCheckSwitchProngsRange> check_ranges = {0};
	6927
	6928	IrInstSrcSwitchElseVar *switch_else_var = nullptr;
	6929
	6930	ResultLocPeerParent *peer_parent = heap::c_allocator.create<ResultLocPeerParent>();
	6931	peer_parent->base.id = ResultLocIdPeerParent;
	6932	peer_parent->base.allow_write_through_const = result_loc->allow_write_through_const;
	6933	peer_parent->end_bb = end_block;
	6934	peer_parent->is_comptime = is_comptime;
	6935	peer_parent->parent = result_loc;
	6936
	6937	ir_build_reset_result(irb, scope, node, &peer_parent->base);
	6938
	6939	// First do the else and the ranges
	6940	Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
	6941	Scope *comptime_scope = create_comptime_scope(irb->codegen, node, scope);
	6942	AstNode *else_prong = nullptr;
	6943	AstNode *underscore_prong = nullptr;
	6944	for (size_t prong_i = 0; prong_i < prong_count; prong_i += 1) {
	6945	AstNode *prong_node = node->data.switch_expr.prongs.at(prong_i);
	6946	size_t prong_item_count = prong_node->data.switch_prong.items.length;
	6947	if (prong_node->data.switch_prong.any_items_are_range) {
	6948	ResultLocPeer *this_peer_result_loc = create_peer_result(peer_parent);
	6949
	6950	IrInstSrc *ok_bit = nullptr;
	6951	AstNode *last_item_node = nullptr;
	6952	for (size_t item_i = 0; item_i < prong_item_count; item_i += 1) {
	6953	AstNode *item_node = prong_node->data.switch_prong.items.at(item_i);
	6954	last_item_node = item_node;
	6955	if (item_node->type == NodeTypeSwitchRange) {
	6956	AstNode *start_node = item_node->data.switch_range.start;
	6957	AstNode *end_node = item_node->data.switch_range.end;
	6958
	6959	IrInstSrc *start_value = ir_gen_node(irb, start_node, comptime_scope);
	6960	if (start_value == irb->codegen->invalid_inst_src)
	6961	return irb->codegen->invalid_inst_src;
	6962
	6963	IrInstSrc *end_value = ir_gen_node(irb, end_node, comptime_scope);
	6964	if (end_value == irb->codegen->invalid_inst_src)
	6965	return irb->codegen->invalid_inst_src;
	6966
	6967	IrInstSrcCheckSwitchProngsRange *check_range = check_ranges.add_one();
	6968	check_range->start = start_value;
	6969	check_range->end = end_value;
	6970
	6971	IrInstSrc *lower_range_ok = ir_build_bin_op(irb, scope, item_node, IrBinOpCmpGreaterOrEq,
	6972	target_value, start_value, false);
	6973	IrInstSrc *upper_range_ok = ir_build_bin_op(irb, scope, item_node, IrBinOpCmpLessOrEq,
	6974	target_value, end_value, false);
	6975	IrInstSrc *both_ok = ir_build_bin_op(irb, scope, item_node, IrBinOpBoolAnd,
	6976	lower_range_ok, upper_range_ok, false);
	6977	if (ok_bit) {
	6978	ok_bit = ir_build_bin_op(irb, scope, item_node, IrBinOpBoolOr, both_ok, ok_bit, false);
	6979	} else {
	6980	ok_bit = both_ok;
	6981	}
	6982	} else {
	6983	IrInstSrc *item_value = ir_gen_node(irb, item_node, comptime_scope);
	6984	if (item_value == irb->codegen->invalid_inst_src)
	6985	return irb->codegen->invalid_inst_src;
	6986
	6987	IrInstSrcCheckSwitchProngsRange *check_range = check_ranges.add_one();
	6988	check_range->start = item_value;
	6989	check_range->end = item_value;
	6990
	6991	IrInstSrc *cmp_ok = ir_build_bin_op(irb, scope, item_node, IrBinOpCmpEq,
	6992	item_value, target_value, false);
	6993	if (ok_bit) {
	6994	ok_bit = ir_build_bin_op(irb, scope, item_node, IrBinOpBoolOr, cmp_ok, ok_bit, false);
	6995	} else {
	6996	ok_bit = cmp_ok;
	6997	}
	6998	}
	6999	}
	7000
	7001	IrBasicBlockSrc *range_block_yes = ir_create_basic_block(irb, scope, "SwitchRangeYes");
	7002	IrBasicBlockSrc *range_block_no = ir_create_basic_block(irb, scope, "SwitchRangeNo");
	7003
	7004	assert(ok_bit);
	7005	assert(last_item_node);
	7006	IrInstSrc *br_inst = ir_mark_gen(ir_build_cond_br(irb, scope, last_item_node, ok_bit,
	7007	range_block_yes, range_block_no, is_comptime));
	7008	if (peer_parent->base.source_instruction == nullptr) {
	7009	peer_parent->base.source_instruction = br_inst;
	7010	}
	7011
	7012	if (peer_parent->peers.length > 0) {
	7013	peer_parent->peers.last()->next_bb = range_block_yes;
	7014	}
	7015	peer_parent->peers.append(this_peer_result_loc);
	7016	ir_set_cursor_at_end_and_append_block(irb, range_block_yes);
	7017	if (!ir_gen_switch_prong_expr(irb, subexpr_scope, node, prong_node, end_block,
	7018	is_comptime, var_is_comptime, target_value_ptr, nullptr, 0,
	7019	&incoming_blocks, &incoming_values, nullptr, LValNone, &this_peer_result_loc->base))
	7020	{
	7021	return irb->codegen->invalid_inst_src;
	7022	}
	7023
	7024	ir_set_cursor_at_end_and_append_block(irb, range_block_no);
	7025	} else {
	7026	if (prong_item_count == 0) {
	7027	if (else_prong) {
	7028	ErrorMsg *msg = add_node_error(irb->codegen, prong_node,
	7029	buf_sprintf("multiple else prongs in switch expression"));
	7030	add_error_note(irb->codegen, msg, else_prong,
	7031	buf_sprintf("previous else prong is here"));
	7032	return irb->codegen->invalid_inst_src;
	7033	}
	7034	else_prong = prong_node;
	7035	} else if (prong_item_count == 1 &&
	7036	prong_node->data.switch_prong.items.at(0)->type == NodeTypeIdentifier &&
	7037	buf_eql_str(node_identifier_buf(prong_node->data.switch_prong.items.at(0)), "_")) {
	7038	if (underscore_prong) {
	7039	ErrorMsg *msg = add_node_error(irb->codegen, prong_node,
	7040	buf_sprintf("multiple '_' prongs in switch expression"));
	7041	add_error_note(irb->codegen, msg, underscore_prong,
	7042	buf_sprintf("previous '_' prong is here"));
	7043	return irb->codegen->invalid_inst_src;
	7044	}
	7045	underscore_prong = prong_node;
	7046	} else {
	7047	continue;
	7048	}
	7049	if (underscore_prong && else_prong) {
	7050	ErrorMsg *msg = add_node_error(irb->codegen, prong_node,
	7051	buf_sprintf("else and '_' prong in switch expression"));
	7052	if (underscore_prong == prong_node)
	7053	add_error_note(irb->codegen, msg, else_prong,
	7054	buf_sprintf("else prong is here"));
	7055	else
	7056	add_error_note(irb->codegen, msg, underscore_prong,
	7057	buf_sprintf("'_' prong is here"));
	7058	return irb->codegen->invalid_inst_src;
	7059	}
	7060	ResultLocPeer *this_peer_result_loc = create_peer_result(peer_parent);
	7061
	7062	IrBasicBlockSrc *prev_block = irb->current_basic_block;
	7063	if (peer_parent->peers.length > 0) {
	7064	peer_parent->peers.last()->next_bb = else_block;
	7065	}
	7066	peer_parent->peers.append(this_peer_result_loc);
	7067	ir_set_cursor_at_end_and_append_block(irb, else_block);
	7068	if (!ir_gen_switch_prong_expr(irb, subexpr_scope, node, prong_node, end_block,
	7069	is_comptime, var_is_comptime, target_value_ptr, nullptr, 0, &incoming_blocks, &incoming_values,
	7070	&switch_else_var, LValNone, &this_peer_result_loc->base))
	7071	{
	7072	return irb->codegen->invalid_inst_src;
	7073	}
	7074	ir_set_cursor_at_end(irb, prev_block);
	7075	}
	7076	}
	7077
	7078	// next do the non-else non-ranges
	7079	for (size_t prong_i = 0; prong_i < prong_count; prong_i += 1) {
	7080	AstNode *prong_node = node->data.switch_expr.prongs.at(prong_i);
	7081	size_t prong_item_count = prong_node->data.switch_prong.items.length;
	7082	if (prong_item_count == 0)
	7083	continue;
	7084	if (prong_node->data.switch_prong.any_items_are_range)
	7085	continue;
	7086	if (underscore_prong == prong_node)
	7087	continue;
	7088
	7089	ResultLocPeer *this_peer_result_loc = create_peer_result(peer_parent);
	7090
	7091	IrBasicBlockSrc *prong_block = ir_create_basic_block(irb, scope, "SwitchProng");
	7092	IrInstSrc *items = heap::c_allocator.allocate<IrInstSrc >(prong_item_count);
	7093
	7094	for (size_t item_i = 0; item_i < prong_item_count; item_i += 1) {
	7095	AstNode *item_node = prong_node->data.switch_prong.items.at(item_i);
	7096	assert(item_node->type != NodeTypeSwitchRange);
	7097
	7098	IrInstSrc *item_value = ir_gen_node(irb, item_node, comptime_scope);
	7099	if (item_value == irb->codegen->invalid_inst_src)
	7100	return irb->codegen->invalid_inst_src;
	7101
	7102	IrInstSrcCheckSwitchProngsRange *check_range = check_ranges.add_one();
	7103	check_range->start = item_value;
	7104	check_range->end = item_value;
	7105
	7106	IrInstSrcSwitchBrCase *this_case = cases.add_one();
	7107	this_case->value = item_value;
	7108	this_case->block = prong_block;
	7109
	7110	items[item_i] = item_value;
	7111	}
	7112
	7113	IrBasicBlockSrc *prev_block = irb->current_basic_block;
	7114	if (peer_parent->peers.length > 0) {
	7115	peer_parent->peers.last()->next_bb = prong_block;
	7116	}
	7117	peer_parent->peers.append(this_peer_result_loc);
	7118	ir_set_cursor_at_end_and_append_block(irb, prong_block);
	7119	if (!ir_gen_switch_prong_expr(irb, subexpr_scope, node, prong_node, end_block,
	7120	is_comptime, var_is_comptime, target_value_ptr, items, prong_item_count,
	7121	&incoming_blocks, &incoming_values, nullptr, LValNone, &this_peer_result_loc->base))
	7122	{
	7123	return irb->codegen->invalid_inst_src;
	7124	}
	7125
	7126	ir_set_cursor_at_end(irb, prev_block);
	7127
	7128	}
	7129
	7130	IrInstSrc *switch_prongs_void = ir_build_check_switch_prongs(irb, scope, node, target_value,
	7131	check_ranges.items, check_ranges.length, else_prong, underscore_prong != nullptr);
	7132
	7133	IrInstSrc *br_instruction;
	7134	if (cases.length == 0) {
	7135	br_instruction = ir_build_br(irb, scope, node, else_block, is_comptime);
	7136	} else {
	7137	IrInstSrcSwitchBr *switch_br = ir_build_switch_br_src(irb, scope, node, target_value, else_block,
	7138	cases.length, cases.items, is_comptime, switch_prongs_void);
	7139	if (switch_else_var != nullptr) {
	7140	switch_else_var->switch_br = switch_br;
	7141	}
	7142	br_instruction = &switch_br->base;
	7143	}
	7144	if (peer_parent->base.source_instruction == nullptr) {
	7145	peer_parent->base.source_instruction = br_instruction;
	7146	}
	7147	for (size_t i = 0; i < peer_parent->peers.length; i += 1) {
	7148	peer_parent->peers.at(i)->base.source_instruction = peer_parent->base.source_instruction;
	7149	}
	7150
	7151	if (!else_prong && !underscore_prong) {
	7152	if (peer_parent->peers.length != 0) {
	7153	peer_parent->peers.last()->next_bb = else_block;
	7154	}
	7155	ir_set_cursor_at_end_and_append_block(irb, else_block);
	7156	ir_build_unreachable(irb, scope, node);
	7157	} else {
	7158	if (peer_parent->peers.length != 0) {
	7159	peer_parent->peers.last()->next_bb = end_block;
	7160	}
	7161	}
	7162
	7163	ir_set_cursor_at_end_and_append_block(irb, end_block);
	7164	assert(incoming_blocks.length == incoming_values.length);
	7165	IrInstSrc *result_instruction;
	7166	if (incoming_blocks.length == 0) {
	7167	result_instruction = ir_build_const_void(irb, scope, node);
	7168	} else {
	7169	result_instruction = ir_build_phi(irb, scope, node, incoming_blocks.length,
	7170	incoming_blocks.items, incoming_values.items, peer_parent);
	7171	}
	7172	return ir_lval_wrap(irb, scope, result_instruction, lval, result_loc);
	7173	}
	7174
	7175	static IrInstSrc ir_gen_comptime(IrBuilderSrc irb, Scope parent_scope, AstNode node, LVal lval) {
	7176	assert(node->type == NodeTypeCompTime);
	7177
	7178	Scope *child_scope = create_comptime_scope(irb->codegen, node, parent_scope);
	7179	// purposefully pass null for result_loc and let EndExpr handle it
	7180	return ir_gen_node_extra(irb, node->data.comptime_expr.expr, child_scope, lval, nullptr);
	7181	}
	7182
	7183	static IrInstSrc ir_gen_nosuspend(IrBuilderSrc irb, Scope parent_scope, AstNode node, LVal lval) {
	7184	assert(node->type == NodeTypeNoSuspend);
	7185
	7186	Scope *child_scope = create_nosuspend_scope(irb->codegen, node, parent_scope);
	7187	// purposefully pass null for result_loc and let EndExpr handle it
	7188	return ir_gen_node_extra(irb, node->data.nosuspend_expr.expr, child_scope, lval, nullptr);
	7189	}
	7190
	7191	static IrInstSrc ir_gen_return_from_block(IrBuilderSrc irb, Scope break_scope, AstNode node, ScopeBlock *block_scope) {
	7192	IrInstSrc *is_comptime;
	7193	if (ir_should_inline(irb->exec, break_scope)) {
	7194	is_comptime = ir_build_const_bool(irb, break_scope, node, true);
	7195	} else {
	7196	is_comptime = block_scope->is_comptime;
	7197	}
	7198
	7199	IrInstSrc *result_value;
	7200	if (node->data.break_expr.expr) {
	7201	ResultLocPeer *peer_result = create_peer_result(block_scope->peer_parent);
	7202	block_scope->peer_parent->peers.append(peer_result);
	7203
	7204	result_value = ir_gen_node_extra(irb, node->data.break_expr.expr, break_scope, block_scope->lval,
	7205	&peer_result->base);
	7206	if (result_value == irb->codegen->invalid_inst_src)
	7207	return irb->codegen->invalid_inst_src;
	7208	} else {
	7209	result_value = ir_build_const_void(irb, break_scope, node);
	7210	}
	7211
	7212	IrBasicBlockSrc *dest_block = block_scope->end_block;
	7213	if (!ir_gen_defers_for_block(irb, break_scope, dest_block->scope, nullptr, nullptr))
	7214	return irb->codegen->invalid_inst_src;
	7215
	7216	block_scope->incoming_blocks->append(irb->current_basic_block);
	7217	block_scope->incoming_values->append(result_value);
	7218	return ir_build_br(irb, break_scope, node, dest_block, is_comptime);
	7219	}
	7220
	7221	static IrInstSrc ir_gen_break(IrBuilderSrc irb, Scope break_scope, AstNode node) {
	7222	assert(node->type == NodeTypeBreak);
	7223
	7224	// Search up the scope. We'll find one of these things first:
	7225	// * function definition scope or global scope => error, break outside loop
	7226	// * defer expression scope => error, cannot break out of defer expression
	7227	// * loop scope => OK
	7228	// * (if it's a labeled break) labeled block => OK
	7229
	7230	Scope *search_scope = break_scope;
	7231	ScopeLoop *loop_scope;
	7232	for (;;) {
	7233	if (search_scope == nullptr \|\| search_scope->id == ScopeIdFnDef) {
	7234	if (node->data.break_expr.name != nullptr) {
	7235	add_node_error(irb->codegen, node, buf_sprintf("label not found: '%s'", buf_ptr(node->data.break_expr.name)));
	7236	return irb->codegen->invalid_inst_src;
	7237	} else {
	7238	add_node_error(irb->codegen, node, buf_sprintf("break expression outside loop"));
	7239	return irb->codegen->invalid_inst_src;
	7240	}
	7241	} else if (search_scope->id == ScopeIdDeferExpr) {
	7242	add_node_error(irb->codegen, node, buf_sprintf("cannot break out of defer expression"));
	7243	return irb->codegen->invalid_inst_src;
	7244	} else if (search_scope->id == ScopeIdLoop) {
	7245	ScopeLoop this_loop_scope = (ScopeLoop )search_scope;
	7246	if (node->data.break_expr.name == nullptr \|\|
	7247	(this_loop_scope->name != nullptr && buf_eql_buf(node->data.break_expr.name, this_loop_scope->name)))
	7248	{
	7249	this_loop_scope->name_used = true;
	7250	loop_scope = this_loop_scope;
	7251	break;
	7252	}
	7253	} else if (search_scope->id == ScopeIdBlock) {
	7254	ScopeBlock this_block_scope = (ScopeBlock )search_scope;
	7255	if (node->data.break_expr.name != nullptr &&
	7256	(this_block_scope->name != nullptr && buf_eql_buf(node->data.break_expr.name, this_block_scope->name)))
	7257	{
	7258	assert(this_block_scope->end_block != nullptr);
	7259	this_block_scope->name_used = true;
	7260	return ir_gen_return_from_block(irb, break_scope, node, this_block_scope);
	7261	}
	7262	} else if (search_scope->id == ScopeIdSuspend) {
	7263	add_node_error(irb->codegen, node, buf_sprintf("cannot break out of suspend block"));
	7264	return irb->codegen->invalid_inst_src;
	7265	}
	7266	search_scope = search_scope->parent;
	7267	}
	7268
	7269	IrInstSrc *is_comptime;
	7270	if (ir_should_inline(irb->exec, break_scope)) {
	7271	is_comptime = ir_build_const_bool(irb, break_scope, node, true);
	7272	} else {
	7273	is_comptime = loop_scope->is_comptime;
	7274	}
	7275
	7276	IrInstSrc *result_value;
	7277	if (node->data.break_expr.expr) {
	7278	ResultLocPeer *peer_result = create_peer_result(loop_scope->peer_parent);
	7279	loop_scope->peer_parent->peers.append(peer_result);
	7280
	7281	result_value = ir_gen_node_extra(irb, node->data.break_expr.expr, break_scope,
	7282	loop_scope->lval, &peer_result->base);
	7283	if (result_value == irb->codegen->invalid_inst_src)
	7284	return irb->codegen->invalid_inst_src;
	7285	} else {
	7286	result_value = ir_build_const_void(irb, break_scope, node);
	7287	}
	7288
	7289	IrBasicBlockSrc *dest_block = loop_scope->break_block;
	7290	if (!ir_gen_defers_for_block(irb, break_scope, dest_block->scope, nullptr, nullptr))
	7291	return irb->codegen->invalid_inst_src;
	7292
	7293	loop_scope->incoming_blocks->append(irb->current_basic_block);
	7294	loop_scope->incoming_values->append(result_value);
	7295	return ir_build_br(irb, break_scope, node, dest_block, is_comptime);
	7296	}
	7297
	7298	static IrInstSrc ir_gen_continue(IrBuilderSrc irb, Scope continue_scope, AstNode node) {
	7299	assert(node->type == NodeTypeContinue);
	7300
	7301	// Search up the scope. We'll find one of these things first:
	7302	// * function definition scope or global scope => error, break outside loop
	7303	// * defer expression scope => error, cannot break out of defer expression
	7304	// * loop scope => OK
	7305
	7306	ZigList<ScopeRuntime *> runtime_scopes = {};
	7307
	7308	Scope *search_scope = continue_scope;
	7309	ScopeLoop *loop_scope;
	7310	for (;;) {
	7311	if (search_scope == nullptr \|\| search_scope->id == ScopeIdFnDef) {
	7312	if (node->data.continue_expr.name != nullptr) {
	7313	add_node_error(irb->codegen, node, buf_sprintf("labeled loop not found: '%s'", buf_ptr(node->data.continue_expr.name)));
	7314	return irb->codegen->invalid_inst_src;
	7315	} else {
	7316	add_node_error(irb->codegen, node, buf_sprintf("continue expression outside loop"));
	7317	return irb->codegen->invalid_inst_src;
	7318	}
	7319	} else if (search_scope->id == ScopeIdDeferExpr) {
	7320	add_node_error(irb->codegen, node, buf_sprintf("cannot continue out of defer expression"));
	7321	return irb->codegen->invalid_inst_src;
	7322	} else if (search_scope->id == ScopeIdLoop) {
	7323	ScopeLoop this_loop_scope = (ScopeLoop )search_scope;
	7324	if (node->data.continue_expr.name == nullptr \|\|
	7325	(this_loop_scope->name != nullptr && buf_eql_buf(node->data.continue_expr.name, this_loop_scope->name)))
	7326	{
	7327	this_loop_scope->name_used = true;
	7328	loop_scope = this_loop_scope;
	7329	break;
	7330	}
	7331	} else if (search_scope->id == ScopeIdRuntime) {
	7332	ScopeRuntime scope_runtime = (ScopeRuntime )search_scope;
	7333	runtime_scopes.append(scope_runtime);
	7334	}
	7335	search_scope = search_scope->parent;
	7336	}
	7337
	7338	IrInstSrc *is_comptime;
	7339	if (ir_should_inline(irb->exec, continue_scope)) {
	7340	is_comptime = ir_build_const_bool(irb, continue_scope, node, true);
	7341	} else {
	7342	is_comptime = loop_scope->is_comptime;
	7343	}
	7344
	7345	for (size_t i = 0; i < runtime_scopes.length; i += 1) {
	7346	ScopeRuntime *scope_runtime = runtime_scopes.at(i);
	7347	ir_mark_gen(ir_build_check_runtime_scope(irb, continue_scope, node, scope_runtime->is_comptime, is_comptime));
	7348	}
	7349	runtime_scopes.deinit();
	7350
	7351	IrBasicBlockSrc *dest_block = loop_scope->continue_block;
	7352	if (!ir_gen_defers_for_block(irb, continue_scope, dest_block->scope, nullptr, nullptr))
	7353	return irb->codegen->invalid_inst_src;
	7354	return ir_mark_gen(ir_build_br(irb, continue_scope, node, dest_block, is_comptime));
	7355	}
	7356
	7357	static IrInstSrc ir_gen_error_type(IrBuilderSrc irb, Scope scope, AstNode node) {
	7358	assert(node->type == NodeTypeErrorType);
	7359	return ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_global_error_set);
	7360	}
	7361
	7362	static IrInstSrc ir_gen_defer(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
	7363	assert(node->type == NodeTypeDefer);
	7364
	7365	ScopeDefer *defer_child_scope = create_defer_scope(irb->codegen, node, parent_scope);
	7366	node->data.defer.child_scope = &defer_child_scope->base;
	7367
	7368	ScopeDeferExpr *defer_expr_scope = create_defer_expr_scope(irb->codegen, node, parent_scope);
	7369	node->data.defer.expr_scope = &defer_expr_scope->base;
	7370
	7371	return ir_build_const_void(irb, parent_scope, node);
	7372	}
	7373
	7374	static IrInstSrc ir_gen_slice(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval, ResultLoc *result_loc) {
	7375	assert(node->type == NodeTypeSliceExpr);
	7376
	7377	AstNodeSliceExpr *slice_expr = &node->data.slice_expr;
	7378	AstNode *array_node = slice_expr->array_ref_expr;
	7379	AstNode *start_node = slice_expr->start;
	7380	AstNode *end_node = slice_expr->end;
	7381	AstNode *sentinel_node = slice_expr->sentinel;
	7382
	7383	IrInstSrc *ptr_value = ir_gen_node_extra(irb, array_node, scope, LValPtr, nullptr);
	7384	if (ptr_value == irb->codegen->invalid_inst_src)
	7385	return irb->codegen->invalid_inst_src;
	7386
	7387	IrInstSrc *start_value = ir_gen_node(irb, start_node, scope);
	7388	if (start_value == irb->codegen->invalid_inst_src)
	7389	return irb->codegen->invalid_inst_src;
	7390
	7391	IrInstSrc *end_value;
	7392	if (end_node) {
	7393	end_value = ir_gen_node(irb, end_node, scope);
	7394	if (end_value == irb->codegen->invalid_inst_src)
	7395	return irb->codegen->invalid_inst_src;
	7396	} else {
	7397	end_value = nullptr;
	7398	}
	7399
	7400	IrInstSrc *sentinel_value;
	7401	if (sentinel_node) {
	7402	sentinel_value = ir_gen_node(irb, sentinel_node, scope);
	7403	if (sentinel_value == irb->codegen->invalid_inst_src)
	7404	return irb->codegen->invalid_inst_src;
	7405	} else {
	7406	sentinel_value = nullptr;
	7407	}
	7408
	7409	IrInstSrc *slice = ir_build_slice_src(irb, scope, node, ptr_value, start_value, end_value,
	7410	sentinel_value, true, result_loc);
	7411	return ir_lval_wrap(irb, scope, slice, lval, result_loc);
	7412	}
	7413
	7414	static IrInstSrc ir_gen_catch(IrBuilderSrc irb, Scope parent_scope, AstNode node, LVal lval,
	7415	ResultLoc *result_loc)
	7416	{
	7417	assert(node->type == NodeTypeCatchExpr);
	7418
	7419	AstNode *op1_node = node->data.unwrap_err_expr.op1;
	7420	AstNode *op2_node = node->data.unwrap_err_expr.op2;
	7421	AstNode *var_node = node->data.unwrap_err_expr.symbol;
	7422
	7423	if (op2_node->type == NodeTypeUnreachable) {
	7424	if (var_node != nullptr) {
	7425	assert(var_node->type == NodeTypeIdentifier);
	7426	Buf *var_name = node_identifier_buf(var_node);
	7427	add_node_error(irb->codegen, var_node, buf_sprintf("unused variable: '%s'", buf_ptr(var_name)));
	7428	return irb->codegen->invalid_inst_src;
	7429	}
	7430	return ir_gen_catch_unreachable(irb, parent_scope, node, op1_node, lval, result_loc);
	7431	}
	7432
	7433
	7434	ScopeExpr *spill_scope = create_expr_scope(irb->codegen, op1_node, parent_scope);
	7435	spill_scope->spill_harder = true;
	7436
	7437	IrInstSrc *err_union_ptr = ir_gen_node_extra(irb, op1_node, &spill_scope->base, LValPtr, nullptr);
	7438	if (err_union_ptr == irb->codegen->invalid_inst_src)
	7439	return irb->codegen->invalid_inst_src;
	7440
	7441	IrInstSrc *is_err = ir_build_test_err_src(irb, parent_scope, node, err_union_ptr, true, false);
	7442
	7443	IrInstSrc *is_comptime;
	7444	if (ir_should_inline(irb->exec, parent_scope)) {
	7445	is_comptime = ir_build_const_bool(irb, parent_scope, node, true);
	7446	} else {
	7447	is_comptime = ir_build_test_comptime(irb, parent_scope, node, is_err);
	7448	}
	7449
	7450	IrBasicBlockSrc *ok_block = ir_create_basic_block(irb, parent_scope, "UnwrapErrOk");
	7451	IrBasicBlockSrc *err_block = ir_create_basic_block(irb, parent_scope, "UnwrapErrError");
	7452	IrBasicBlockSrc *end_block = ir_create_basic_block(irb, parent_scope, "UnwrapErrEnd");
	7453	IrInstSrc *cond_br_inst = ir_build_cond_br(irb, parent_scope, node, is_err, err_block, ok_block, is_comptime);
	7454
	7455	ResultLocPeerParent *peer_parent = ir_build_binary_result_peers(irb, cond_br_inst, ok_block, end_block, result_loc,
	7456	is_comptime);
	7457
	7458	ir_set_cursor_at_end_and_append_block(irb, err_block);
	7459	Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, &spill_scope->base, is_comptime);
	7460	Scope *err_scope;
	7461	if (var_node) {
	7462	assert(var_node->type == NodeTypeIdentifier);
	7463	Buf *var_name = node_identifier_buf(var_node);
	7464	bool is_const = true;
	7465	bool is_shadowable = false;
	7466	ZigVar *var = ir_create_var(irb, node, subexpr_scope, var_name,
	7467	is_const, is_const, is_shadowable, is_comptime);
	7468	err_scope = var->child_scope;
	7469	IrInstSrc *err_ptr = ir_build_unwrap_err_code_src(irb, err_scope, node, err_union_ptr);
	7470	IrInstSrc *err_value = ir_build_load_ptr(irb, err_scope, var_node, err_ptr);
	7471	build_decl_var_and_init(irb, err_scope, var_node, var, err_value, buf_ptr(var_name), is_comptime);
	7472	} else {
	7473	err_scope = subexpr_scope;
	7474	}
	7475	IrInstSrc *err_result = ir_gen_node_extra(irb, op2_node, err_scope, LValNone, &peer_parent->peers.at(0)->base);
	7476	if (err_result == irb->codegen->invalid_inst_src)
	7477	return irb->codegen->invalid_inst_src;
	7478	IrBasicBlockSrc *after_err_block = irb->current_basic_block;
	7479	if (!instr_is_unreachable(err_result))
	7480	ir_mark_gen(ir_build_br(irb, parent_scope, node, end_block, is_comptime));
	7481
	7482	ir_set_cursor_at_end_and_append_block(irb, ok_block);
	7483	IrInstSrc *unwrapped_ptr = ir_build_unwrap_err_payload_src(irb, parent_scope, node, err_union_ptr, false, false);
	7484	IrInstSrc *unwrapped_payload = ir_build_load_ptr(irb, parent_scope, node, unwrapped_ptr);
	7485	ir_build_end_expr(irb, parent_scope, node, unwrapped_payload, &peer_parent->peers.at(1)->base);
	7486	IrBasicBlockSrc *after_ok_block = irb->current_basic_block;
	7487	ir_build_br(irb, parent_scope, node, end_block, is_comptime);
	7488
	7489	ir_set_cursor_at_end_and_append_block(irb, end_block);
	7490	IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
	7491	incoming_values[0] = err_result;
	7492	incoming_values[1] = unwrapped_payload;
	7493	IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
	7494	incoming_blocks[0] = after_err_block;
	7495	incoming_blocks[1] = after_ok_block;
	7496	IrInstSrc *phi = ir_build_phi(irb, parent_scope, node, 2, incoming_blocks, incoming_values, peer_parent);
	7497	return ir_lval_wrap(irb, parent_scope, phi, lval, result_loc);
	7498	}
	7499
	7500	static bool render_instance_name_recursive(CodeGen codegen, Buf name, Scope outer_scope, Scope inner_scope) {
	7501	if (inner_scope == nullptr \|\| inner_scope == outer_scope) return false;
	7502	bool need_comma = render_instance_name_recursive(codegen, name, outer_scope, inner_scope->parent);
	7503	if (inner_scope->id != ScopeIdVarDecl)
	7504	return need_comma;
	7505
	7506	ScopeVarDecl var_scope = (ScopeVarDecl )inner_scope;
	7507	if (need_comma)
	7508	buf_append_char(name, ',');
	7509	// TODO: const ptr reinterpret here to make the var type agree with the value?
	7510	render_const_value(codegen, name, var_scope->var->const_value);
	7511	return true;
	7512	}
	7513
	7514	Buf get_anon_type_name(CodeGen codegen, IrExecutableSrc exec, const char kind_name,
	7515	Scope scope, AstNode source_node, Buf *out_bare_name)
	7516	{
	7517	if (exec != nullptr && exec->name) {
	7518	ZigType *import = get_scope_import(scope);
	7519	Buf *namespace_name = buf_alloc();
	7520	append_namespace_qualification(codegen, namespace_name, import);
	7521	buf_append_buf(namespace_name, exec->name);
	7522	buf_init_from_buf(out_bare_name, exec->name);
	7523	return namespace_name;
	7524	} else if (exec != nullptr && exec->name_fn != nullptr) {
	7525	Buf *name = buf_alloc();
	7526	buf_append_buf(name, &exec->name_fn->symbol_name);
	7527	buf_appendf(name, "(");
	7528	render_instance_name_recursive(codegen, name, &exec->name_fn->fndef_scope->base, exec->begin_scope);
	7529	buf_appendf(name, ")");
	7530	buf_init_from_buf(out_bare_name, name);
	7531	return name;
	7532	} else {
	7533	ZigType *import = get_scope_import(scope);
	7534	Buf *namespace_name = buf_alloc();
	7535	append_namespace_qualification(codegen, namespace_name, import);
	7536	RootStruct *root_struct = source_node->owner->data.structure.root_struct;
	7537	TokenLoc tok_loc = root_struct->token_locs[source_node->main_token];
	7538	buf_appendf(namespace_name, "%s:%u:%u", kind_name,
	7539	tok_loc.line + 1, tok_loc.column + 1);
	7540	buf_init_from_buf(out_bare_name, namespace_name);
	7541	return namespace_name;
	7542	}
	7543	}
	7544
	7545	static IrInstSrc ir_gen_container_decl(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
	7546	assert(node->type == NodeTypeContainerDecl);
	7547
	7548	ContainerKind kind = node->data.container_decl.kind;
	7549	Buf *bare_name = buf_alloc();
	7550	Buf *name = get_anon_type_name(irb->codegen, irb->exec, container_string(kind), parent_scope, node, bare_name);
	7551
	7552	ContainerLayout layout = node->data.container_decl.layout;
	7553	ZigType *container_type = get_partial_container_type(irb->codegen, parent_scope,
	7554	kind, node, buf_ptr(name), bare_name, layout);
	7555	ScopeDecls *child_scope = get_container_scope(container_type);
	7556
	7557	for (size_t i = 0; i < node->data.container_decl.decls.length; i += 1) {
	7558	AstNode *child_node = node->data.container_decl.decls.at(i);
	7559	scan_decls(irb->codegen, child_scope, child_node);
	7560	}
	7561
	7562	TldContainer *tld_container = heap::c_allocator.create<TldContainer>();
	7563	init_tld(&tld_container->base, TldIdContainer, bare_name, VisibModPub, node, parent_scope);
	7564	tld_container->type_entry = container_type;
	7565	tld_container->decls_scope = child_scope;
	7566	irb->codegen->resolve_queue.append(&tld_container->base);
	7567
	7568	// Add this to the list to mark as invalid if analyzing this exec fails.
	7569	irb->exec->tld_list.append(&tld_container->base);
	7570
	7571	return ir_build_const_type(irb, parent_scope, node, container_type);
	7572	}
	7573
	7574	static IrInstSrc ir_gen_err_set_decl(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
	7575	assert(node->type == NodeTypeErrorSetDecl);
	7576
	7577	uint32_t err_count = node->data.err_set_decl.decls.length;
	7578
	7579	Buf bare_name = BUF_INIT;
	7580	Buf *type_name = get_anon_type_name(irb->codegen, irb->exec, "error", parent_scope, node, &bare_name);
	7581	ZigType *err_set_type = new_type_table_entry(ZigTypeIdErrorSet);
	7582	buf_init_from_buf(&err_set_type->name, type_name);
	7583	err_set_type->data.error_set.err_count = err_count;
	7584	err_set_type->size_in_bits = irb->codegen->builtin_types.entry_global_error_set->size_in_bits;
	7585	err_set_type->abi_align = irb->codegen->builtin_types.entry_global_error_set->abi_align;
	7586	err_set_type->abi_size = irb->codegen->builtin_types.entry_global_error_set->abi_size;
	7587	err_set_type->data.error_set.errors = heap::c_allocator.allocate<ErrorTableEntry *>(err_count);
	7588
	7589	size_t errors_count = irb->codegen->errors_by_index.length + err_count;
	7590	ErrorTableEntry *errors = heap::c_allocator.allocate<ErrorTableEntry >(errors_count);
	7591
	7592	for (uint32_t i = 0; i < err_count; i += 1) {
	7593	AstNode *field_node = node->data.err_set_decl.decls.at(i);
	7594	AstNode *symbol_node = ast_field_to_symbol_node(field_node);
	7595	Buf *err_name = node_identifier_buf(symbol_node);
	7596	ErrorTableEntry *err = heap::c_allocator.create<ErrorTableEntry>();
	7597	err->decl_node = field_node;
	7598	buf_init_from_buf(&err->name, err_name);
	7599
	7600	auto existing_entry = irb->codegen->error_table.put_unique(err_name, err);
	7601	if (existing_entry) {
	7602	err->value = existing_entry->value->value;
	7603	} else {
	7604	size_t error_value_count = irb->codegen->errors_by_index.length;
	7605	assert((uint32_t)error_value_count < (((uint32_t)1) << (uint32_t)irb->codegen->err_tag_type->data.integral.bit_count));
	7606	err->value = error_value_count;
	7607	irb->codegen->errors_by_index.append(err);
	7608	}
	7609	err_set_type->data.error_set.errors[i] = err;
	7610
	7611	ErrorTableEntry *prev_err = errors[err->value];
	7612	if (prev_err != nullptr) {
	7613	ErrorMsg *msg = add_node_error(irb->codegen, ast_field_to_symbol_node(err->decl_node),
	7614	buf_sprintf("duplicate error: '%s'", buf_ptr(&err->name)));
	7615	add_error_note(irb->codegen, msg, ast_field_to_symbol_node(prev_err->decl_node),
	7616	buf_sprintf("other error here"));
	7617	return irb->codegen->invalid_inst_src;
	7618	}
	7619	errors[err->value] = err;
	7620	}
	7621	heap::c_allocator.deallocate(errors, errors_count);
	7622	return ir_build_const_type(irb, parent_scope, node, err_set_type);
	7623	}
	7624
	7625	static IrInstSrc ir_gen_fn_proto(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
	7626	assert(node->type == NodeTypeFnProto);
	7627
	7628	size_t param_count = node->data.fn_proto.params.length;
	7629	IrInstSrc *param_types = heap::c_allocator.allocate<IrInstSrc>(param_count);
	7630
	7631	bool is_var_args = false;
	7632	for (size_t i = 0; i < param_count; i += 1) {
	7633	AstNode *param_node = node->data.fn_proto.params.at(i);
	7634	if (param_node->data.param_decl.is_var_args) {
	7635	is_var_args = true;
	7636	break;
	7637	}
	7638	if (param_node->data.param_decl.anytype_token == 0) {
	7639	AstNode *type_node = param_node->data.param_decl.type;
	7640	IrInstSrc *type_value = ir_gen_node(irb, type_node, parent_scope);
	7641	if (type_value == irb->codegen->invalid_inst_src)
	7642	return irb->codegen->invalid_inst_src;
	7643	param_types[i] = type_value;
	7644	} else {
	7645	param_types[i] = nullptr;
	7646	}
	7647	}
	7648
	7649	IrInstSrc *align_value = nullptr;
	7650	if (node->data.fn_proto.align_expr != nullptr) {
	7651	align_value = ir_gen_node(irb, node->data.fn_proto.align_expr, parent_scope);
	7652	if (align_value == irb->codegen->invalid_inst_src)
	7653	return irb->codegen->invalid_inst_src;
	7654	}
	7655
	7656	IrInstSrc *callconv_value = nullptr;
	7657	if (node->data.fn_proto.callconv_expr != nullptr) {
	7658	callconv_value = ir_gen_node(irb, node->data.fn_proto.callconv_expr, parent_scope);
	7659	if (callconv_value == irb->codegen->invalid_inst_src)
	7660	return irb->codegen->invalid_inst_src;
	7661	}
	7662
	7663	IrInstSrc *return_type;
	7664	if (node->data.fn_proto.return_type == nullptr) {
	7665	return_type = ir_build_const_type(irb, parent_scope, node, irb->codegen->builtin_types.entry_void);
	7666	} else {
	7667	return_type = ir_gen_node(irb, node->data.fn_proto.return_type, parent_scope);
	7668	if (return_type == irb->codegen->invalid_inst_src)
	7669	return irb->codegen->invalid_inst_src;
	7670	}
	7671
	7672	return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, callconv_value, return_type, is_var_args);
	7673	}
	7674
	7675	static IrInstSrc ir_gen_resume(IrBuilderSrc irb, Scope scope, AstNode node) {
	7676	assert(node->type == NodeTypeResume);
	7677
	7678	IrInstSrc *target_inst = ir_gen_node_extra(irb, node->data.resume_expr.expr, scope, LValPtr, nullptr);
	7679	if (target_inst == irb->codegen->invalid_inst_src)
	7680	return irb->codegen->invalid_inst_src;
	7681
	7682	return ir_build_resume_src(irb, scope, node, target_inst);
	7683	}
	7684
	7685	static IrInstSrc ir_gen_await_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
	7686	ResultLoc *result_loc)
	7687	{
	7688	assert(node->type == NodeTypeAwaitExpr);
	7689
	7690	bool is_nosuspend = get_scope_nosuspend(scope) != nullptr;
	7691
	7692	AstNode *expr_node = node->data.await_expr.expr;
	7693	if (expr_node->type == NodeTypeFnCallExpr && expr_node->data.fn_call_expr.modifier == CallModifierBuiltin) {
	7694	AstNode *fn_ref_expr = expr_node->data.fn_call_expr.fn_ref_expr;
	7695	Buf *name = node_identifier_buf(fn_ref_expr);
	7696	auto entry = irb->codegen->builtin_fn_table.maybe_get(name);
	7697	if (entry != nullptr) {
	7698	BuiltinFnEntry *builtin_fn = entry->value;
	7699	if (builtin_fn->id == BuiltinFnIdAsyncCall) {
	7700	return ir_gen_async_call(irb, scope, node, expr_node, lval, result_loc);
	7701	}
	7702	}
	7703	}
	7704
	7705	ZigFn *fn_entry = exec_fn_entry(irb->exec);
	7706	if (!fn_entry) {
	7707	add_node_error(irb->codegen, node, buf_sprintf("await outside function definition"));
	7708	return irb->codegen->invalid_inst_src;
	7709	}
	7710	ScopeSuspend *existing_suspend_scope = get_scope_suspend(scope);
	7711	if (existing_suspend_scope) {
	7712	if (!existing_suspend_scope->reported_err) {
	7713	ErrorMsg *msg = add_node_error(irb->codegen, node, buf_sprintf("cannot await inside suspend block"));
	7714	add_error_note(irb->codegen, msg, existing_suspend_scope->base.source_node, buf_sprintf("suspend block here"));
	7715	existing_suspend_scope->reported_err = true;
	7716	}
	7717	return irb->codegen->invalid_inst_src;
	7718	}
	7719
	7720	IrInstSrc *target_inst = ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr);
	7721	if (target_inst == irb->codegen->invalid_inst_src)
	7722	return irb->codegen->invalid_inst_src;
	7723
	7724	IrInstSrc *await_inst = ir_build_await_src(irb, scope, node, target_inst, result_loc, is_nosuspend);
	7725	return ir_lval_wrap(irb, scope, await_inst, lval, result_loc);
	7726	}
	7727
	7728	static IrInstSrc ir_gen_suspend(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
	7729	assert(node->type == NodeTypeSuspend);
	7730
	7731	ZigFn *fn_entry = exec_fn_entry(irb->exec);
	7732	if (!fn_entry) {
	7733	add_node_error(irb->codegen, node, buf_sprintf("suspend outside function definition"));
	7734	return irb->codegen->invalid_inst_src;
	7735	}
	7736	if (get_scope_nosuspend(parent_scope) != nullptr) {
	7737	add_node_error(irb->codegen, node, buf_sprintf("suspend in nosuspend scope"));
	7738	return irb->codegen->invalid_inst_src;
	7739	}
	7740
	7741	ScopeSuspend *existing_suspend_scope = get_scope_suspend(parent_scope);
	7742	if (existing_suspend_scope) {
	7743	if (!existing_suspend_scope->reported_err) {
	7744	ErrorMsg *msg = add_node_error(irb->codegen, node, buf_sprintf("cannot suspend inside suspend block"));
	7745	add_error_note(irb->codegen, msg, existing_suspend_scope->base.source_node, buf_sprintf("other suspend block here"));
	7746	existing_suspend_scope->reported_err = true;
	7747	}
	7748	return irb->codegen->invalid_inst_src;
	7749	}
	7750
	7751	IrInstSrcSuspendBegin *begin = ir_build_suspend_begin_src(irb, parent_scope, node);
	7752	ScopeSuspend *suspend_scope = create_suspend_scope(irb->codegen, node, parent_scope);
	7753	Scope *child_scope = &suspend_scope->base;
	7754	IrInstSrc *susp_res = ir_gen_node(irb, node->data.suspend.block, child_scope);
	7755	if (susp_res == irb->codegen->invalid_inst_src)
	7756	return irb->codegen->invalid_inst_src;
	7757	ir_mark_gen(ir_build_check_statement_is_void(irb, child_scope, node->data.suspend.block, susp_res));
	7758
	7759	return ir_mark_gen(ir_build_suspend_finish_src(irb, parent_scope, node, begin));
	7760	}
	7761
	7762	static IrInstSrc ir_gen_node_raw(IrBuilderSrc irb, AstNode node, Scope scope,
	7763	LVal lval, ResultLoc *result_loc)
	7764	{
	7765	assert(scope);
	7766	switch (node->type) {
	7767	case NodeTypeStructValueField:
	7768	case NodeTypeParamDecl:
	7769	case NodeTypeUsingNamespace:
	7770	case NodeTypeSwitchProng:
	7771	case NodeTypeSwitchRange:
	7772	case NodeTypeStructField:
	7773	case NodeTypeErrorSetField:
	7774	case NodeTypeFnDef:
	7775	case NodeTypeTestDecl:
	7776	zig_unreachable();
	7777	case NodeTypeBlock:
	7778	return ir_gen_block(irb, scope, node, lval, result_loc);
	7779	case NodeTypeGroupedExpr:
	7780	return ir_gen_node_raw(irb, node->data.grouped_expr, scope, lval, result_loc);
	7781	case NodeTypeBinOpExpr:
	7782	return ir_gen_bin_op(irb, scope, node, lval, result_loc);
	7783	case NodeTypeIntLiteral:
	7784	return ir_lval_wrap(irb, scope, ir_gen_int_lit(irb, scope, node), lval, result_loc);
	7785	case NodeTypeFloatLiteral:
	7786	return ir_lval_wrap(irb, scope, ir_gen_float_lit(irb, scope, node), lval, result_loc);
	7787	case NodeTypeCharLiteral:
	7788	return ir_lval_wrap(irb, scope, ir_gen_char_lit(irb, scope, node), lval, result_loc);
	7789	case NodeTypeIdentifier:
	7790	return ir_gen_symbol(irb, scope, node, lval, result_loc);
	7791	case NodeTypeFnCallExpr:
	7792	return ir_gen_fn_call(irb, scope, node, lval, result_loc);
	7793	case NodeTypeIfBoolExpr:
	7794	return ir_gen_if_bool_expr(irb, scope, node, lval, result_loc);
	7795	case NodeTypePrefixOpExpr:
	7796	return ir_gen_prefix_op_expr(irb, scope, node, lval, result_loc);
	7797	case NodeTypeContainerInitExpr:
	7798	return ir_gen_container_init_expr(irb, scope, node, lval, result_loc);
	7799	case NodeTypeVariableDeclaration:
	7800	return ir_gen_var_decl(irb, scope, node);
	7801	case NodeTypeWhileExpr:
	7802	return ir_gen_while_expr(irb, scope, node, lval, result_loc);
	7803	case NodeTypeForExpr:
	7804	return ir_gen_for_expr(irb, scope, node, lval, result_loc);
	7805	case NodeTypeArrayAccessExpr:
	7806	return ir_gen_array_access(irb, scope, node, lval, result_loc);
	7807	case NodeTypeReturnExpr:
	7808	return ir_gen_return(irb, scope, node, lval, result_loc);
	7809	case NodeTypeFieldAccessExpr:
	7810	{
	7811	IrInstSrc *ptr_instruction = ir_gen_field_access(irb, scope, node);
	7812	if (ptr_instruction == irb->codegen->invalid_inst_src)
	7813	return ptr_instruction;
	7814	if (lval == LValPtr \|\| lval == LValAssign)
	7815	return ptr_instruction;
	7816
	7817	IrInstSrc *load_ptr = ir_build_load_ptr(irb, scope, node, ptr_instruction);
	7818	return ir_expr_wrap(irb, scope, load_ptr, result_loc);
	7819	}
	7820	case NodeTypePtrDeref: {
	7821	AstNode *expr_node = node->data.ptr_deref_expr.target;
	7822
	7823	LVal child_lval = lval;
	7824	if (child_lval == LValAssign)
	7825	child_lval = LValPtr;
	7826
	7827	IrInstSrc *value = ir_gen_node_extra(irb, expr_node, scope, child_lval, nullptr);
	7828	if (value == irb->codegen->invalid_inst_src)
	7829	return value;
	7830
	7831	// We essentially just converted any lvalue from &(x.) to (&x).;
	7832	// this inhibits checking that x is a pointer later, so we directly
	7833	// record whether the pointer check is needed
	7834	IrInstSrc *un_op = ir_build_un_op_lval(irb, scope, node, IrUnOpDereference, value, lval, result_loc);
	7835	return ir_expr_wrap(irb, scope, un_op, result_loc);
	7836	}
	7837	case NodeTypeUnwrapOptional: {
	7838	AstNode *expr_node = node->data.unwrap_optional.expr;
	7839
	7840	IrInstSrc *maybe_ptr = ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr);
	7841	if (maybe_ptr == irb->codegen->invalid_inst_src)
	7842	return irb->codegen->invalid_inst_src;
	7843
	7844	IrInstSrc *unwrapped_ptr = ir_build_optional_unwrap_ptr(irb, scope, node, maybe_ptr, true );
	7845	if (lval == LValPtr \|\| lval == LValAssign)
	7846	return unwrapped_ptr;
	7847
	7848	IrInstSrc *load_ptr = ir_build_load_ptr(irb, scope, node, unwrapped_ptr);
	7849	return ir_expr_wrap(irb, scope, load_ptr, result_loc);
	7850	}
	7851	case NodeTypeBoolLiteral:
	7852	return ir_lval_wrap(irb, scope, ir_gen_bool_literal(irb, scope, node), lval, result_loc);
	7853	case NodeTypeArrayType:
	7854	return ir_lval_wrap(irb, scope, ir_gen_array_type(irb, scope, node), lval, result_loc);
	7855	case NodeTypePointerType:
	7856	return ir_lval_wrap(irb, scope, ir_gen_pointer_type(irb, scope, node), lval, result_loc);
	7857	case NodeTypeAnyFrameType:
	7858	return ir_lval_wrap(irb, scope, ir_gen_anyframe_type(irb, scope, node), lval, result_loc);
	7859	case NodeTypeStringLiteral:
	7860	return ir_lval_wrap(irb, scope, ir_gen_string_literal(irb, scope, node), lval, result_loc);
	7861	case NodeTypeUndefinedLiteral:
	7862	return ir_lval_wrap(irb, scope, ir_gen_undefined_literal(irb, scope, node), lval, result_loc);
	7863	case NodeTypeAsmExpr:
	7864	return ir_lval_wrap(irb, scope, ir_gen_asm_expr(irb, scope, node), lval, result_loc);
	7865	case NodeTypeNullLiteral:
	7866	return ir_lval_wrap(irb, scope, ir_gen_null_literal(irb, scope, node), lval, result_loc);
	7867	case NodeTypeIfErrorExpr:
	7868	return ir_gen_if_err_expr(irb, scope, node, lval, result_loc);
	7869	case NodeTypeIfOptional:
	7870	return ir_gen_if_optional_expr(irb, scope, node, lval, result_loc);
	7871	case NodeTypeSwitchExpr:
	7872	return ir_gen_switch_expr(irb, scope, node, lval, result_loc);
	7873	case NodeTypeCompTime:
	7874	return ir_expr_wrap(irb, scope, ir_gen_comptime(irb, scope, node, lval), result_loc);
	7875	case NodeTypeNoSuspend:
	7876	return ir_expr_wrap(irb, scope, ir_gen_nosuspend(irb, scope, node, lval), result_loc);
	7877	case NodeTypeErrorType:
	7878	return ir_lval_wrap(irb, scope, ir_gen_error_type(irb, scope, node), lval, result_loc);
	7879	case NodeTypeBreak:
	7880	return ir_lval_wrap(irb, scope, ir_gen_break(irb, scope, node), lval, result_loc);
	7881	case NodeTypeContinue:
	7882	return ir_lval_wrap(irb, scope, ir_gen_continue(irb, scope, node), lval, result_loc);
	7883	case NodeTypeUnreachable:
	7884	return ir_build_unreachable(irb, scope, node);
	7885	case NodeTypeDefer:
	7886	return ir_lval_wrap(irb, scope, ir_gen_defer(irb, scope, node), lval, result_loc);
	7887	case NodeTypeSliceExpr:
	7888	return ir_gen_slice(irb, scope, node, lval, result_loc);
	7889	case NodeTypeCatchExpr:
	7890	return ir_gen_catch(irb, scope, node, lval, result_loc);
	7891	case NodeTypeContainerDecl:
	7892	return ir_lval_wrap(irb, scope, ir_gen_container_decl(irb, scope, node), lval, result_loc);
	7893	case NodeTypeFnProto:
	7894	return ir_lval_wrap(irb, scope, ir_gen_fn_proto(irb, scope, node), lval, result_loc);
	7895	case NodeTypeErrorSetDecl:
	7896	return ir_lval_wrap(irb, scope, ir_gen_err_set_decl(irb, scope, node), lval, result_loc);
	7897	case NodeTypeResume:
	7898	return ir_lval_wrap(irb, scope, ir_gen_resume(irb, scope, node), lval, result_loc);
	7899	case NodeTypeAwaitExpr:
	7900	return ir_gen_await_expr(irb, scope, node, lval, result_loc);
	7901	case NodeTypeSuspend:
	7902	return ir_lval_wrap(irb, scope, ir_gen_suspend(irb, scope, node), lval, result_loc);
	7903	case NodeTypeEnumLiteral:
	7904	return ir_lval_wrap(irb, scope, ir_gen_enum_literal(irb, scope, node), lval, result_loc);
	7905	case NodeTypeInferredArrayType:
	7906	add_node_error(irb->codegen, node,
	7907	buf_sprintf("inferred array size invalid here"));
	7908	return irb->codegen->invalid_inst_src;
	7909	case NodeTypeAnyTypeField:
	7910	return ir_lval_wrap(irb, scope,
	7911	ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_anytype), lval, result_loc);
	7912	}
	7913	zig_unreachable();
	7914	}
	7915
	7916	ResultLoc *no_result_loc(void) {
	7917	ResultLocNone *result_loc_none = heap::c_allocator.create<ResultLocNone>();
	7918	result_loc_none->base.id = ResultLocIdNone;
	7919	return &result_loc_none->base;
	7920	}
	7921
	7922	static IrInstSrc ir_gen_node_extra(IrBuilderSrc irb, AstNode node, Scope scope, LVal lval,
	7923	ResultLoc *result_loc)
	7924	{
	7925	if (lval == LValAssign) {
	7926	switch (node->type) {
	7927	case NodeTypeStructValueField:
	7928	case NodeTypeParamDecl:
	7929	case NodeTypeUsingNamespace:
	7930	case NodeTypeSwitchProng:
	7931	case NodeTypeSwitchRange:
	7932	case NodeTypeStructField:
	7933	case NodeTypeErrorSetField:
	7934	case NodeTypeFnDef:
	7935	case NodeTypeTestDecl:
	7936	zig_unreachable();
	7937
	7938	// cannot be assigned to
	7939	case NodeTypeBlock:
	7940	case NodeTypeGroupedExpr:
	7941	case NodeTypeBinOpExpr:
	7942	case NodeTypeIntLiteral:
	7943	case NodeTypeFloatLiteral:
	7944	case NodeTypeCharLiteral:
	7945	case NodeTypeIfBoolExpr:
	7946	case NodeTypeContainerInitExpr:
	7947	case NodeTypeVariableDeclaration:
	7948	case NodeTypeWhileExpr:
	7949	case NodeTypeForExpr:
	7950	case NodeTypeReturnExpr:
	7951	case NodeTypeBoolLiteral:
	7952	case NodeTypeArrayType:
	7953	case NodeTypePointerType:
	7954	case NodeTypeAnyFrameType:
	7955	case NodeTypeStringLiteral:
	7956	case NodeTypeUndefinedLiteral:
	7957	case NodeTypeAsmExpr:
	7958	case NodeTypeNullLiteral:
	7959	case NodeTypeIfErrorExpr:
	7960	case NodeTypeIfOptional:
	7961	case NodeTypeSwitchExpr:
	7962	case NodeTypeCompTime:
	7963	case NodeTypeNoSuspend:
	7964	case NodeTypeErrorType:
	7965	case NodeTypeBreak:
	7966	case NodeTypeContinue:
	7967	case NodeTypeUnreachable:
	7968	case NodeTypeDefer:
	7969	case NodeTypeSliceExpr:
	7970	case NodeTypeCatchExpr:
	7971	case NodeTypeContainerDecl:
	7972	case NodeTypeFnProto:
	7973	case NodeTypeErrorSetDecl:
	7974	case NodeTypeResume:
	7975	case NodeTypeAwaitExpr:
	7976	case NodeTypeSuspend:
	7977	case NodeTypeEnumLiteral:
	7978	case NodeTypeInferredArrayType:
	7979	case NodeTypeAnyTypeField:
	7980	case NodeTypePrefixOpExpr:
	7981	add_node_error(irb->codegen, node,
	7982	buf_sprintf("invalid left-hand side to assignment"));
	7983	return irb->codegen->invalid_inst_src;
	7984
	7985	// @field can be assigned to
	7986	case NodeTypeFnCallExpr:
	7987	if (node->data.fn_call_expr.modifier == CallModifierBuiltin) {
	7988	AstNode *fn_ref_expr = node->data.fn_call_expr.fn_ref_expr;
	7989	Buf *name = node_identifier_buf(fn_ref_expr);
	7990	auto entry = irb->codegen->builtin_fn_table.maybe_get(name);
	7991
	7992	if (!entry) {
	7993	add_node_error(irb->codegen, node,
	7994	buf_sprintf("invalid builtin function: '%s'", buf_ptr(name)));
	7995	return irb->codegen->invalid_inst_src;
	7996	}
	7997
	7998	if (entry->value->id == BuiltinFnIdField) {
	7999	break;
	8000	}
	8001	}
	8002	add_node_error(irb->codegen, node,
	8003	buf_sprintf("invalid left-hand side to assignment"));
	8004	return irb->codegen->invalid_inst_src;
	8005
	8006
	8007	// can be assigned to
	8008	case NodeTypeUnwrapOptional:
	8009	case NodeTypePtrDeref:
	8010	case NodeTypeFieldAccessExpr:
	8011	case NodeTypeArrayAccessExpr:
	8012	case NodeTypeIdentifier:
	8013	break;
	8014	}
	8015	}
	8016	if (result_loc == nullptr) {
	8017	// Create a result location indicating there is none - but if one gets created
	8018	// it will be properly distributed.
	8019	result_loc = no_result_loc();
	8020	ir_build_reset_result(irb, scope, node, result_loc);
	8021	}
	8022	Scope *child_scope;
	8023	if (irb->exec->is_inline \|\|
	8024	(irb->exec->fn_entry != nullptr && irb->exec->fn_entry->child_scope == scope))
	8025	{
	8026	child_scope = scope;
	8027	} else {
	8028	child_scope = &create_expr_scope(irb->codegen, node, scope)->base;
	8029	}
	8030	IrInstSrc *result = ir_gen_node_raw(irb, node, child_scope, lval, result_loc);
	8031	if (result == irb->codegen->invalid_inst_src) {
	8032	if (irb->exec->first_err_trace_msg == nullptr) {
	8033	irb->exec->first_err_trace_msg = irb->codegen->trace_err;
	8034	}
	8035	}
	8036	return result;
	8037	}
	8038
	8039	static IrInstSrc ir_gen_node(IrBuilderSrc irb, AstNode node, Scope scope) {
	8040	return ir_gen_node_extra(irb, node, scope, LValNone, nullptr);
	8041	}
	8042
	8043	bool ir_gen(CodeGen codegen, AstNode node, Scope scope, IrExecutableSrc ir_executable) {
	8044	assert(node->owner);
	8045
	8046	IrBuilderSrc ir_builder = {0};
	8047	IrBuilderSrc *irb = &ir_builder;
	8048
	8049	irb->codegen = codegen;
	8050	irb->exec = ir_executable;
	8051	irb->main_block_node = node;
	8052
	8053	IrBasicBlockSrc *entry_block = ir_create_basic_block(irb, scope, "Entry");
	8054	ir_set_cursor_at_end_and_append_block(irb, entry_block);
	8055	// Entry block gets a reference because we enter it to begin.
	8056	ir_ref_bb(irb->current_basic_block);
	8057
	8058	IrInstSrc *result = ir_gen_node_extra(irb, node, scope, LValNone, nullptr);
	8059
	8060	if (result == irb->codegen->invalid_inst_src)
	8061	return false;
	8062
	8063	if (irb->exec->first_err_trace_msg != nullptr) {
	8064	codegen->trace_err = irb->exec->first_err_trace_msg;
	8065	return false;
	8066	}
	8067
	8068	if (!instr_is_unreachable(result)) {
	8069	ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, result->base.source_node, result, nullptr));
	8070	// no need for save_err_ret_addr because this cannot return error
	8071	ResultLocReturn *result_loc_ret = heap::c_allocator.create<ResultLocReturn>();
	8072	result_loc_ret->base.id = ResultLocIdReturn;
	8073	ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
	8074	ir_mark_gen(ir_build_end_expr(irb, scope, node, result, &result_loc_ret->base));
	8075	ir_mark_gen(ir_build_return_src(irb, scope, result->base.source_node, result));
	8076	}
	8077
	8078	return true;
	8079	}
	8080
	8081	bool ir_gen_fn(CodeGen codegen, ZigFn fn_entry) {
	8082	assert(fn_entry);
	8083
	8084	IrExecutableSrc *ir_executable = fn_entry->ir_executable;
	8085	AstNode *body_node = fn_entry->body_node;
	8086
	8087	assert(fn_entry->child_scope);
	8088
	8089	return ir_gen(codegen, body_node, fn_entry->child_scope, ir_executable);
	8090	}
	8091
	8092	void invalidate_exec(IrExecutableSrc exec, ErrorMsg msg) {
	8093	if (exec->first_err_trace_msg != nullptr)
	8094	return;
	8095
	8096	exec->first_err_trace_msg = msg;
	8097
	8098	for (size_t i = 0; i < exec->tld_list.length; i += 1) {
	8099	exec->tld_list.items[i]->resolution = TldResolutionInvalid;
	8100	}
	8101	}
	8102
	8103	AstNode ast_field_to_symbol_node(AstNode err_set_field_node) {
	8104	if (err_set_field_node->type == NodeTypeIdentifier) {
	8105	return err_set_field_node;
	8106	} else if (err_set_field_node->type == NodeTypeErrorSetField) {
	8107	assert(err_set_field_node->data.err_set_field.field_name->type == NodeTypeIdentifier);
	8108	return err_set_field_node->data.err_set_field.field_name;
	8109	} else {
	8110	return err_set_field_node;
	8111	}
	8112	}
	8113
	8114	void ir_add_call_stack_errors_gen(CodeGen codegen, IrExecutableGen exec, ErrorMsg *err_msg, int limit) {
	8115	if (!exec \|\| !exec->source_node \|\| limit < 0) return;
	8116	add_error_note(codegen, err_msg, exec->source_node, buf_sprintf("called from here"));
	8117
	8118	ir_add_call_stack_errors_gen(codegen, exec->parent_exec, err_msg, limit - 1);
	8119	}
	8120

src/stage1/astgen.hpp created+43

...	...	@@ -0,0 +1,43 @@
	1	/*
	2	* Copyright (c) 2021 Andrew Kelley
	3	*
	4	* This file is part of zig, which is MIT licensed.
	5	* See http://opensource.org/licenses/MIT
	6	*/
	7
	8	#ifndef ZIG_ASTGEN_HPP
	9	#define ZIG_ASTGEN_HPP
	10
	11	#include "all_types.hpp"
	12
	13	bool ir_gen(CodeGen g, AstNode node, Scope scope, IrExecutableSrc ir_executable);
	14	bool ir_gen_fn(CodeGen g, ZigFn fn_entry);
	15
	16	bool ir_inst_src_has_side_effects(IrInstSrc *inst);
	17
	18	ZigVar create_local_var(CodeGen codegen, AstNode node, Scope parent_scope,
	19	Buf name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstSrc is_comptime,
	20	bool skip_name_check);
	21
	22	ResultLoc *no_result_loc(void);
	23
	24	void invalidate_exec(IrExecutableSrc exec, ErrorMsg msg);
	25
	26	AstNode ast_field_to_symbol_node(AstNode err_set_field_node);
	27	void ir_add_call_stack_errors_gen(CodeGen codegen, IrExecutableGen exec, ErrorMsg *err_msg,
	28	int limit);
	29
	30	void destroy_instruction_src(IrInstSrc *inst);
	31
	32	struct IrBuilderSrc {
	33	CodeGen *codegen;
	34	IrExecutableSrc *exec;
	35	IrBasicBlockSrc *current_basic_block;
	36	AstNode *main_block_node;
	37	};
	38
	39	bool ir_should_inline(IrExecutableSrc exec, Scope scope);
	40	Buf get_anon_type_name(CodeGen codegen, IrExecutableSrc exec, const char kind_name,
	41	Scope scope, AstNode source_node, Buf *out_bare_name);
	42
	43	#endif

src/stage1/bigfloat.cpp+1-2

...	...	@@ -69,7 +69,7 @@ void bigfloat_init_bigint(BigFloat dest, const BigInt op) {
69	69	}
70	70	}
71	71
72		Error bigfloat_init_buf(BigFloat dest, const uint8_t buf_ptr, size_t buf_len) {
	72	Error bigfloat_init_buf(BigFloat dest, const uint8_t buf_ptr) {
73	73	char str_begin = (char )buf_ptr;
74	74	char *str_end;
75	75
...	...	@@ -79,7 +79,6 @@ Error bigfloat_init_buf(BigFloat dest, const uint8_t buf_ptr, size_t buf_len)
79	79	return ErrorOverflow;
80	80	}
81	81
82		assert(str_end <= ((char*)buf_ptr) + buf_len);
83	82	return ErrorNone;
84	83	}
85	84

src/stage1/bigfloat.hpp+1-1

...	...	@@ -28,7 +28,7 @@ void bigfloat_init_64(BigFloat *dest, double x);
28	28	void bigfloat_init_128(BigFloat *dest, float128_t x);
29	29	void bigfloat_init_bigfloat(BigFloat dest, const BigFloat x);
30	30	void bigfloat_init_bigint(BigFloat dest, const BigInt op);
31		Error bigfloat_init_buf(BigFloat dest, const uint8_t buf_ptr, size_t buf_len);
	31	Error bigfloat_init_buf(BigFloat dest, const uint8_t buf_ptr);
32	32
33	33	float16_t bigfloat_to_f16(const BigFloat *bigfloat);
34	34	float bigfloat_to_f32(const BigFloat *bigfloat);

src/stage1/codegen.cpp+28-16

...	...	@@ -6,7 +6,6 @@
6	6	*/
7	7
8	8	#include "analyze.hpp"
9		#include "ast_render.hpp"
10	9	#include "codegen.hpp"
11	10	#include "errmsg.hpp"
12	11	#include "error.hpp"
...	...	@@ -642,6 +641,18 @@ static LLVMValueRef fn_llvm_value(CodeGen g, ZigFn fn) {
642	641	return fn->llvm_value;
643	642	}
644	643
	644	static uint32_t node_line_onebased(AstNode *node) {
	645	RootStruct *root_struct = node->owner->data.structure.root_struct;
	646	assert(node->main_token < root_struct->token_count);
	647	return root_struct->token_locs[node->main_token].line + 1;
	648	}
	649
	650	static uint32_t node_column_onebased(AstNode *node) {
	651	RootStruct *root_struct = node->owner->data.structure.root_struct;
	652	assert(node->main_token < root_struct->token_count);
	653	return root_struct->token_locs[node->main_token].column + 1;
	654	}
	655
645	656	static ZigLLVMDIScope get_di_scope(CodeGen g, Scope *scope) {
646	657	if (scope->di_scope)
647	658	return scope->di_scope;
...	...	@@ -657,8 +668,7 @@ static ZigLLVMDIScope get_di_scope(CodeGen g, Scope *scope) {
657	668	ZigFn *fn_table_entry = fn_scope->fn_entry;
658	669	if (!fn_table_entry->proto_node)
659	670	return get_di_scope(g, scope->parent);
660		unsigned line_number = (unsigned)(fn_table_entry->proto_node->line == 0) ?
661		0 : (fn_table_entry->proto_node->line + 1);
	671	unsigned line_number = node_line_onebased(fn_table_entry->proto_node);
662	672	unsigned scope_line = line_number;
663	673	bool is_definition = fn_table_entry->body_node != nullptr;
664	674	bool is_optimized = g->build_mode != BuildModeDebug;
...	...	@@ -696,8 +706,8 @@ static ZigLLVMDIScope get_di_scope(CodeGen g, Scope *scope) {
696	706	ZigLLVMDILexicalBlock *di_block = ZigLLVMCreateLexicalBlock(g->dbuilder,
697	707	get_di_scope(g, scope->parent),
698	708	import->data.structure.root_struct->di_file,
699		(unsigned)scope->source_node->line + 1,
700		(unsigned)scope->source_node->column + 1);
	709	node_line_onebased(scope->source_node),
	710	node_column_onebased(scope->source_node));
701	711	scope->di_scope = ZigLLVMLexicalBlockToScope(di_block);
702	712	return scope->di_scope;
703	713	}
...	...	@@ -1798,8 +1808,8 @@ static void gen_var_debug_decl(CodeGen g, ZigVar var) {
1798	1808	if (g->strip_debug_symbols) return;
1799	1809	assert(var->di_loc_var != nullptr);
1800	1810	AstNode *source_node = var->decl_node;
1801		ZigLLVMDILocation *debug_loc = ZigLLVMGetDebugLoc((unsigned)source_node->line + 1,
1802		(unsigned)source_node->column + 1, get_di_scope(g, var->parent_scope));
	1811	ZigLLVMDILocation *debug_loc = ZigLLVMGetDebugLoc(node_line_onebased(source_node),
	1812	node_column_onebased(source_node), get_di_scope(g, var->parent_scope));
1803	1813	ZigLLVMInsertDeclareAtEnd(g->dbuilder, var->value_ref, var->di_loc_var, debug_loc,
1804	1814	LLVMGetInsertBlock(g->builder));
1805	1815	}
...	...	@@ -2198,7 +2208,7 @@ var_ok:
2198	2208	// arg index + 1 because the 0 index is return value
2199	2209	var->di_loc_var = ZigLLVMCreateParameterVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
2200	2210	var->name, fn_walk->data.vars.import->data.structure.root_struct->di_file,
2201		(unsigned)(var->decl_node->line + 1),
	2211	node_line_onebased(var->decl_node),
2202	2212	get_llvm_di_type(g, dest_ty), !g->strip_debug_symbols, 0, di_arg_index + 1);
2203	2213	}
2204	2214	return true;
...	...	@@ -4126,7 +4136,7 @@ static void render_async_spills(CodeGen *g) {
4126	4136	if (var->decl_node) {
4127	4137	var->di_loc_var = ZigLLVMCreateAutoVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
4128	4138	var->name, import->data.structure.root_struct->di_file,
4129		(unsigned)(var->decl_node->line + 1),
	4139	node_line_onebased(var->decl_node),
4130	4140	get_llvm_di_type(g, var->var_type), !g->strip_debug_symbols, 0);
4131	4141	gen_var_debug_decl(g, var);
4132	4142	}
...	...	@@ -6797,8 +6807,8 @@ static void set_debug_location(CodeGen g, IrInstGen instruction) {
6797	6807	assert(source_node);
6798	6808	assert(scope);
6799	6809
6800		ZigLLVMSetCurrentDebugLocation(g->builder, (int)source_node->line + 1,
6801		(int)source_node->column + 1, get_di_scope(g, scope));
	6810	ZigLLVMSetCurrentDebugLocation(g->builder, node_line_onebased(source_node),
	6811	node_column_onebased(source_node), get_di_scope(g, scope));
6802	6812	}
6803	6813
6804	6814	static LLVMValueRef ir_render_instruction(CodeGen g, IrExecutableGen executable, IrInstGen *instruction) {
...	...	@@ -8021,7 +8031,7 @@ static void gen_global_var(CodeGen g, ZigVar var, LLVMValueRef init_val,
8021	8031	bool is_local_to_unit = true;
8022	8032	ZigLLVMCreateGlobalVariable(g->dbuilder, get_di_scope(g, var->parent_scope), var->name,
8023	8033	var->name, import->data.structure.root_struct->di_file,
8024		(unsigned)(var->decl_node->line + 1),
	8034	node_line_onebased(var->decl_node),
8025	8035	get_llvm_di_type(g, type_entry), is_local_to_unit);
8026	8036
8027	8037	// TODO ^^ make an actual global variable
...	...	@@ -8296,7 +8306,8 @@ static void do_code_gen(CodeGen *g) {
8296	8306
8297	8307	if (var->src_arg_index == SIZE_MAX) {
8298	8308	var->di_loc_var = ZigLLVMCreateAutoVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
8299		var->name, import->data.structure.root_struct->di_file, (unsigned)(var->decl_node->line + 1),
	8309	var->name, import->data.structure.root_struct->di_file,
	8310	node_line_onebased(var->decl_node),
8300	8311	get_llvm_di_type(g, var->var_type), !g->strip_debug_symbols, 0);
8301	8312
8302	8313	} else if (is_c_abi) {
...	...	@@ -8321,7 +8332,7 @@ static void do_code_gen(CodeGen *g) {
8321	8332	if (var->decl_node) {
8322	8333	var->di_loc_var = ZigLLVMCreateParameterVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
8323	8334	var->name, import->data.structure.root_struct->di_file,
8324		(unsigned)(var->decl_node->line + 1),
	8335	node_line_onebased(var->decl_node),
8325	8336	get_llvm_di_type(g, gen_type), !g->strip_debug_symbols, 0, (unsigned)(gen_info->gen_index+1));
8326	8337	}
8327	8338
...	...	@@ -8365,8 +8376,9 @@ static void do_code_gen(CodeGen *g) {
8365	8376
8366	8377	if (!g->strip_debug_symbols) {
8367	8378	AstNode *source_node = fn_table_entry->proto_node;
8368		ZigLLVMSetCurrentDebugLocation(g->builder, (int)source_node->line + 1,
8369		(int)source_node->column + 1, get_di_scope(g, fn_table_entry->child_scope));
	8379	ZigLLVMSetCurrentDebugLocation(g->builder,
	8380	node_line_onebased(source_node), node_column_onebased(source_node),
	8381	get_di_scope(g, fn_table_entry->child_scope));
8370	8382	}
8371	8383	IrExecutableGen *executable = &fn_table_entry->analyzed_executable;
8372	8384	LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");

src/stage1/dump_analysis.cpp+33-9

...	...	@@ -1084,19 +1084,43 @@ static void anal_dump_type(AnalDumpCtx ctx, ZigType ty) {
1084	1084	jw_end_object(jw);
1085	1085	}
1086	1086
	1087	static Buf collect_doc_comments(RootStruct root_struct, TokenIndex first_token) {
	1088	if (first_token == 0)
	1089	return nullptr;
	1090
	1091	TokenId *token_ids = root_struct->token_ids;
	1092	TokenLoc *token_locs = root_struct->token_locs;
	1093	Buf *str = buf_alloc();
	1094	const char *source = buf_ptr(root_struct->source_code);
	1095	TokenIndex doc_token = first_token;
	1096	for (;token_ids[doc_token] == TokenIdDocComment; doc_token += 1) {
	1097	// chops off '///' but leaves '\n'
	1098	uint32_t start_pos = token_locs[doc_token].offset;
	1099	uint32_t token_len = 0;
	1100	while (source[start_pos + token_len] != '\n' &&
	1101	source[start_pos + token_len] != 0)
	1102	{
	1103	token_len += 1;
	1104	}
	1105	buf_append_mem(str, source + start_pos + 3, token_len - 3);
	1106	}
	1107	return str;
	1108	}
	1109
1087	1110	static void anal_dump_node(AnalDumpCtx ctx, const AstNode node) {
1088	1111	JsonWriter *jw = &ctx->jw;
1089	1112
1090	1113	jw_begin_object(jw);
1091	1114
1092	1115	jw_object_field(jw, "file");
1093		anal_dump_file_ref(ctx, node->owner->data.structure.root_struct->path);
	1116	RootStruct *root_struct = node->owner->data.structure.root_struct;
	1117	anal_dump_file_ref(ctx, root_struct->path);
1094	1118
1095	1119	jw_object_field(jw, "line");
1096		jw_int(jw, node->line);
	1120	jw_int(jw, root_struct->token_locs[node->main_token].line);
1097	1121
1098	1122	jw_object_field(jw, "col");
1099		jw_int(jw, node->column);
	1123	jw_int(jw, root_struct->token_locs[node->main_token].column);
1100	1124
1101	1125	const Buf *doc_comments_buf = nullptr;
1102	1126	const Buf *name_buf = nullptr;
...	...	@@ -1107,30 +1131,30 @@ static void anal_dump_node(AnalDumpCtx ctx, const AstNode node) {
1107	1131
1108	1132	switch (node->type) {
1109	1133	case NodeTypeParamDecl:
1110		doc_comments_buf = &node->data.param_decl.doc_comments;
	1134	doc_comments_buf = collect_doc_comments(root_struct, node->data.param_decl.doc_comments);
1111	1135	name_buf = node->data.param_decl.name;
1112	1136	is_var_args = node->data.param_decl.is_var_args;
1113	1137	is_noalias = node->data.param_decl.is_noalias;
1114	1138	is_comptime = node->data.param_decl.is_comptime;
1115	1139	break;
1116	1140	case NodeTypeFnProto:
1117		doc_comments_buf = &node->data.fn_proto.doc_comments;
	1141	doc_comments_buf = collect_doc_comments(root_struct, node->data.fn_proto.doc_comments);
1118	1142	field_nodes = &node->data.fn_proto.params;
1119	1143	is_var_args = node->data.fn_proto.is_var_args;
1120	1144	break;
1121	1145	case NodeTypeVariableDeclaration:
1122		doc_comments_buf = &node->data.variable_declaration.doc_comments;
	1146	doc_comments_buf = collect_doc_comments(root_struct, node->data.variable_declaration.doc_comments);
1123	1147	break;
1124	1148	case NodeTypeErrorSetField:
1125		doc_comments_buf = &node->data.err_set_field.doc_comments;
	1149	doc_comments_buf = collect_doc_comments(root_struct, node->data.err_set_field.doc_comments);
1126	1150	break;
1127	1151	case NodeTypeStructField:
1128		doc_comments_buf = &node->data.struct_field.doc_comments;
	1152	doc_comments_buf = collect_doc_comments(root_struct, node->data.struct_field.doc_comments);
1129	1153	name_buf = node->data.struct_field.name;
1130	1154	break;
1131	1155	case NodeTypeContainerDecl:
1132	1156	field_nodes = &node->data.container_decl.fields;
1133		doc_comments_buf = &node->data.container_decl.doc_comments;
	1157	doc_comments_buf = collect_doc_comments(root_struct, node->data.container_decl.doc_comments);
1134	1158	break;
1135	1159	default:
1136	1160	break;

src/stage1/errmsg.cpp+19-40

...	...	@@ -96,14 +96,14 @@ void err_msg_add_note(ErrorMsg parent, ErrorMsg note) {
96	96	parent->notes.append(note);
97	97	}
98	98
99		ErrorMsg err_msg_create_with_offset(Buf path, size_t line, size_t column, size_t offset,
100		const char source, Buf msg)
	99	ErrorMsg err_msg_create_with_offset(Buf path, uint32_t byte_offset, const char *source,
	100	Buf *msg)
101	101	{
102	102	ErrorMsg *err_msg = heap::c_allocator.create<ErrorMsg>();
103	103	err_msg->path = path;
104		err_msg->line_start = line;
105		err_msg->column_start = column;
106	104	err_msg->msg = msg;
	105	err_msg->line_start = 0;
	106	err_msg->column_start = 0;
107	107
108	108	if (source == nullptr) {
109	109	// Must initialize the buffer anyway
...	...	@@ -111,46 +111,25 @@ ErrorMsg err_msg_create_with_offset(Buf path, size_t line, size_t column, size
111	111	return err_msg;
112	112	}
113	113
114		size_t line_start_offset = offset;
115		for (;;) {
116		if (line_start_offset == 0) {
117		break;
118		}
119
120		line_start_offset -= 1;
121
122		if (source[line_start_offset] == '\n') {
123		line_start_offset += 1;
124		break;
	114	size_t line_start = 0;
	115	size_t i = 0;
	116	for (;i < byte_offset; i += 1) {
	117	switch (source[i]) {
	118	case '\n':
	119	err_msg->line_start += 1;
	120	err_msg->column_start = 0;
	121	line_start = i + 1;
	122	continue;
	123	default:
	124	err_msg->column_start += 1;
	125	continue;
125	126	}
126	127	}
127
128		size_t line_end_offset = offset;
129		while (source[line_end_offset] && source[line_end_offset] != '\n') {
130		line_end_offset += 1;
	128	while (source[i] != '\n' && source[i] != 0) {
	129	i += 1;
131	130	}
132	131
133		buf_init_from_mem(&err_msg->line_buf, source + line_start_offset, line_end_offset - line_start_offset);
134
135		return err_msg;
136		}
137
138		ErrorMsg err_msg_create_with_line(Buf path, size_t line, size_t column,
139		Buf source, ZigList<size_t> line_offsets, Buf *msg)
140		{
141		ErrorMsg *err_msg = heap::c_allocator.create<ErrorMsg>();
142		err_msg->path = path;
143		err_msg->line_start = line;
144		err_msg->column_start = column;
145		err_msg->msg = msg;
146
147		size_t line_start_offset = line_offsets->at(line);
148		size_t end_line = line + 1;
149		size_t line_end_offset = (end_line >= line_offsets->length) ? buf_len(source) : line_offsets->at(line + 1);
150		size_t len = (line_end_offset + 1 > line_start_offset) ? (line_end_offset - line_start_offset - 1) : 0;
151		if (len == SIZE_MAX) len = 0;
152
153		buf_init_from_mem(&err_msg->line_buf, buf_ptr(source) + line_start_offset, len);
	132	buf_init_from_mem(&err_msg->line_buf, source + line_start, i - line_start);
154	133
155	134	return err_msg;
156	135	}

src/stage1/errmsg.hpp+1-5

...	...	@@ -25,10 +25,6 @@ struct ErrorMsg {
25	25	void print_err_msg(ErrorMsg *msg, ErrColor color);
26	26
27	27	void err_msg_add_note(ErrorMsg parent, ErrorMsg note);
28		ErrorMsg err_msg_create_with_offset(Buf path, size_t line, size_t column, size_t offset,
29		const char source, Buf msg);
30
31		ErrorMsg err_msg_create_with_line(Buf path, size_t line, size_t column,
32		Buf source, ZigList<size_t> line_offsets, Buf *msg);
	28	ErrorMsg err_msg_create_with_offset(Buf path, uint32_t byte_offset, const char source, Buf msg);
33	29
34	30	#endif

src/stage1/error.cpp+2

...	...	@@ -88,6 +88,8 @@ const char *err_str(Error err) {
88	88	case ErrorZigIsTheCCompiler: return "Zig was not provided with libc installation information, and so it does not know where the libc paths are on the system. Zig attempted to use the system C compiler to find out where the libc paths are, but discovered that Zig is being used as the system C compiler.";
89	89	case ErrorFileBusy: return "file is busy";
90	90	case ErrorLocked: return "file is locked by another process";
	91	case ErrorInvalidCharacter: return "invalid character";
	92	case ErrorUnicodePointTooLarge: return "unicode codepoint too large";
91	93	}
92	94	return "(invalid error)";
93	95	}

src/stage1/ir.cpp+1186-9227

...	...	@@ -5,8 +5,8 @@
5	5	* See http://opensource.org/licenses/MIT
6	6	*/
7	7
	8	#include "astgen.hpp"
8	9	#include "analyze.hpp"
9		#include "ast_render.hpp"
10	10	#include "error.hpp"
11	11	#include "ir.hpp"
12	12	#include "ir_print.hpp"
...	...	@@ -22,13 +22,6 @@
22	22	#include <errno.h>
23	23	#include <math.h>
24	24
25		struct IrBuilderSrc {
26		CodeGen *codegen;
27		IrExecutableSrc *exec;
28		IrBasicBlockSrc *current_basic_block;
29		AstNode *main_block_node;
30		};
31
32	25	struct IrBuilderGen {
33	26	CodeGen *codegen;
34	27	IrExecutableGen *exec;
...	...	@@ -216,27 +209,18 @@ struct DbgIrBreakPoint {
216	209	const char *src_file;
217	210	uint32_t line;
218	211	};
219		DbgIrBreakPoint dbg_ir_breakpoints_buf[20];
220		size_t dbg_ir_breakpoints_count = 0;
221	212
222		static IrInstSrc ir_gen_node(IrBuilderSrc irb, AstNode node, Scope scope);
223		static IrInstSrc ir_gen_node_extra(IrBuilderSrc irb, AstNode node, Scope scope, LVal lval,
224		ResultLoc *result_loc);
225	213	static IrInstGen ir_implicit_cast(IrAnalyze ira, IrInstGen value, ZigType expected_type);
226	214	static IrInstGen ir_implicit_cast2(IrAnalyze ira, IrInst *value_source_instr,
227	215	IrInstGen value, ZigType expected_type);
228	216	static IrInstGen ir_get_deref(IrAnalyze ira, IrInst source_instr, IrInstGen ptr,
229	217	ResultLoc *result_loc);
230		static ErrorMsg exec_add_error_node(CodeGen codegen, IrExecutableSrc exec, AstNode source_node, Buf *msg);
231	218	static IrInstGen ir_analyze_container_field_ptr(IrAnalyze ira, Buf *field_name,
232	219	IrInst* source_instr, IrInstGen container_ptr, IrInst container_ptr_src,
233	220	ZigType *container_type, bool initializing);
234		static void ir_assert_impl(bool ok, IrInst* source_instruction, const char *file, unsigned int line);
235	221	static void ir_assert_gen_impl(bool ok, IrInstGen source_instruction, const char file, unsigned int line);
236	222	static IrInstGen ir_get_var_ptr(IrAnalyze ira, IrInst source_instr, ZigVar var);
237	223	static ZigType ir_resolve_atomic_operand_type(IrAnalyze ira, IrInstGen *op);
238		static IrInstSrc ir_lval_wrap(IrBuilderSrc irb, Scope scope, IrInstSrc value, LVal lval, ResultLoc *result_loc);
239		static IrInstSrc ir_expr_wrap(IrBuilderSrc irb, Scope scope, IrInstSrc inst, ResultLoc *result_loc);
240	224	static ZigType adjust_ptr_align(CodeGen g, ZigType *ptr_type, uint32_t new_align);
241	225	static ZigType adjust_ptr_const(CodeGen g, ZigType *ptr_type, bool is_const);
242	226	static ZigType adjust_slice_align(CodeGen g, ZigType *slice_type, uint32_t new_align);
...	...	@@ -265,26 +249,14 @@ static IrInstGen ir_analyze_unwrap_err_code(IrAnalyze ira, IrInst* source_inst
265	249	IrInstGen *base_ptr, bool initializing);
266	250	static IrInstGen ir_analyze_store_ptr(IrAnalyze ira, IrInst* source_instr,
267	251	IrInstGen ptr, IrInstGen uncasted_value, bool allow_write_through_const);
268		static IrInstSrc ir_gen_union_init_expr(IrBuilderSrc irb, Scope scope, AstNode source_node,
269		IrInstSrc union_type, IrInstSrc field_name, AstNode *expr_node,
270		LVal lval, ResultLoc *parent_result_loc);
271	252	static void ir_reset_result(ResultLoc *result_loc);
272		static Buf get_anon_type_name(CodeGen codegen, IrExecutableSrc exec, const char kind_name,
273		Scope scope, AstNode source_node, Buf *out_bare_name);
274		static ResultLocCast ir_build_cast_result_loc(IrBuilderSrc irb, IrInstSrc *dest_type,
275		ResultLoc *parent_result_loc);
276	253	static IrInstGen ir_analyze_struct_field_ptr(IrAnalyze ira, IrInst* source_instr,
277	254	TypeStructField field, IrInstGen struct_ptr, ZigType *struct_type, bool initializing);
278	255	static IrInstGen ir_analyze_inferred_field_ptr(IrAnalyze ira, Buf *field_name,
279	256	IrInst* source_instr, IrInstGen container_ptr, ZigType container_type);
280		static ResultLoc *no_result_loc(void);
281	257	static IrInstGen ir_analyze_test_non_null(IrAnalyze ira, IrInst source_inst, IrInstGen value);
282	258	static IrInstGen ir_error_dependency_loop(IrAnalyze ira, IrInst *source_instr);
283	259	static IrInstGen ir_const_undef(IrAnalyze ira, IrInst source_instruction, ZigType ty);
284		static ZigVar ir_create_var(IrBuilderSrc irb, AstNode node, Scope scope, Buf *name,
285		bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstSrc *is_comptime);
286		static void build_decl_var_and_init(IrBuilderSrc irb, Scope scope, AstNode source_node, ZigVar var,
287		IrInstSrc init, const char name_hint, IrInstSrc *is_comptime);
288	260	static IrInstGen ir_analyze_union_init(IrAnalyze ira, IrInst* source_instruction,
289	261	AstNode field_source_node, ZigType union_type, Buf field_name, IrInstGen field_result_loc,
290	262	IrInstGen *result_loc);
...	...	@@ -295,292 +267,15 @@ static bool value_cmp_numeric_val_all(ZigValue left, Cmp predicate, ZigValue r
295	267	static void memoize_field_init_val(CodeGen codegen, ZigType container_type, TypeStructField *field);
296	268	static void value_to_bigfloat(BigFloat out, ZigValue val);
297	269
	270	static void ir_assert_impl(bool ok, IrInst source_instruction, char const file, unsigned int line) {
	271	if (ok) return;
	272	src_assert_impl(ok, source_instruction->source_node, file, line);
	273	}
	274
	275
298	276	#define ir_assert(OK, SOURCE_INSTRUCTION) ir_assert_impl((OK), (SOURCE_INSTRUCTION), __FILE__, __LINE__)
299	277	#define ir_assert_gen(OK, SOURCE_INSTRUCTION) ir_assert_gen_impl((OK), (SOURCE_INSTRUCTION), __FILE__, __LINE__)
300	278
301		static void destroy_instruction_src(IrInstSrc *inst) {
302		switch (inst->id) {
303		case IrInstSrcIdInvalid:
304		zig_unreachable();
305		case IrInstSrcIdReturn:
306		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcReturn *>(inst));
307		case IrInstSrcIdConst:
308		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcConst *>(inst));
309		case IrInstSrcIdBinOp:
310		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBinOp *>(inst));
311		case IrInstSrcIdMergeErrSets:
312		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcMergeErrSets *>(inst));
313		case IrInstSrcIdDeclVar:
314		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcDeclVar *>(inst));
315		case IrInstSrcIdCall:
316		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCall *>(inst));
317		case IrInstSrcIdCallExtra:
318		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCallExtra *>(inst));
319		case IrInstSrcIdAsyncCallExtra:
320		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAsyncCallExtra *>(inst));
321		case IrInstSrcIdUnOp:
322		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUnOp *>(inst));
323		case IrInstSrcIdCondBr:
324		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCondBr *>(inst));
325		case IrInstSrcIdBr:
326		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBr *>(inst));
327		case IrInstSrcIdPhi:
328		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPhi *>(inst));
329		case IrInstSrcIdContainerInitList:
330		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcContainerInitList *>(inst));
331		case IrInstSrcIdContainerInitFields:
332		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcContainerInitFields *>(inst));
333		case IrInstSrcIdUnreachable:
334		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUnreachable *>(inst));
335		case IrInstSrcIdElemPtr:
336		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcElemPtr *>(inst));
337		case IrInstSrcIdVarPtr:
338		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcVarPtr *>(inst));
339		case IrInstSrcIdLoadPtr:
340		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcLoadPtr *>(inst));
341		case IrInstSrcIdStorePtr:
342		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcStorePtr *>(inst));
343		case IrInstSrcIdTypeOf:
344		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTypeOf *>(inst));
345		case IrInstSrcIdFieldPtr:
346		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFieldPtr *>(inst));
347		case IrInstSrcIdSetCold:
348		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSetCold *>(inst));
349		case IrInstSrcIdSetRuntimeSafety:
350		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSetRuntimeSafety *>(inst));
351		case IrInstSrcIdSetFloatMode:
352		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSetFloatMode *>(inst));
353		case IrInstSrcIdArrayType:
354		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcArrayType *>(inst));
355		case IrInstSrcIdSliceType:
356		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSliceType *>(inst));
357		case IrInstSrcIdAnyFrameType:
358		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAnyFrameType *>(inst));
359		case IrInstSrcIdAsm:
360		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAsm *>(inst));
361		case IrInstSrcIdSizeOf:
362		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSizeOf *>(inst));
363		case IrInstSrcIdTestNonNull:
364		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTestNonNull *>(inst));
365		case IrInstSrcIdOptionalUnwrapPtr:
366		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcOptionalUnwrapPtr *>(inst));
367		case IrInstSrcIdPopCount:
368		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPopCount *>(inst));
369		case IrInstSrcIdClz:
370		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcClz *>(inst));
371		case IrInstSrcIdCtz:
372		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCtz *>(inst));
373		case IrInstSrcIdBswap:
374		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBswap *>(inst));
375		case IrInstSrcIdBitReverse:
376		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBitReverse *>(inst));
377		case IrInstSrcIdSwitchBr:
378		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSwitchBr *>(inst));
379		case IrInstSrcIdSwitchVar:
380		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSwitchVar *>(inst));
381		case IrInstSrcIdSwitchElseVar:
382		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSwitchElseVar *>(inst));
383		case IrInstSrcIdSwitchTarget:
384		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSwitchTarget *>(inst));
385		case IrInstSrcIdImport:
386		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcImport *>(inst));
387		case IrInstSrcIdRef:
388		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcRef *>(inst));
389		case IrInstSrcIdCompileErr:
390		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCompileErr *>(inst));
391		case IrInstSrcIdCompileLog:
392		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCompileLog *>(inst));
393		case IrInstSrcIdErrName:
394		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcErrName *>(inst));
395		case IrInstSrcIdCImport:
396		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCImport *>(inst));
397		case IrInstSrcIdCInclude:
398		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCInclude *>(inst));
399		case IrInstSrcIdCDefine:
400		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCDefine *>(inst));
401		case IrInstSrcIdCUndef:
402		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCUndef *>(inst));
403		case IrInstSrcIdEmbedFile:
404		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcEmbedFile *>(inst));
405		case IrInstSrcIdCmpxchg:
406		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCmpxchg *>(inst));
407		case IrInstSrcIdFence:
408		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFence *>(inst));
409		case IrInstSrcIdReduce:
410		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcReduce *>(inst));
411		case IrInstSrcIdTruncate:
412		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTruncate *>(inst));
413		case IrInstSrcIdIntCast:
414		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcIntCast *>(inst));
415		case IrInstSrcIdFloatCast:
416		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFloatCast *>(inst));
417		case IrInstSrcIdErrSetCast:
418		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcErrSetCast *>(inst));
419		case IrInstSrcIdIntToFloat:
420		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcIntToFloat *>(inst));
421		case IrInstSrcIdFloatToInt:
422		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFloatToInt *>(inst));
423		case IrInstSrcIdBoolToInt:
424		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBoolToInt *>(inst));
425		case IrInstSrcIdVectorType:
426		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcVectorType *>(inst));
427		case IrInstSrcIdShuffleVector:
428		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcShuffleVector *>(inst));
429		case IrInstSrcIdSplat:
430		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSplat *>(inst));
431		case IrInstSrcIdBoolNot:
432		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBoolNot *>(inst));
433		case IrInstSrcIdMemset:
434		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcMemset *>(inst));
435		case IrInstSrcIdMemcpy:
436		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcMemcpy *>(inst));
437		case IrInstSrcIdSlice:
438		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSlice *>(inst));
439		case IrInstSrcIdBreakpoint:
440		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBreakpoint *>(inst));
441		case IrInstSrcIdReturnAddress:
442		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcReturnAddress *>(inst));
443		case IrInstSrcIdFrameAddress:
444		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFrameAddress *>(inst));
445		case IrInstSrcIdFrameHandle:
446		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFrameHandle *>(inst));
447		case IrInstSrcIdFrameType:
448		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFrameType *>(inst));
449		case IrInstSrcIdFrameSize:
450		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFrameSize *>(inst));
451		case IrInstSrcIdAlignOf:
452		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAlignOf *>(inst));
453		case IrInstSrcIdOverflowOp:
454		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcOverflowOp *>(inst));
455		case IrInstSrcIdTestErr:
456		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTestErr *>(inst));
457		case IrInstSrcIdUnwrapErrCode:
458		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUnwrapErrCode *>(inst));
459		case IrInstSrcIdUnwrapErrPayload:
460		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUnwrapErrPayload *>(inst));
461		case IrInstSrcIdFnProto:
462		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFnProto *>(inst));
463		case IrInstSrcIdTestComptime:
464		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTestComptime *>(inst));
465		case IrInstSrcIdPtrCast:
466		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPtrCast *>(inst));
467		case IrInstSrcIdBitCast:
468		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBitCast *>(inst));
469		case IrInstSrcIdPtrToInt:
470		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPtrToInt *>(inst));
471		case IrInstSrcIdIntToPtr:
472		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcIntToPtr *>(inst));
473		case IrInstSrcIdIntToEnum:
474		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcIntToEnum *>(inst));
475		case IrInstSrcIdIntToErr:
476		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcIntToErr *>(inst));
477		case IrInstSrcIdErrToInt:
478		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcErrToInt *>(inst));
479		case IrInstSrcIdCheckSwitchProngsUnderNo:
480		case IrInstSrcIdCheckSwitchProngsUnderYes:
481		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCheckSwitchProngs *>(inst));
482		case IrInstSrcIdCheckStatementIsVoid:
483		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCheckStatementIsVoid *>(inst));
484		case IrInstSrcIdTypeName:
485		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTypeName *>(inst));
486		case IrInstSrcIdTagName:
487		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTagName *>(inst));
488		case IrInstSrcIdPtrType:
489		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPtrType *>(inst));
490		case IrInstSrcIdPtrTypeSimple:
491		case IrInstSrcIdPtrTypeSimpleConst:
492		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPtrTypeSimple *>(inst));
493		case IrInstSrcIdDeclRef:
494		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcDeclRef *>(inst));
495		case IrInstSrcIdPanic:
496		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcPanic *>(inst));
497		case IrInstSrcIdFieldParentPtr:
498		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFieldParentPtr *>(inst));
499		case IrInstSrcIdByteOffsetOf:
500		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcByteOffsetOf *>(inst));
501		case IrInstSrcIdBitOffsetOf:
502		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcBitOffsetOf *>(inst));
503		case IrInstSrcIdTypeInfo:
504		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTypeInfo *>(inst));
505		case IrInstSrcIdType:
506		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcType *>(inst));
507		case IrInstSrcIdHasField:
508		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcHasField *>(inst));
509		case IrInstSrcIdSetEvalBranchQuota:
510		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSetEvalBranchQuota *>(inst));
511		case IrInstSrcIdAlignCast:
512		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAlignCast *>(inst));
513		case IrInstSrcIdImplicitCast:
514		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcImplicitCast *>(inst));
515		case IrInstSrcIdResolveResult:
516		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcResolveResult *>(inst));
517		case IrInstSrcIdResetResult:
518		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcResetResult *>(inst));
519		case IrInstSrcIdSetAlignStack:
520		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSetAlignStack *>(inst));
521		case IrInstSrcIdArgTypeAllowVarFalse:
522		case IrInstSrcIdArgTypeAllowVarTrue:
523		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcArgType *>(inst));
524		case IrInstSrcIdExport:
525		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcExport *>(inst));
526		case IrInstSrcIdExtern:
527		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcExtern *>(inst));
528		case IrInstSrcIdErrorReturnTrace:
529		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcErrorReturnTrace *>(inst));
530		case IrInstSrcIdErrorUnion:
531		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcErrorUnion *>(inst));
532		case IrInstSrcIdAtomicRmw:
533		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAtomicRmw *>(inst));
534		case IrInstSrcIdSaveErrRetAddr:
535		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSaveErrRetAddr *>(inst));
536		case IrInstSrcIdAddImplicitReturnType:
537		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAddImplicitReturnType *>(inst));
538		case IrInstSrcIdFloatOp:
539		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFloatOp *>(inst));
540		case IrInstSrcIdMulAdd:
541		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcMulAdd *>(inst));
542		case IrInstSrcIdAtomicLoad:
543		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAtomicLoad *>(inst));
544		case IrInstSrcIdAtomicStore:
545		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAtomicStore *>(inst));
546		case IrInstSrcIdEnumToInt:
547		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcEnumToInt *>(inst));
548		case IrInstSrcIdCheckRuntimeScope:
549		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCheckRuntimeScope *>(inst));
550		case IrInstSrcIdHasDecl:
551		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcHasDecl *>(inst));
552		case IrInstSrcIdUndeclaredIdent:
553		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUndeclaredIdent *>(inst));
554		case IrInstSrcIdAlloca:
555		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAlloca *>(inst));
556		case IrInstSrcIdEndExpr:
557		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcEndExpr *>(inst));
558		case IrInstSrcIdUnionInitNamedField:
559		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcUnionInitNamedField *>(inst));
560		case IrInstSrcIdSuspendBegin:
561		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSuspendBegin *>(inst));
562		case IrInstSrcIdSuspendFinish:
563		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSuspendFinish *>(inst));
564		case IrInstSrcIdResume:
565		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcResume *>(inst));
566		case IrInstSrcIdAwait:
567		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcAwait *>(inst));
568		case IrInstSrcIdSpillBegin:
569		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSpillBegin *>(inst));
570		case IrInstSrcIdSpillEnd:
571		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSpillEnd *>(inst));
572		case IrInstSrcIdCallArgs:
573		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCallArgs *>(inst));
574		case IrInstSrcIdWasmMemorySize:
575		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcWasmMemorySize *>(inst));
576		case IrInstSrcIdWasmMemoryGrow:
577		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcWasmMemoryGrow *>(inst));
578		case IrInstSrcIdSrc:
579		return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcSrc *>(inst));
580		}
581		zig_unreachable();
582		}
583
584	279	void destroy_instruction_gen(IrInstGen *inst) {
585	280	switch (inst->id) {
586	281	case IrInstGenIdInvalid:
...	...	@@ -970,54 +665,18 @@ static bool types_have_same_zig_comptime_repr(CodeGen codegen, ZigType expecte
970	665	zig_unreachable();
971	666	}
972	667
973		static bool ir_should_inline(IrExecutableSrc exec, Scope scope) {
974		if (exec->is_inline)
975		return true;
976
977		while (scope != nullptr) {
978		if (scope->id == ScopeIdCompTime)
979		return true;
980		if (scope->id == ScopeIdTypeOf)
981		return false;
982		if (scope->id == ScopeIdFnDef)
983		break;
984		scope = scope->parent;
985		}
986		return false;
987		}
988
989		static void ir_instruction_append(IrBasicBlockSrc basic_block, IrInstSrc instruction) {
990		assert(basic_block);
991		assert(instruction);
992		basic_block->instruction_list.append(instruction);
993		}
994
995	668	static void ir_inst_gen_append(IrBasicBlockGen basic_block, IrInstGen instruction) {
996	669	assert(basic_block);
997	670	assert(instruction);
998	671	basic_block->instruction_list.append(instruction);
999	672	}
1000	673
1001		static size_t exec_next_debug_id(IrExecutableSrc *exec) {
1002		size_t result = exec->next_debug_id;
1003		exec->next_debug_id += 1;
1004		return result;
1005		}
1006
1007	674	static size_t exec_next_debug_id_gen(IrExecutableGen *exec) {
1008	675	size_t result = exec->next_debug_id;
1009	676	exec->next_debug_id += 1;
1010	677	return result;
1011	678	}
1012	679
1013		static ZigFn exec_fn_entry(IrExecutableSrc exec) {
1014		return exec->fn_entry;
1015		}
1016
1017		static Buf exec_c_import_buf(IrExecutableSrc exec) {
1018		return exec->c_import_buf;
1019		}
1020
1021	680	static bool value_is_comptime(ZigValue *const_val) {
1022	681	return const_val->special != ConstValSpecialRuntime;
1023	682	}
...	...	@@ -1026,33 +685,11 @@ static bool instr_is_comptime(IrInstGen *instruction) {
1026	685	return value_is_comptime(instruction->value);
1027	686	}
1028	687
1029		static bool instr_is_unreachable(IrInstSrc *instruction) {
1030		return instruction->is_noreturn;
1031		}
1032
1033		static void ir_ref_bb(IrBasicBlockSrc *bb) {
1034		bb->ref_count += 1;
1035		}
1036
1037		static void ir_ref_instruction(IrInstSrc instruction, IrBasicBlockSrc cur_bb) {
1038		assert(instruction->id != IrInstSrcIdInvalid);
1039		instruction->base.ref_count += 1;
1040		if (instruction->owner_bb != cur_bb && !instr_is_unreachable(instruction)
1041		&& instruction->id != IrInstSrcIdConst)
1042		{
1043		ir_ref_bb(instruction->owner_bb);
1044		}
1045		}
1046
1047	688	static void ir_ref_inst_gen(IrInstGen *instruction) {
1048	689	assert(instruction->id != IrInstGenIdInvalid);
1049	690	instruction->base.ref_count += 1;
1050	691	}
1051	692
1052		static void ir_ref_var(ZigVar *var) {
1053		var->ref_count += 1;
1054		}
1055
1056	693	static void create_result_ptr(CodeGen codegen, ZigType expected_type,
1057	694	ZigValue out_result, ZigValue out_result_ptr)
1058	695	{
...	...	@@ -1092,15 +729,6 @@ ZigType ir_analyze_type_expr(IrAnalyze ira, Scope scope, AstNode node) {
1092	729	return res_type;
1093	730	}
1094	731
1095		static IrBasicBlockSrc ir_create_basic_block(IrBuilderSrc irb, Scope scope, const char name_hint) {
1096		IrBasicBlockSrc *result = heap::c_allocator.create<IrBasicBlockSrc>();
1097		result->scope = scope;
1098		result->name_hint = name_hint;
1099		result->debug_id = exec_next_debug_id(irb->exec);
1100		result->index = UINT32_MAX; // set later
1101		return result;
1102		}
1103
1104	732	static IrBasicBlockGen ir_create_basic_block_gen(IrAnalyze ira, Scope scope, const char name_hint) {
1105	733	IrBasicBlockGen *result = heap::c_allocator.create<IrBasicBlockGen>();
1106	734	result->scope = scope;
...	...	@@ -1115,8807 +743,1687 @@ static IrBasicBlockGen ir_build_bb_from(IrAnalyze ira, IrBasicBlockSrc *other_
1115	743	return new_bb;
1116	744	}
1117	745
1118		static constexpr IrInstSrcId ir_inst_id(IrInstSrcDeclVar *) {
1119		return IrInstSrcIdDeclVar;
	746	static constexpr IrInstGenId ir_inst_id(IrInstGenDeclVar *) {
	747	return IrInstGenIdDeclVar;
1120	748	}
1121	749
1122		static constexpr IrInstSrcId ir_inst_id(IrInstSrcBr *) {
1123		return IrInstSrcIdBr;
	750	static constexpr IrInstGenId ir_inst_id(IrInstGenBr *) {
	751	return IrInstGenIdBr;
1124	752	}
1125	753
1126		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCondBr *) {
1127		return IrInstSrcIdCondBr;
	754	static constexpr IrInstGenId ir_inst_id(IrInstGenCondBr *) {
	755	return IrInstGenIdCondBr;
1128	756	}
1129	757
1130		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSwitchBr *) {
1131		return IrInstSrcIdSwitchBr;
	758	static constexpr IrInstGenId ir_inst_id(IrInstGenSwitchBr *) {
	759	return IrInstGenIdSwitchBr;
1132	760	}
1133	761
1134		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSwitchVar *) {
1135		return IrInstSrcIdSwitchVar;
	762	static constexpr IrInstGenId ir_inst_id(IrInstGenPhi *) {
	763	return IrInstGenIdPhi;
1136	764	}
1137	765
1138		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSwitchElseVar *) {
1139		return IrInstSrcIdSwitchElseVar;
	766	static constexpr IrInstGenId ir_inst_id(IrInstGenBinaryNot *) {
	767	return IrInstGenIdBinaryNot;
1140	768	}
1141	769
1142		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSwitchTarget *) {
1143		return IrInstSrcIdSwitchTarget;
	770	static constexpr IrInstGenId ir_inst_id(IrInstGenNegation *) {
	771	return IrInstGenIdNegation;
1144	772	}
1145	773
1146		static constexpr IrInstSrcId ir_inst_id(IrInstSrcPhi *) {
1147		return IrInstSrcIdPhi;
	774	static constexpr IrInstGenId ir_inst_id(IrInstGenBinOp *) {
	775	return IrInstGenIdBinOp;
1148	776	}
1149	777
1150		static constexpr IrInstSrcId ir_inst_id(IrInstSrcUnOp *) {
1151		return IrInstSrcIdUnOp;
	778	static constexpr IrInstGenId ir_inst_id(IrInstGenLoadPtr *) {
	779	return IrInstGenIdLoadPtr;
1152	780	}
1153	781
1154		static constexpr IrInstSrcId ir_inst_id(IrInstSrcBinOp *) {
1155		return IrInstSrcIdBinOp;
	782	static constexpr IrInstGenId ir_inst_id(IrInstGenStorePtr *) {
	783	return IrInstGenIdStorePtr;
1156	784	}
1157	785
1158		static constexpr IrInstSrcId ir_inst_id(IrInstSrcMergeErrSets *) {
1159		return IrInstSrcIdMergeErrSets;
	786	static constexpr IrInstGenId ir_inst_id(IrInstGenVectorStoreElem *) {
	787	return IrInstGenIdVectorStoreElem;
1160	788	}
1161	789
1162		static constexpr IrInstSrcId ir_inst_id(IrInstSrcLoadPtr *) {
1163		return IrInstSrcIdLoadPtr;
	790	static constexpr IrInstGenId ir_inst_id(IrInstGenStructFieldPtr *) {
	791	return IrInstGenIdStructFieldPtr;
1164	792	}
1165	793
1166		static constexpr IrInstSrcId ir_inst_id(IrInstSrcStorePtr *) {
1167		return IrInstSrcIdStorePtr;
	794	static constexpr IrInstGenId ir_inst_id(IrInstGenUnionFieldPtr *) {
	795	return IrInstGenIdUnionFieldPtr;
1168	796	}
1169	797
1170		static constexpr IrInstSrcId ir_inst_id(IrInstSrcFieldPtr *) {
1171		return IrInstSrcIdFieldPtr;
	798	static constexpr IrInstGenId ir_inst_id(IrInstGenElemPtr *) {
	799	return IrInstGenIdElemPtr;
1172	800	}
1173	801
1174		static constexpr IrInstSrcId ir_inst_id(IrInstSrcElemPtr *) {
1175		return IrInstSrcIdElemPtr;
	802	static constexpr IrInstGenId ir_inst_id(IrInstGenVarPtr *) {
	803	return IrInstGenIdVarPtr;
1176	804	}
1177	805
1178		static constexpr IrInstSrcId ir_inst_id(IrInstSrcVarPtr *) {
1179		return IrInstSrcIdVarPtr;
	806	static constexpr IrInstGenId ir_inst_id(IrInstGenReturnPtr *) {
	807	return IrInstGenIdReturnPtr;
1180	808	}
1181	809
1182		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCall *) {
1183		return IrInstSrcIdCall;
	810	static constexpr IrInstGenId ir_inst_id(IrInstGenCall *) {
	811	return IrInstGenIdCall;
1184	812	}
1185	813
1186		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCallArgs *) {
1187		return IrInstSrcIdCallArgs;
	814	static constexpr IrInstGenId ir_inst_id(IrInstGenReturn *) {
	815	return IrInstGenIdReturn;
1188	816	}
1189	817
1190		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCallExtra *) {
1191		return IrInstSrcIdCallExtra;
	818	static constexpr IrInstGenId ir_inst_id(IrInstGenCast *) {
	819	return IrInstGenIdCast;
1192	820	}
1193	821
1194		static constexpr IrInstSrcId ir_inst_id(IrInstSrcAsyncCallExtra *) {
1195		return IrInstSrcIdAsyncCallExtra;
	822	static constexpr IrInstGenId ir_inst_id(IrInstGenUnreachable *) {
	823	return IrInstGenIdUnreachable;
1196	824	}
1197	825
1198		static constexpr IrInstSrcId ir_inst_id(IrInstSrcConst *) {
1199		return IrInstSrcIdConst;
	826	static constexpr IrInstGenId ir_inst_id(IrInstGenAsm *) {
	827	return IrInstGenIdAsm;
1200	828	}
1201	829
1202		static constexpr IrInstSrcId ir_inst_id(IrInstSrcReturn *) {
1203		return IrInstSrcIdReturn;
	830	static constexpr IrInstGenId ir_inst_id(IrInstGenTestNonNull *) {
	831	return IrInstGenIdTestNonNull;
1204	832	}
1205	833
1206		static constexpr IrInstSrcId ir_inst_id(IrInstSrcContainerInitList *) {
1207		return IrInstSrcIdContainerInitList;
	834	static constexpr IrInstGenId ir_inst_id(IrInstGenOptionalUnwrapPtr *) {
	835	return IrInstGenIdOptionalUnwrapPtr;
1208	836	}
1209	837
1210		static constexpr IrInstSrcId ir_inst_id(IrInstSrcContainerInitFields *) {
1211		return IrInstSrcIdContainerInitFields;
	838	static constexpr IrInstGenId ir_inst_id(IrInstGenOptionalWrap *) {
	839	return IrInstGenIdOptionalWrap;
1212	840	}
1213	841
1214		static constexpr IrInstSrcId ir_inst_id(IrInstSrcUnreachable *) {
1215		return IrInstSrcIdUnreachable;
	842	static constexpr IrInstGenId ir_inst_id(IrInstGenUnionTag *) {
	843	return IrInstGenIdUnionTag;
1216	844	}
1217	845
1218		static constexpr IrInstSrcId ir_inst_id(IrInstSrcTypeOf *) {
1219		return IrInstSrcIdTypeOf;
	846	static constexpr IrInstGenId ir_inst_id(IrInstGenClz *) {
	847	return IrInstGenIdClz;
1220	848	}
1221	849
1222		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSetCold *) {
1223		return IrInstSrcIdSetCold;
	850	static constexpr IrInstGenId ir_inst_id(IrInstGenCtz *) {
	851	return IrInstGenIdCtz;
1224	852	}
1225	853
1226		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSetRuntimeSafety *) {
1227		return IrInstSrcIdSetRuntimeSafety;
	854	static constexpr IrInstGenId ir_inst_id(IrInstGenPopCount *) {
	855	return IrInstGenIdPopCount;
1228	856	}
1229	857
1230		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSetFloatMode *) {
1231		return IrInstSrcIdSetFloatMode;
	858	static constexpr IrInstGenId ir_inst_id(IrInstGenBswap *) {
	859	return IrInstGenIdBswap;
1232	860	}
1233	861
1234		static constexpr IrInstSrcId ir_inst_id(IrInstSrcArrayType *) {
1235		return IrInstSrcIdArrayType;
1236		}
1237
1238		static constexpr IrInstSrcId ir_inst_id(IrInstSrcAnyFrameType *) {
1239		return IrInstSrcIdAnyFrameType;
1240		}
1241
1242		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSliceType *) {
1243		return IrInstSrcIdSliceType;
1244		}
1245
1246		static constexpr IrInstSrcId ir_inst_id(IrInstSrcAsm *) {
1247		return IrInstSrcIdAsm;
	862	static constexpr IrInstGenId ir_inst_id(IrInstGenBitReverse *) {
	863	return IrInstGenIdBitReverse;
1248	864	}
1249	865
1250		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSizeOf *) {
1251		return IrInstSrcIdSizeOf;
	866	static constexpr IrInstGenId ir_inst_id(IrInstGenRef *) {
	867	return IrInstGenIdRef;
1252	868	}
1253	869
1254		static constexpr IrInstSrcId ir_inst_id(IrInstSrcTestNonNull *) {
1255		return IrInstSrcIdTestNonNull;
	870	static constexpr IrInstGenId ir_inst_id(IrInstGenErrName *) {
	871	return IrInstGenIdErrName;
1256	872	}
1257	873
1258		static constexpr IrInstSrcId ir_inst_id(IrInstSrcOptionalUnwrapPtr *) {
1259		return IrInstSrcIdOptionalUnwrapPtr;
	874	static constexpr IrInstGenId ir_inst_id(IrInstGenCmpxchg *) {
	875	return IrInstGenIdCmpxchg;
1260	876	}
1261	877
1262		static constexpr IrInstSrcId ir_inst_id(IrInstSrcClz *) {
1263		return IrInstSrcIdClz;
	878	static constexpr IrInstGenId ir_inst_id(IrInstGenFence *) {
	879	return IrInstGenIdFence;
1264	880	}
1265	881
1266		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCtz *) {
1267		return IrInstSrcIdCtz;
	882	static constexpr IrInstGenId ir_inst_id(IrInstGenReduce *) {
	883	return IrInstGenIdReduce;
1268	884	}
1269	885
1270		static constexpr IrInstSrcId ir_inst_id(IrInstSrcPopCount *) {
1271		return IrInstSrcIdPopCount;
	886	static constexpr IrInstGenId ir_inst_id(IrInstGenTruncate *) {
	887	return IrInstGenIdTruncate;
1272	888	}
1273	889
1274		static constexpr IrInstSrcId ir_inst_id(IrInstSrcBswap *) {
1275		return IrInstSrcIdBswap;
	890	static constexpr IrInstGenId ir_inst_id(IrInstGenShuffleVector *) {
	891	return IrInstGenIdShuffleVector;
1276	892	}
1277	893
1278		static constexpr IrInstSrcId ir_inst_id(IrInstSrcBitReverse *) {
1279		return IrInstSrcIdBitReverse;
	894	static constexpr IrInstGenId ir_inst_id(IrInstGenSplat *) {
	895	return IrInstGenIdSplat;
1280	896	}
1281	897
1282		static constexpr IrInstSrcId ir_inst_id(IrInstSrcImport *) {
1283		return IrInstSrcIdImport;
	898	static constexpr IrInstGenId ir_inst_id(IrInstGenBoolNot *) {
	899	return IrInstGenIdBoolNot;
1284	900	}
1285	901
1286		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCImport *) {
1287		return IrInstSrcIdCImport;
	902	static constexpr IrInstGenId ir_inst_id(IrInstGenMemset *) {
	903	return IrInstGenIdMemset;
1288	904	}
1289	905
1290		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCInclude *) {
1291		return IrInstSrcIdCInclude;
	906	static constexpr IrInstGenId ir_inst_id(IrInstGenMemcpy *) {
	907	return IrInstGenIdMemcpy;
1292	908	}
1293	909
1294		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCDefine *) {
1295		return IrInstSrcIdCDefine;
	910	static constexpr IrInstGenId ir_inst_id(IrInstGenSlice *) {
	911	return IrInstGenIdSlice;
1296	912	}
1297	913
1298		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCUndef *) {
1299		return IrInstSrcIdCUndef;
	914	static constexpr IrInstGenId ir_inst_id(IrInstGenBreakpoint *) {
	915	return IrInstGenIdBreakpoint;
1300	916	}
1301	917
1302		static constexpr IrInstSrcId ir_inst_id(IrInstSrcRef *) {
1303		return IrInstSrcIdRef;
	918	static constexpr IrInstGenId ir_inst_id(IrInstGenReturnAddress *) {
	919	return IrInstGenIdReturnAddress;
1304	920	}
1305	921
1306		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCompileErr *) {
1307		return IrInstSrcIdCompileErr;
	922	static constexpr IrInstGenId ir_inst_id(IrInstGenFrameAddress *) {
	923	return IrInstGenIdFrameAddress;
1308	924	}
1309	925
1310		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCompileLog *) {
1311		return IrInstSrcIdCompileLog;
	926	static constexpr IrInstGenId ir_inst_id(IrInstGenFrameHandle *) {
	927	return IrInstGenIdFrameHandle;
1312	928	}
1313	929
1314		static constexpr IrInstSrcId ir_inst_id(IrInstSrcErrName *) {
1315		return IrInstSrcIdErrName;
	930	static constexpr IrInstGenId ir_inst_id(IrInstGenFrameSize *) {
	931	return IrInstGenIdFrameSize;
1316	932	}
1317	933
1318		static constexpr IrInstSrcId ir_inst_id(IrInstSrcEmbedFile *) {
1319		return IrInstSrcIdEmbedFile;
	934	static constexpr IrInstGenId ir_inst_id(IrInstGenOverflowOp *) {
	935	return IrInstGenIdOverflowOp;
1320	936	}
1321	937
1322		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCmpxchg *) {
1323		return IrInstSrcIdCmpxchg;
	938	static constexpr IrInstGenId ir_inst_id(IrInstGenTestErr *) {
	939	return IrInstGenIdTestErr;
1324	940	}
1325	941
1326		static constexpr IrInstSrcId ir_inst_id(IrInstSrcFence *) {
1327		return IrInstSrcIdFence;
	942	static constexpr IrInstGenId ir_inst_id(IrInstGenMulAdd *) {
	943	return IrInstGenIdMulAdd;
1328	944	}
1329	945
1330		static constexpr IrInstSrcId ir_inst_id(IrInstSrcReduce *) {
1331		return IrInstSrcIdReduce;
	946	static constexpr IrInstGenId ir_inst_id(IrInstGenFloatOp *) {
	947	return IrInstGenIdFloatOp;
1332	948	}
1333	949
1334		static constexpr IrInstSrcId ir_inst_id(IrInstSrcTruncate *) {
1335		return IrInstSrcIdTruncate;
	950	static constexpr IrInstGenId ir_inst_id(IrInstGenUnwrapErrCode *) {
	951	return IrInstGenIdUnwrapErrCode;
1336	952	}
1337	953
1338		static constexpr IrInstSrcId ir_inst_id(IrInstSrcIntCast *) {
1339		return IrInstSrcIdIntCast;
	954	static constexpr IrInstGenId ir_inst_id(IrInstGenUnwrapErrPayload *) {
	955	return IrInstGenIdUnwrapErrPayload;
1340	956	}
1341	957
1342		static constexpr IrInstSrcId ir_inst_id(IrInstSrcFloatCast *) {
1343		return IrInstSrcIdFloatCast;
	958	static constexpr IrInstGenId ir_inst_id(IrInstGenErrWrapCode *) {
	959	return IrInstGenIdErrWrapCode;
1344	960	}
1345	961
1346		static constexpr IrInstSrcId ir_inst_id(IrInstSrcIntToFloat *) {
1347		return IrInstSrcIdIntToFloat;
	962	static constexpr IrInstGenId ir_inst_id(IrInstGenErrWrapPayload *) {
	963	return IrInstGenIdErrWrapPayload;
1348	964	}
1349	965
1350		static constexpr IrInstSrcId ir_inst_id(IrInstSrcFloatToInt *) {
1351		return IrInstSrcIdFloatToInt;
	966	static constexpr IrInstGenId ir_inst_id(IrInstGenPtrCast *) {
	967	return IrInstGenIdPtrCast;
1352	968	}
1353	969
1354		static constexpr IrInstSrcId ir_inst_id(IrInstSrcBoolToInt *) {
1355		return IrInstSrcIdBoolToInt;
	970	static constexpr IrInstGenId ir_inst_id(IrInstGenBitCast *) {
	971	return IrInstGenIdBitCast;
1356	972	}
1357	973
1358		static constexpr IrInstSrcId ir_inst_id(IrInstSrcVectorType *) {
1359		return IrInstSrcIdVectorType;
	974	static constexpr IrInstGenId ir_inst_id(IrInstGenWidenOrShorten *) {
	975	return IrInstGenIdWidenOrShorten;
1360	976	}
1361	977
1362		static constexpr IrInstSrcId ir_inst_id(IrInstSrcShuffleVector *) {
1363		return IrInstSrcIdShuffleVector;
	978	static constexpr IrInstGenId ir_inst_id(IrInstGenIntToPtr *) {
	979	return IrInstGenIdIntToPtr;
1364	980	}
1365	981
1366		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSplat *) {
1367		return IrInstSrcIdSplat;
	982	static constexpr IrInstGenId ir_inst_id(IrInstGenPtrToInt *) {
	983	return IrInstGenIdPtrToInt;
1368	984	}
1369	985
1370		static constexpr IrInstSrcId ir_inst_id(IrInstSrcBoolNot *) {
1371		return IrInstSrcIdBoolNot;
	986	static constexpr IrInstGenId ir_inst_id(IrInstGenIntToEnum *) {
	987	return IrInstGenIdIntToEnum;
1372	988	}
1373	989
1374		static constexpr IrInstSrcId ir_inst_id(IrInstSrcMemset *) {
1375		return IrInstSrcIdMemset;
	990	static constexpr IrInstGenId ir_inst_id(IrInstGenIntToErr *) {
	991	return IrInstGenIdIntToErr;
1376	992	}
1377	993
1378		static constexpr IrInstSrcId ir_inst_id(IrInstSrcMemcpy *) {
1379		return IrInstSrcIdMemcpy;
	994	static constexpr IrInstGenId ir_inst_id(IrInstGenErrToInt *) {
	995	return IrInstGenIdErrToInt;
1380	996	}
1381	997
1382		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSlice *) {
1383		return IrInstSrcIdSlice;
	998	static constexpr IrInstGenId ir_inst_id(IrInstGenPanic *) {
	999	return IrInstGenIdPanic;
1384	1000	}
1385	1001
1386		static constexpr IrInstSrcId ir_inst_id(IrInstSrcBreakpoint *) {
1387		return IrInstSrcIdBreakpoint;
	1002	static constexpr IrInstGenId ir_inst_id(IrInstGenTagName *) {
	1003	return IrInstGenIdTagName;
1388	1004	}
1389	1005
1390		static constexpr IrInstSrcId ir_inst_id(IrInstSrcReturnAddress *) {
1391		return IrInstSrcIdReturnAddress;
	1006	static constexpr IrInstGenId ir_inst_id(IrInstGenFieldParentPtr *) {
	1007	return IrInstGenIdFieldParentPtr;
1392	1008	}
1393	1009
1394		static constexpr IrInstSrcId ir_inst_id(IrInstSrcFrameAddress *) {
1395		return IrInstSrcIdFrameAddress;
	1010	static constexpr IrInstGenId ir_inst_id(IrInstGenAlignCast *) {
	1011	return IrInstGenIdAlignCast;
1396	1012	}
1397	1013
1398		static constexpr IrInstSrcId ir_inst_id(IrInstSrcFrameHandle *) {
1399		return IrInstSrcIdFrameHandle;
	1014	static constexpr IrInstGenId ir_inst_id(IrInstGenErrorReturnTrace *) {
	1015	return IrInstGenIdErrorReturnTrace;
1400	1016	}
1401	1017
1402		static constexpr IrInstSrcId ir_inst_id(IrInstSrcFrameType *) {
1403		return IrInstSrcIdFrameType;
	1018	static constexpr IrInstGenId ir_inst_id(IrInstGenAtomicRmw *) {
	1019	return IrInstGenIdAtomicRmw;
1404	1020	}
1405	1021
1406		static constexpr IrInstSrcId ir_inst_id(IrInstSrcFrameSize *) {
1407		return IrInstSrcIdFrameSize;
	1022	static constexpr IrInstGenId ir_inst_id(IrInstGenAtomicLoad *) {
	1023	return IrInstGenIdAtomicLoad;
1408	1024	}
1409	1025
1410		static constexpr IrInstSrcId ir_inst_id(IrInstSrcAlignOf *) {
1411		return IrInstSrcIdAlignOf;
	1026	static constexpr IrInstGenId ir_inst_id(IrInstGenAtomicStore *) {
	1027	return IrInstGenIdAtomicStore;
1412	1028	}
1413	1029
1414		static constexpr IrInstSrcId ir_inst_id(IrInstSrcOverflowOp *) {
1415		return IrInstSrcIdOverflowOp;
	1030	static constexpr IrInstGenId ir_inst_id(IrInstGenSaveErrRetAddr *) {
	1031	return IrInstGenIdSaveErrRetAddr;
1416	1032	}
1417	1033
1418		static constexpr IrInstSrcId ir_inst_id(IrInstSrcTestErr *) {
1419		return IrInstSrcIdTestErr;
	1034	static constexpr IrInstGenId ir_inst_id(IrInstGenVectorToArray *) {
	1035	return IrInstGenIdVectorToArray;
1420	1036	}
1421	1037
1422		static constexpr IrInstSrcId ir_inst_id(IrInstSrcMulAdd *) {
1423		return IrInstSrcIdMulAdd;
	1038	static constexpr IrInstGenId ir_inst_id(IrInstGenArrayToVector *) {
	1039	return IrInstGenIdArrayToVector;
1424	1040	}
1425	1041
1426		static constexpr IrInstSrcId ir_inst_id(IrInstSrcFloatOp *) {
1427		return IrInstSrcIdFloatOp;
	1042	static constexpr IrInstGenId ir_inst_id(IrInstGenAssertZero *) {
	1043	return IrInstGenIdAssertZero;
1428	1044	}
1429	1045
1430		static constexpr IrInstSrcId ir_inst_id(IrInstSrcUnwrapErrCode *) {
1431		return IrInstSrcIdUnwrapErrCode;
	1046	static constexpr IrInstGenId ir_inst_id(IrInstGenAssertNonNull *) {
	1047	return IrInstGenIdAssertNonNull;
1432	1048	}
1433	1049
1434		static constexpr IrInstSrcId ir_inst_id(IrInstSrcUnwrapErrPayload *) {
1435		return IrInstSrcIdUnwrapErrPayload;
	1050	static constexpr IrInstGenId ir_inst_id(IrInstGenPtrOfArrayToSlice *) {
	1051	return IrInstGenIdPtrOfArrayToSlice;
1436	1052	}
1437	1053
1438		static constexpr IrInstSrcId ir_inst_id(IrInstSrcFnProto *) {
1439		return IrInstSrcIdFnProto;
	1054	static constexpr IrInstGenId ir_inst_id(IrInstGenSuspendBegin *) {
	1055	return IrInstGenIdSuspendBegin;
1440	1056	}
1441	1057
1442		static constexpr IrInstSrcId ir_inst_id(IrInstSrcTestComptime *) {
1443		return IrInstSrcIdTestComptime;
	1058	static constexpr IrInstGenId ir_inst_id(IrInstGenSuspendFinish *) {
	1059	return IrInstGenIdSuspendFinish;
1444	1060	}
1445	1061
1446		static constexpr IrInstSrcId ir_inst_id(IrInstSrcPtrCast *) {
1447		return IrInstSrcIdPtrCast;
	1062	static constexpr IrInstGenId ir_inst_id(IrInstGenAwait *) {
	1063	return IrInstGenIdAwait;
1448	1064	}
1449	1065
1450		static constexpr IrInstSrcId ir_inst_id(IrInstSrcBitCast *) {
1451		return IrInstSrcIdBitCast;
	1066	static constexpr IrInstGenId ir_inst_id(IrInstGenResume *) {
	1067	return IrInstGenIdResume;
1452	1068	}
1453	1069
1454		static constexpr IrInstSrcId ir_inst_id(IrInstSrcIntToPtr *) {
1455		return IrInstSrcIdIntToPtr;
	1070	static constexpr IrInstGenId ir_inst_id(IrInstGenSpillBegin *) {
	1071	return IrInstGenIdSpillBegin;
1456	1072	}
1457	1073
1458		static constexpr IrInstSrcId ir_inst_id(IrInstSrcPtrToInt *) {
1459		return IrInstSrcIdPtrToInt;
	1074	static constexpr IrInstGenId ir_inst_id(IrInstGenSpillEnd *) {
	1075	return IrInstGenIdSpillEnd;
1460	1076	}
1461	1077
1462		static constexpr IrInstSrcId ir_inst_id(IrInstSrcIntToEnum *) {
1463		return IrInstSrcIdIntToEnum;
	1078	static constexpr IrInstGenId ir_inst_id(IrInstGenVectorExtractElem *) {
	1079	return IrInstGenIdVectorExtractElem;
1464	1080	}
1465	1081
1466		static constexpr IrInstSrcId ir_inst_id(IrInstSrcEnumToInt *) {
1467		return IrInstSrcIdEnumToInt;
	1082	static constexpr IrInstGenId ir_inst_id(IrInstGenAlloca *) {
	1083	return IrInstGenIdAlloca;
1468	1084	}
1469	1085
1470		static constexpr IrInstSrcId ir_inst_id(IrInstSrcIntToErr *) {
1471		return IrInstSrcIdIntToErr;
	1086	static constexpr IrInstGenId ir_inst_id(IrInstGenConst *) {
	1087	return IrInstGenIdConst;
1472	1088	}
1473	1089
1474		static constexpr IrInstSrcId ir_inst_id(IrInstSrcErrToInt *) {
1475		return IrInstSrcIdErrToInt;
	1090	static constexpr IrInstGenId ir_inst_id(IrInstGenWasmMemorySize *) {
	1091	return IrInstGenIdWasmMemorySize;
1476	1092	}
1477	1093
1478		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCheckStatementIsVoid *) {
1479		return IrInstSrcIdCheckStatementIsVoid;
	1094	static constexpr IrInstGenId ir_inst_id(IrInstGenWasmMemoryGrow *) {
	1095	return IrInstGenIdWasmMemoryGrow;
1480	1096	}
1481	1097
1482		static constexpr IrInstSrcId ir_inst_id(IrInstSrcTypeName *) {
1483		return IrInstSrcIdTypeName;
	1098	static constexpr IrInstGenId ir_inst_id(IrInstGenExtern *) {
	1099	return IrInstGenIdExtern;
1484	1100	}
1485	1101
1486		static constexpr IrInstSrcId ir_inst_id(IrInstSrcDeclRef *) {
1487		return IrInstSrcIdDeclRef;
	1102	template<typename T>
	1103	static T ir_create_inst_gen(IrBuilderGen irb, Scope scope, AstNode source_node) {
	1104	T *special_instruction = heap::c_allocator.create<T>();
	1105	special_instruction->base.id = ir_inst_id(special_instruction);
	1106	special_instruction->base.base.scope = scope;
	1107	special_instruction->base.base.source_node = source_node;
	1108	special_instruction->base.base.debug_id = exec_next_debug_id_gen(irb->exec);
	1109	special_instruction->base.owner_bb = irb->current_basic_block;
	1110	special_instruction->base.value = irb->codegen->pass1_arena->create<ZigValue>();
	1111	return special_instruction;
1488	1112	}
1489	1113
1490		static constexpr IrInstSrcId ir_inst_id(IrInstSrcPanic *) {
1491		return IrInstSrcIdPanic;
	1114	template<typename T>
	1115	static T ir_create_inst_noval(IrBuilderGen irb, Scope scope, AstNode source_node) {
	1116	T *special_instruction = heap::c_allocator.create<T>();
	1117	special_instruction->base.id = ir_inst_id(special_instruction);
	1118	special_instruction->base.base.scope = scope;
	1119	special_instruction->base.base.source_node = source_node;
	1120	special_instruction->base.base.debug_id = exec_next_debug_id_gen(irb->exec);
	1121	special_instruction->base.owner_bb = irb->current_basic_block;
	1122	return special_instruction;
1492	1123	}
1493	1124
1494		static constexpr IrInstSrcId ir_inst_id(IrInstSrcTagName *) {
1495		return IrInstSrcIdTagName;
	1125	template<typename T>
	1126	static T ir_build_inst_gen(IrBuilderGen irb, Scope scope, AstNode source_node) {
	1127	T *special_instruction = ir_create_inst_gen<T>(irb, scope, source_node);
	1128	ir_inst_gen_append(irb->current_basic_block, &special_instruction->base);
	1129	return special_instruction;
1496	1130	}
1497	1131
1498		static constexpr IrInstSrcId ir_inst_id(IrInstSrcFieldParentPtr *) {
1499		return IrInstSrcIdFieldParentPtr;
	1132	template<typename T>
	1133	static T ir_build_inst_noreturn(IrBuilderGen irb, Scope scope, AstNode source_node) {
	1134	T *special_instruction = ir_create_inst_noval<T>(irb, scope, source_node);
	1135	special_instruction->base.value = irb->codegen->intern.for_unreachable();
	1136	ir_inst_gen_append(irb->current_basic_block, &special_instruction->base);
	1137	return special_instruction;
1500	1138	}
1501	1139
1502		static constexpr IrInstSrcId ir_inst_id(IrInstSrcByteOffsetOf *) {
1503		return IrInstSrcIdByteOffsetOf;
	1140	template<typename T>
	1141	static T ir_build_inst_void(IrBuilderGen irb, Scope scope, AstNode source_node) {
	1142	T *special_instruction = ir_create_inst_noval<T>(irb, scope, source_node);
	1143	special_instruction->base.value = irb->codegen->intern.for_void();
	1144	ir_inst_gen_append(irb->current_basic_block, &special_instruction->base);
	1145	return special_instruction;
1504	1146	}
1505	1147
1506		static constexpr IrInstSrcId ir_inst_id(IrInstSrcBitOffsetOf *) {
1507		return IrInstSrcIdBitOffsetOf;
	1148	IrInstGen ir_create_alloca(CodeGen g, Scope scope, AstNode source_node, ZigFn *fn,
	1149	ZigType var_type, const char name_hint)
	1150	{
	1151	IrInstGenAlloca *alloca_gen = heap::c_allocator.create<IrInstGenAlloca>();
	1152	alloca_gen->base.id = IrInstGenIdAlloca;
	1153	alloca_gen->base.base.source_node = source_node;
	1154	alloca_gen->base.base.scope = scope;
	1155	alloca_gen->base.value = g->pass1_arena->create<ZigValue>();
	1156	alloca_gen->base.value->type = get_pointer_to_type(g, var_type, false);
	1157	alloca_gen->base.base.ref_count = 1;
	1158	alloca_gen->name_hint = name_hint;
	1159	fn->alloca_gen_list.append(alloca_gen);
	1160	return &alloca_gen->base;
1508	1161	}
1509	1162
1510		static constexpr IrInstSrcId ir_inst_id(IrInstSrcTypeInfo *) {
1511		return IrInstSrcIdTypeInfo;
1512		}
	1163	static IrInstGen ir_build_cast(IrAnalyze ira, IrInst source_instr,ZigType dest_type,
	1164	IrInstGen *value, CastOp cast_op)
	1165	{
	1166	IrInstGenCast *inst = ir_build_inst_gen<IrInstGenCast>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	1167	inst->base.value->type = dest_type;
	1168	inst->value = value;
	1169	inst->cast_op = cast_op;
1513	1170
1514		static constexpr IrInstSrcId ir_inst_id(IrInstSrcType *) {
1515		return IrInstSrcIdType;
1516		}
	1171	ir_ref_inst_gen(value);
1517	1172
1518		static constexpr IrInstSrcId ir_inst_id(IrInstSrcHasField *) {
1519		return IrInstSrcIdHasField;
	1173	return &inst->base;
1520	1174	}
1521	1175
1522		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSetEvalBranchQuota *) {
1523		return IrInstSrcIdSetEvalBranchQuota;
1524		}
	1176	static IrInstGen ir_build_cond_br_gen(IrAnalyze ira, IrInst source_instr, IrInstGen condition,
	1177	IrBasicBlockGen then_block, IrBasicBlockGen else_block)
	1178	{
	1179	IrInstGenCondBr *inst = ir_build_inst_noreturn<IrInstGenCondBr>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	1180	inst->condition = condition;
	1181	inst->then_block = then_block;
	1182	inst->else_block = else_block;
1525	1183
1526		static constexpr IrInstSrcId ir_inst_id(IrInstSrcPtrType *) {
1527		return IrInstSrcIdPtrType;
1528		}
	1184	ir_ref_inst_gen(condition);
1529	1185
1530		static constexpr IrInstSrcId ir_inst_id(IrInstSrcAlignCast *) {
1531		return IrInstSrcIdAlignCast;
	1186	return &inst->base;
1532	1187	}
1533	1188
1534		static constexpr IrInstSrcId ir_inst_id(IrInstSrcImplicitCast *) {
1535		return IrInstSrcIdImplicitCast;
1536		}
	1189	static IrInstGen ir_build_return_gen(IrAnalyze ira, IrInst source_inst, IrInstGen operand) {
	1190	IrInstGenReturn *inst = ir_build_inst_noreturn<IrInstGenReturn>(&ira->new_irb,
	1191	source_inst->scope, source_inst->source_node);
	1192	inst->operand = operand;
1537	1193
1538		static constexpr IrInstSrcId ir_inst_id(IrInstSrcResolveResult *) {
1539		return IrInstSrcIdResolveResult;
1540		}
	1194	if (operand != nullptr) ir_ref_inst_gen(operand);
1541	1195
1542		static constexpr IrInstSrcId ir_inst_id(IrInstSrcResetResult *) {
1543		return IrInstSrcIdResetResult;
	1196	return &inst->base;
1544	1197	}
1545	1198
1546		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSetAlignStack *) {
1547		return IrInstSrcIdSetAlignStack;
1548		}
	1199	static IrInstGen ir_build_bin_op_gen(IrAnalyze ira, IrInst source_instr, ZigType res_type,
	1200	IrBinOp op_id, IrInstGen op1, IrInstGen op2, bool safety_check_on)
	1201	{
	1202	IrInstGenBinOp *inst = ir_build_inst_gen<IrInstGenBinOp>(&ira->new_irb,
	1203	source_instr->scope, source_instr->source_node);
	1204	inst->base.value->type = res_type;
	1205	inst->op_id = op_id;
	1206	inst->op1 = op1;
	1207	inst->op2 = op2;
	1208	inst->safety_check_on = safety_check_on;
1549	1209
1550		static constexpr IrInstSrcId ir_inst_id(IrInstSrcExport *) {
1551		return IrInstSrcIdExport;
1552		}
	1210	ir_ref_inst_gen(op1);
	1211	ir_ref_inst_gen(op2);
1553	1212
1554		static constexpr IrInstSrcId ir_inst_id(IrInstSrcExtern *) {
1555		return IrInstSrcIdExtern;
	1213	return &inst->base;
1556	1214	}
1557	1215
1558		static constexpr IrInstSrcId ir_inst_id(IrInstSrcErrorReturnTrace *) {
1559		return IrInstSrcIdErrorReturnTrace;
1560		}
1561	1216
1562		static constexpr IrInstSrcId ir_inst_id(IrInstSrcErrorUnion *) {
1563		return IrInstSrcIdErrorUnion;
1564		}
	1217	static IrInstGen ir_build_var_ptr_gen(IrAnalyze ira, IrInst source_instr, ZigVar var) {
	1218	IrInstGenVarPtr *instruction = ir_build_inst_gen<IrInstGenVarPtr>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	1219	instruction->var = var;
1565	1220
1566		static constexpr IrInstSrcId ir_inst_id(IrInstSrcAtomicRmw *) {
1567		return IrInstSrcIdAtomicRmw;
1568		}
	1221	var->ref_count += 1;
1569	1222
1570		static constexpr IrInstSrcId ir_inst_id(IrInstSrcAtomicLoad *) {
1571		return IrInstSrcIdAtomicLoad;
	1223	return &instruction->base;
1572	1224	}
1573	1225
1574		static constexpr IrInstSrcId ir_inst_id(IrInstSrcAtomicStore *) {
1575		return IrInstSrcIdAtomicStore;
	1226	static IrInstGen ir_build_return_ptr(IrAnalyze ira, Scope scope, AstNode source_node, ZigType *ty) {
	1227	IrInstGenReturnPtr *instruction = ir_build_inst_gen<IrInstGenReturnPtr>(&ira->new_irb, scope, source_node);
	1228	instruction->base.value->type = ty;
	1229	return &instruction->base;
1576	1230	}
1577	1231
1578		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSaveErrRetAddr *) {
1579		return IrInstSrcIdSaveErrRetAddr;
1580		}
	1232	static IrInstGen ir_build_elem_ptr_gen(IrAnalyze ira, Scope scope, AstNode source_node,
	1233	IrInstGen array_ptr, IrInstGen elem_index, bool safety_check_on, ZigType *return_type)
	1234	{
	1235	IrInstGenElemPtr *instruction = ir_build_inst_gen<IrInstGenElemPtr>(&ira->new_irb, scope, source_node);
	1236	instruction->base.value->type = return_type;
	1237	instruction->array_ptr = array_ptr;
	1238	instruction->elem_index = elem_index;
	1239	instruction->safety_check_on = safety_check_on;
1581	1240
1582		static constexpr IrInstSrcId ir_inst_id(IrInstSrcAddImplicitReturnType *) {
1583		return IrInstSrcIdAddImplicitReturnType;
1584		}
	1241	ir_ref_inst_gen(array_ptr);
	1242	ir_ref_inst_gen(elem_index);
1585	1243
1586		static constexpr IrInstSrcId ir_inst_id(IrInstSrcErrSetCast *) {
1587		return IrInstSrcIdErrSetCast;
	1244	return &instruction->base;
1588	1245	}
1589	1246
1590		static constexpr IrInstSrcId ir_inst_id(IrInstSrcCheckRuntimeScope *) {
1591		return IrInstSrcIdCheckRuntimeScope;
1592		}
	1247	static IrInstGen ir_build_struct_field_ptr(IrAnalyze ira, IrInst *source_instr,
	1248	IrInstGen struct_ptr, TypeStructField field, ZigType *ptr_type)
	1249	{
	1250	IrInstGenStructFieldPtr *inst = ir_build_inst_gen<IrInstGenStructFieldPtr>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	1251	inst->base.value->type = ptr_type;
	1252	inst->struct_ptr = struct_ptr;
	1253	inst->field = field;
1593	1254
1594		static constexpr IrInstSrcId ir_inst_id(IrInstSrcHasDecl *) {
1595		return IrInstSrcIdHasDecl;
1596		}
	1255	ir_ref_inst_gen(struct_ptr);
1597	1256
1598		static constexpr IrInstSrcId ir_inst_id(IrInstSrcUndeclaredIdent *) {
1599		return IrInstSrcIdUndeclaredIdent;
	1257	return &inst->base;
1600	1258	}
1601	1259
1602		static constexpr IrInstSrcId ir_inst_id(IrInstSrcAlloca *) {
1603		return IrInstSrcIdAlloca;
1604		}
	1260	static IrInstGen ir_build_union_field_ptr(IrAnalyze ira, IrInst *source_instr,
	1261	IrInstGen union_ptr, TypeUnionField field, bool safety_check_on, bool initializing, ZigType *ptr_type)
	1262	{
	1263	IrInstGenUnionFieldPtr *inst = ir_build_inst_gen<IrInstGenUnionFieldPtr>(&ira->new_irb,
	1264	source_instr->scope, source_instr->source_node);
	1265	inst->base.value->type = ptr_type;
	1266	inst->initializing = initializing;
	1267	inst->safety_check_on = safety_check_on;
	1268	inst->union_ptr = union_ptr;
	1269	inst->field = field;
1605	1270
1606		static constexpr IrInstSrcId ir_inst_id(IrInstSrcEndExpr *) {
1607		return IrInstSrcIdEndExpr;
1608		}
	1271	ir_ref_inst_gen(union_ptr);
1609	1272
1610		static constexpr IrInstSrcId ir_inst_id(IrInstSrcUnionInitNamedField *) {
1611		return IrInstSrcIdUnionInitNamedField;
	1273	return &inst->base;
1612	1274	}
1613	1275
1614		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSuspendBegin *) {
1615		return IrInstSrcIdSuspendBegin;
1616		}
	1276	static IrInstGenCall ir_build_call_gen(IrAnalyze ira, IrInst *source_instruction,
	1277	ZigFn fn_entry, IrInstGen fn_ref, size_t arg_count, IrInstGen **args,
	1278	CallModifier modifier, IrInstGen *new_stack, bool is_async_call_builtin,
	1279	IrInstGen result_loc, ZigType return_type)
	1280	{
	1281	IrInstGenCall *call_instruction = ir_build_inst_gen<IrInstGenCall>(&ira->new_irb,
	1282	source_instruction->scope, source_instruction->source_node);
	1283	call_instruction->base.value->type = return_type;
	1284	call_instruction->fn_entry = fn_entry;
	1285	call_instruction->fn_ref = fn_ref;
	1286	call_instruction->args = args;
	1287	call_instruction->arg_count = arg_count;
	1288	call_instruction->modifier = modifier;
	1289	call_instruction->is_async_call_builtin = is_async_call_builtin;
	1290	call_instruction->new_stack = new_stack;
	1291	call_instruction->result_loc = result_loc;
1617	1292
1618		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSuspendFinish *) {
1619		return IrInstSrcIdSuspendFinish;
1620		}
	1293	if (fn_ref != nullptr) ir_ref_inst_gen(fn_ref);
	1294	for (size_t i = 0; i < arg_count; i += 1)
	1295	ir_ref_inst_gen(args[i]);
	1296	if (new_stack != nullptr) ir_ref_inst_gen(new_stack);
	1297	if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
1621	1298
1622		static constexpr IrInstSrcId ir_inst_id(IrInstSrcAwait *) {
1623		return IrInstSrcIdAwait;
	1299	return call_instruction;
1624	1300	}
1625	1301
1626		static constexpr IrInstSrcId ir_inst_id(IrInstSrcResume *) {
1627		return IrInstSrcIdResume;
1628		}
	1302	static IrInstGen ir_build_phi_gen(IrAnalyze ira, IrInst *source_instr, size_t incoming_count,
	1303	IrBasicBlockGen incoming_blocks, IrInstGen incoming_values, ZigType *result_type)
	1304	{
	1305	assert(incoming_count != 0);
	1306	assert(incoming_count != SIZE_MAX);
1629	1307
1630		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSpillBegin *) {
1631		return IrInstSrcIdSpillBegin;
1632		}
	1308	IrInstGenPhi *phi_instruction = ir_build_inst_gen<IrInstGenPhi>(&ira->new_irb,
	1309	source_instr->scope, source_instr->source_node);
	1310	phi_instruction->base.value->type = result_type;
	1311	phi_instruction->incoming_count = incoming_count;
	1312	phi_instruction->incoming_blocks = incoming_blocks;
	1313	phi_instruction->incoming_values = incoming_values;
1633	1314
1634		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSpillEnd *) {
1635		return IrInstSrcIdSpillEnd;
1636		}
	1315	for (size_t i = 0; i < incoming_count; i += 1) {
	1316	ir_ref_inst_gen(incoming_values[i]);
	1317	}
1637	1318
1638		static constexpr IrInstSrcId ir_inst_id(IrInstSrcWasmMemorySize *) {
1639		return IrInstSrcIdWasmMemorySize;
	1319	return &phi_instruction->base;
1640	1320	}
1641	1321
1642		static constexpr IrInstSrcId ir_inst_id(IrInstSrcWasmMemoryGrow *) {
1643		return IrInstSrcIdWasmMemoryGrow;
1644		}
	1322	static IrInstGen ir_build_br_gen(IrAnalyze ira, IrInst source_instr, IrBasicBlockGen dest_block) {
	1323	IrInstGenBr *inst = ir_build_inst_noreturn<IrInstGenBr>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	1324	inst->dest_block = dest_block;
1645	1325
1646		static constexpr IrInstSrcId ir_inst_id(IrInstSrcSrc *) {
1647		return IrInstSrcIdSrc;
	1326	return &inst->base;
1648	1327	}
1649	1328
1650		static constexpr IrInstGenId ir_inst_id(IrInstGenDeclVar *) {
1651		return IrInstGenIdDeclVar;
1652		}
	1329	static IrInstGen ir_build_negation(IrAnalyze ira, IrInst source_instr, IrInstGen operand, ZigType *expr_type, bool wrapping) {
	1330	IrInstGenNegation *instruction = ir_build_inst_gen<IrInstGenNegation>(&ira->new_irb,
	1331	source_instr->scope, source_instr->source_node);
	1332	instruction->base.value->type = expr_type;
	1333	instruction->operand = operand;
	1334	instruction->wrapping = wrapping;
1653	1335
1654		static constexpr IrInstGenId ir_inst_id(IrInstGenBr *) {
1655		return IrInstGenIdBr;
1656		}
	1336	ir_ref_inst_gen(operand);
1657	1337
1658		static constexpr IrInstGenId ir_inst_id(IrInstGenCondBr *) {
1659		return IrInstGenIdCondBr;
	1338	return &instruction->base;
1660	1339	}
1661	1340
1662		static constexpr IrInstGenId ir_inst_id(IrInstGenSwitchBr *) {
1663		return IrInstGenIdSwitchBr;
1664		}
	1341	static IrInstGen ir_build_binary_not(IrAnalyze ira, IrInst source_instr, IrInstGen operand,
	1342	ZigType *expr_type)
	1343	{
	1344	IrInstGenBinaryNot *instruction = ir_build_inst_gen<IrInstGenBinaryNot>(&ira->new_irb,
	1345	source_instr->scope, source_instr->source_node);
	1346	instruction->base.value->type = expr_type;
	1347	instruction->operand = operand;
1665	1348
1666		static constexpr IrInstGenId ir_inst_id(IrInstGenPhi *) {
1667		return IrInstGenIdPhi;
1668		}
	1349	ir_ref_inst_gen(operand);
1669	1350
1670		static constexpr IrInstGenId ir_inst_id(IrInstGenBinaryNot *) {
1671		return IrInstGenIdBinaryNot;
	1351	return &instruction->base;
1672	1352	}
1673	1353
1674		static constexpr IrInstGenId ir_inst_id(IrInstGenNegation *) {
1675		return IrInstGenIdNegation;
	1354	static IrInstGen ir_build_unreachable_gen(IrAnalyze ira, IrInst *source_instr) {
	1355	IrInstGenUnreachable *inst = ir_build_inst_noreturn<IrInstGenUnreachable>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	1356	return &inst->base;
1676	1357	}
1677	1358
1678		static constexpr IrInstGenId ir_inst_id(IrInstGenBinOp *) {
1679		return IrInstGenIdBinOp;
1680		}
	1359	static IrInstGen ir_build_store_ptr_gen(IrAnalyze ira, IrInst source_instr, IrInstGen ptr, IrInstGen *value) {
	1360	IrInstGenStorePtr *instruction = ir_build_inst_void<IrInstGenStorePtr>(&ira->new_irb,
	1361	source_instr->scope, source_instr->source_node);
	1362	instruction->ptr = ptr;
	1363	instruction->value = value;
1681	1364
1682		static constexpr IrInstGenId ir_inst_id(IrInstGenLoadPtr *) {
1683		return IrInstGenIdLoadPtr;
1684		}
	1365	ir_ref_inst_gen(ptr);
	1366	ir_ref_inst_gen(value);
1685	1367
1686		static constexpr IrInstGenId ir_inst_id(IrInstGenStorePtr *) {
1687		return IrInstGenIdStorePtr;
	1368	return &instruction->base;
1688	1369	}
1689	1370
1690		static constexpr IrInstGenId ir_inst_id(IrInstGenVectorStoreElem *) {
1691		return IrInstGenIdVectorStoreElem;
1692		}
	1371	static IrInstGen ir_build_vector_store_elem(IrAnalyze ira, IrInst *src_inst,
	1372	IrInstGen vector_ptr, IrInstGen index, IrInstGen *value)
	1373	{
	1374	IrInstGenVectorStoreElem *inst = ir_build_inst_void<IrInstGenVectorStoreElem>(
	1375	&ira->new_irb, src_inst->scope, src_inst->source_node);
	1376	inst->vector_ptr = vector_ptr;
	1377	inst->index = index;
	1378	inst->value = value;
1693	1379
1694		static constexpr IrInstGenId ir_inst_id(IrInstGenStructFieldPtr *) {
1695		return IrInstGenIdStructFieldPtr;
1696		}
	1380	ir_ref_inst_gen(vector_ptr);
	1381	ir_ref_inst_gen(index);
	1382	ir_ref_inst_gen(value);
1697	1383
1698		static constexpr IrInstGenId ir_inst_id(IrInstGenUnionFieldPtr *) {
1699		return IrInstGenIdUnionFieldPtr;
	1384	return &inst->base;
1700	1385	}
1701	1386
1702		static constexpr IrInstGenId ir_inst_id(IrInstGenElemPtr *) {
1703		return IrInstGenIdElemPtr;
1704		}
	1387	static IrInstGen ir_build_var_decl_gen(IrAnalyze ira, IrInst *source_instruction,
	1388	ZigVar var, IrInstGen var_ptr)
	1389	{
	1390	IrInstGenDeclVar *inst = ir_build_inst_gen<IrInstGenDeclVar>(&ira->new_irb,
	1391	source_instruction->scope, source_instruction->source_node);
	1392	inst->base.value->special = ConstValSpecialStatic;
	1393	inst->base.value->type = ira->codegen->builtin_types.entry_void;
	1394	inst->var = var;
	1395	inst->var_ptr = var_ptr;
1705	1396
1706		static constexpr IrInstGenId ir_inst_id(IrInstGenVarPtr *) {
1707		return IrInstGenIdVarPtr;
1708		}
	1397	ir_ref_inst_gen(var_ptr);
1709	1398
1710		static constexpr IrInstGenId ir_inst_id(IrInstGenReturnPtr *) {
1711		return IrInstGenIdReturnPtr;
	1399	return &inst->base;
1712	1400	}
1713	1401
1714		static constexpr IrInstGenId ir_inst_id(IrInstGenCall *) {
1715		return IrInstGenIdCall;
1716		}
	1402	static IrInstGen ir_build_extern_gen(IrAnalyze ira, IrInst source_instr, Buf name,
	1403	GlobalLinkageId linkage, bool is_thread_local, ZigType *expr_type)
	1404	{
	1405	IrInstGenExtern *instruction = ir_build_inst_gen<IrInstGenExtern>(&ira->new_irb,
	1406	source_instr->scope, source_instr->source_node);
	1407	instruction->base.value->type = expr_type;
	1408	instruction->name = name;
	1409	instruction->linkage = linkage;
	1410	instruction->is_thread_local = is_thread_local;
1717	1411
1718		static constexpr IrInstGenId ir_inst_id(IrInstGenReturn *) {
1719		return IrInstGenIdReturn;
	1412	return &instruction->base;
1720	1413	}
1721	1414
1722		static constexpr IrInstGenId ir_inst_id(IrInstGenCast *) {
1723		return IrInstGenIdCast;
1724		}
	1415	static IrInstGen ir_build_load_ptr_gen(IrAnalyze ira, IrInst *source_instruction,
	1416	IrInstGen ptr, ZigType ty, IrInstGen *result_loc)
	1417	{
	1418	IrInstGenLoadPtr *instruction = ir_build_inst_gen<IrInstGenLoadPtr>(
	1419	&ira->new_irb, source_instruction->scope, source_instruction->source_node);
	1420	instruction->base.value->type = ty;
	1421	instruction->ptr = ptr;
	1422	instruction->result_loc = result_loc;
1725	1423
1726		static constexpr IrInstGenId ir_inst_id(IrInstGenUnreachable *) {
1727		return IrInstGenIdUnreachable;
1728		}
	1424	ir_ref_inst_gen(ptr);
	1425	if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
1729	1426
1730		static constexpr IrInstGenId ir_inst_id(IrInstGenAsm *) {
1731		return IrInstGenIdAsm;
	1427	return &instruction->base;
1732	1428	}
1733	1429
1734		static constexpr IrInstGenId ir_inst_id(IrInstGenTestNonNull *) {
1735		return IrInstGenIdTestNonNull;
1736		}
	1430	static IrInstGen ir_build_asm_gen(IrAnalyze ira, IrInst *source_instr,
	1431	Buf asm_template, AsmToken token_list, size_t token_list_len,
	1432	IrInstGen input_list, IrInstGen output_types, ZigVar **output_vars, size_t return_count,
	1433	bool has_side_effects, ZigType *return_type)
	1434	{
	1435	IrInstGenAsm *instruction = ir_build_inst_gen<IrInstGenAsm>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	1436	instruction->base.value->type = return_type;
	1437	instruction->asm_template = asm_template;
	1438	instruction->token_list = token_list;
	1439	instruction->token_list_len = token_list_len;
	1440	instruction->input_list = input_list;
	1441	instruction->output_types = output_types;
	1442	instruction->output_vars = output_vars;
	1443	instruction->return_count = return_count;
	1444	instruction->has_side_effects = has_side_effects;
1737	1445
1738		static constexpr IrInstGenId ir_inst_id(IrInstGenOptionalUnwrapPtr *) {
1739		return IrInstGenIdOptionalUnwrapPtr;
1740		}
	1446	assert(source_instr->source_node->type == NodeTypeAsmExpr);
	1447	for (size_t i = 0; i < source_instr->source_node->data.asm_expr.output_list.length; i += 1) {
	1448	IrInstGen *output_type = output_types[i];
	1449	if (output_type) ir_ref_inst_gen(output_type);
	1450	}
1741	1451
1742		static constexpr IrInstGenId ir_inst_id(IrInstGenOptionalWrap *) {
1743		return IrInstGenIdOptionalWrap;
1744		}
	1452	for (size_t i = 0; i < source_instr->source_node->data.asm_expr.input_list.length; i += 1) {
	1453	IrInstGen *input_value = input_list[i];
	1454	ir_ref_inst_gen(input_value);
	1455	}
1745	1456
1746		static constexpr IrInstGenId ir_inst_id(IrInstGenUnionTag *) {
1747		return IrInstGenIdUnionTag;
	1457	return &instruction->base;
1748	1458	}
1749	1459
1750		static constexpr IrInstGenId ir_inst_id(IrInstGenClz *) {
1751		return IrInstGenIdClz;
1752		}
	1460	static IrInstGen ir_build_test_non_null_gen(IrAnalyze ira, IrInst source_instr, IrInstGen value) {
	1461	IrInstGenTestNonNull *inst = ir_build_inst_gen<IrInstGenTestNonNull>(&ira->new_irb,
	1462	source_instr->scope, source_instr->source_node);
	1463	inst->base.value->type = ira->codegen->builtin_types.entry_bool;
	1464	inst->value = value;
1753	1465
1754		static constexpr IrInstGenId ir_inst_id(IrInstGenCtz *) {
1755		return IrInstGenIdCtz;
1756		}
	1466	ir_ref_inst_gen(value);
1757	1467
1758		static constexpr IrInstGenId ir_inst_id(IrInstGenPopCount *) {
1759		return IrInstGenIdPopCount;
	1468	return &inst->base;
1760	1469	}
1761	1470
1762		static constexpr IrInstGenId ir_inst_id(IrInstGenBswap *) {
1763		return IrInstGenIdBswap;
1764		}
	1471	static IrInstGen ir_build_optional_unwrap_ptr_gen(IrAnalyze ira, IrInst *source_instr,
	1472	IrInstGen base_ptr, bool safety_check_on, bool initializing, ZigType result_type)
	1473	{
	1474	IrInstGenOptionalUnwrapPtr *inst = ir_build_inst_gen<IrInstGenOptionalUnwrapPtr>(&ira->new_irb,
	1475	source_instr->scope, source_instr->source_node);
	1476	inst->base.value->type = result_type;
	1477	inst->base_ptr = base_ptr;
	1478	inst->safety_check_on = safety_check_on;
	1479	inst->initializing = initializing;
1765	1480
1766		static constexpr IrInstGenId ir_inst_id(IrInstGenBitReverse *) {
1767		return IrInstGenIdBitReverse;
	1481	ir_ref_inst_gen(base_ptr);
	1482
	1483	return &inst->base;
1768	1484	}
1769	1485
1770		static constexpr IrInstGenId ir_inst_id(IrInstGenRef *) {
1771		return IrInstGenIdRef;
1772		}
	1486	static IrInstGen ir_build_optional_wrap(IrAnalyze ira, IrInst source_instruction, ZigType result_ty,
	1487	IrInstGen operand, IrInstGen result_loc)
	1488	{
	1489	IrInstGenOptionalWrap *instruction = ir_build_inst_gen<IrInstGenOptionalWrap>(
	1490	&ira->new_irb, source_instruction->scope, source_instruction->source_node);
	1491	instruction->base.value->type = result_ty;
	1492	instruction->operand = operand;
	1493	instruction->result_loc = result_loc;
1773	1494
1774		static constexpr IrInstGenId ir_inst_id(IrInstGenErrName *) {
1775		return IrInstGenIdErrName;
1776		}
	1495	ir_ref_inst_gen(operand);
	1496	if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
1777	1497
1778		static constexpr IrInstGenId ir_inst_id(IrInstGenCmpxchg *) {
1779		return IrInstGenIdCmpxchg;
	1498	return &instruction->base;
1780	1499	}
1781	1500
1782		static constexpr IrInstGenId ir_inst_id(IrInstGenFence *) {
1783		return IrInstGenIdFence;
1784		}
	1501	static IrInstGen ir_build_err_wrap_payload(IrAnalyze ira, IrInst *source_instruction,
	1502	ZigType result_type, IrInstGen operand, IrInstGen *result_loc)
	1503	{
	1504	IrInstGenErrWrapPayload *instruction = ir_build_inst_gen<IrInstGenErrWrapPayload>(
	1505	&ira->new_irb, source_instruction->scope, source_instruction->source_node);
	1506	instruction->base.value->type = result_type;
	1507	instruction->operand = operand;
	1508	instruction->result_loc = result_loc;
1785	1509
1786		static constexpr IrInstGenId ir_inst_id(IrInstGenReduce *) {
1787		return IrInstGenIdReduce;
1788		}
	1510	ir_ref_inst_gen(operand);
	1511	if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
1789	1512
1790		static constexpr IrInstGenId ir_inst_id(IrInstGenTruncate *) {
1791		return IrInstGenIdTruncate;
	1513	return &instruction->base;
1792	1514	}
1793	1515
1794		static constexpr IrInstGenId ir_inst_id(IrInstGenShuffleVector *) {
1795		return IrInstGenIdShuffleVector;
1796		}
	1516	static IrInstGen ir_build_err_wrap_code(IrAnalyze ira, IrInst *source_instruction,
	1517	ZigType result_type, IrInstGen operand, IrInstGen *result_loc)
	1518	{
	1519	IrInstGenErrWrapCode *instruction = ir_build_inst_gen<IrInstGenErrWrapCode>(
	1520	&ira->new_irb, source_instruction->scope, source_instruction->source_node);
	1521	instruction->base.value->type = result_type;
	1522	instruction->operand = operand;
	1523	instruction->result_loc = result_loc;
1797	1524
1798		static constexpr IrInstGenId ir_inst_id(IrInstGenSplat *) {
1799		return IrInstGenIdSplat;
1800		}
	1525	ir_ref_inst_gen(operand);
	1526	if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
1801	1527
1802		static constexpr IrInstGenId ir_inst_id(IrInstGenBoolNot *) {
1803		return IrInstGenIdBoolNot;
	1528	return &instruction->base;
1804	1529	}
1805	1530
1806		static constexpr IrInstGenId ir_inst_id(IrInstGenMemset *) {
1807		return IrInstGenIdMemset;
1808		}
	1531	static IrInstGen ir_build_clz_gen(IrAnalyze ira, IrInst source_instr, ZigType result_type, IrInstGen *op) {
	1532	IrInstGenClz *instruction = ir_build_inst_gen<IrInstGenClz>(&ira->new_irb,
	1533	source_instr->scope, source_instr->source_node);
	1534	instruction->base.value->type = result_type;
	1535	instruction->op = op;
1809	1536
1810		static constexpr IrInstGenId ir_inst_id(IrInstGenMemcpy *) {
1811		return IrInstGenIdMemcpy;
1812		}
	1537	ir_ref_inst_gen(op);
1813	1538
1814		static constexpr IrInstGenId ir_inst_id(IrInstGenSlice *) {
1815		return IrInstGenIdSlice;
	1539	return &instruction->base;
1816	1540	}
1817	1541
1818		static constexpr IrInstGenId ir_inst_id(IrInstGenBreakpoint *) {
1819		return IrInstGenIdBreakpoint;
1820		}
	1542	static IrInstGen ir_build_ctz_gen(IrAnalyze ira, IrInst source_instr, ZigType result_type, IrInstGen *op) {
	1543	IrInstGenCtz *instruction = ir_build_inst_gen<IrInstGenCtz>(&ira->new_irb,
	1544	source_instr->scope, source_instr->source_node);
	1545	instruction->base.value->type = result_type;
	1546	instruction->op = op;
1821	1547
1822		static constexpr IrInstGenId ir_inst_id(IrInstGenReturnAddress *) {
1823		return IrInstGenIdReturnAddress;
1824		}
	1548	ir_ref_inst_gen(op);
1825	1549
1826		static constexpr IrInstGenId ir_inst_id(IrInstGenFrameAddress *) {
1827		return IrInstGenIdFrameAddress;
	1550	return &instruction->base;
1828	1551	}
1829	1552
1830		static constexpr IrInstGenId ir_inst_id(IrInstGenFrameHandle *) {
1831		return IrInstGenIdFrameHandle;
1832		}
	1553	static IrInstGen ir_build_pop_count_gen(IrAnalyze ira, IrInst source_instr, ZigType result_type,
	1554	IrInstGen *op)
	1555	{
	1556	IrInstGenPopCount *instruction = ir_build_inst_gen<IrInstGenPopCount>(&ira->new_irb,
	1557	source_instr->scope, source_instr->source_node);
	1558	instruction->base.value->type = result_type;
	1559	instruction->op = op;
1833	1560
1834		static constexpr IrInstGenId ir_inst_id(IrInstGenFrameSize *) {
1835		return IrInstGenIdFrameSize;
1836		}
	1561	ir_ref_inst_gen(op);
1837	1562
1838		static constexpr IrInstGenId ir_inst_id(IrInstGenOverflowOp *) {
1839		return IrInstGenIdOverflowOp;
	1563	return &instruction->base;
1840	1564	}
1841	1565
1842		static constexpr IrInstGenId ir_inst_id(IrInstGenTestErr *) {
1843		return IrInstGenIdTestErr;
1844		}
	1566	static IrInstGen ir_build_bswap_gen(IrAnalyze ira, IrInst source_instr, ZigType op_type,
	1567	IrInstGen *op)
	1568	{
	1569	IrInstGenBswap *instruction = ir_build_inst_gen<IrInstGenBswap>(&ira->new_irb,
	1570	source_instr->scope, source_instr->source_node);
	1571	instruction->base.value->type = op_type;
	1572	instruction->op = op;
1845	1573
1846		static constexpr IrInstGenId ir_inst_id(IrInstGenMulAdd *) {
1847		return IrInstGenIdMulAdd;
1848		}
	1574	ir_ref_inst_gen(op);
1849	1575
1850		static constexpr IrInstGenId ir_inst_id(IrInstGenFloatOp *) {
1851		return IrInstGenIdFloatOp;
	1576	return &instruction->base;
1852	1577	}
1853	1578
1854		static constexpr IrInstGenId ir_inst_id(IrInstGenUnwrapErrCode *) {
1855		return IrInstGenIdUnwrapErrCode;
1856		}
	1579	static IrInstGen ir_build_bit_reverse_gen(IrAnalyze ira, IrInst source_instr, ZigType int_type,
	1580	IrInstGen *op)
	1581	{
	1582	IrInstGenBitReverse *instruction = ir_build_inst_gen<IrInstGenBitReverse>(&ira->new_irb,
	1583	source_instr->scope, source_instr->source_node);
	1584	instruction->base.value->type = int_type;
	1585	instruction->op = op;
1857	1586
1858		static constexpr IrInstGenId ir_inst_id(IrInstGenUnwrapErrPayload *) {
1859		return IrInstGenIdUnwrapErrPayload;
1860		}
	1587	ir_ref_inst_gen(op);
1861	1588
1862		static constexpr IrInstGenId ir_inst_id(IrInstGenErrWrapCode *) {
1863		return IrInstGenIdErrWrapCode;
	1589	return &instruction->base;
1864	1590	}
1865	1591
1866		static constexpr IrInstGenId ir_inst_id(IrInstGenErrWrapPayload *) {
1867		return IrInstGenIdErrWrapPayload;
1868		}
	1592	static IrInstGenSwitchBr ir_build_switch_br_gen(IrAnalyze ira, IrInst *source_instr,
	1593	IrInstGen target_value, IrBasicBlockGen else_block, size_t case_count, IrInstGenSwitchBrCase *cases)
	1594	{
	1595	IrInstGenSwitchBr *instruction = ir_build_inst_noreturn<IrInstGenSwitchBr>(&ira->new_irb,
	1596	source_instr->scope, source_instr->source_node);
	1597	instruction->target_value = target_value;
	1598	instruction->else_block = else_block;
	1599	instruction->case_count = case_count;
	1600	instruction->cases = cases;
1869	1601
1870		static constexpr IrInstGenId ir_inst_id(IrInstGenPtrCast *) {
1871		return IrInstGenIdPtrCast;
1872		}
	1602	ir_ref_inst_gen(target_value);
1873	1603
1874		static constexpr IrInstGenId ir_inst_id(IrInstGenBitCast *) {
1875		return IrInstGenIdBitCast;
1876		}
	1604	for (size_t i = 0; i < case_count; i += 1) {
	1605	ir_ref_inst_gen(cases[i].value);
	1606	}
1877	1607
1878		static constexpr IrInstGenId ir_inst_id(IrInstGenWidenOrShorten *) {
1879		return IrInstGenIdWidenOrShorten;
	1608	return instruction;
1880	1609	}
1881	1610
1882		static constexpr IrInstGenId ir_inst_id(IrInstGenIntToPtr *) {
1883		return IrInstGenIdIntToPtr;
1884		}
	1611	static IrInstGen ir_build_union_tag(IrAnalyze ira, IrInst source_instr, IrInstGen value,
	1612	ZigType *tag_type)
	1613	{
	1614	IrInstGenUnionTag *instruction = ir_build_inst_gen<IrInstGenUnionTag>(&ira->new_irb,
	1615	source_instr->scope, source_instr->source_node);
	1616	instruction->value = value;
	1617	instruction->base.value->type = tag_type;
1885	1618
1886		static constexpr IrInstGenId ir_inst_id(IrInstGenPtrToInt *) {
1887		return IrInstGenIdPtrToInt;
1888		}
	1619	ir_ref_inst_gen(value);
1889	1620
1890		static constexpr IrInstGenId ir_inst_id(IrInstGenIntToEnum *) {
1891		return IrInstGenIdIntToEnum;
	1621	return &instruction->base;
1892	1622	}
1893	1623
1894		static constexpr IrInstGenId ir_inst_id(IrInstGenIntToErr *) {
1895		return IrInstGenIdIntToErr;
1896		}
	1624	static IrInstGen ir_build_ref_gen(IrAnalyze ira, IrInst source_instruction, ZigType result_type,
	1625	IrInstGen operand, IrInstGen result_loc)
	1626	{
	1627	IrInstGenRef *instruction = ir_build_inst_gen<IrInstGenRef>(&ira->new_irb,
	1628	source_instruction->scope, source_instruction->source_node);
	1629	instruction->base.value->type = result_type;
	1630	instruction->operand = operand;
	1631	instruction->result_loc = result_loc;
1897	1632
1898		static constexpr IrInstGenId ir_inst_id(IrInstGenErrToInt *) {
1899		return IrInstGenIdErrToInt;
1900		}
	1633	ir_ref_inst_gen(operand);
	1634	if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
1901	1635
1902		static constexpr IrInstGenId ir_inst_id(IrInstGenPanic *) {
1903		return IrInstGenIdPanic;
	1636	return &instruction->base;
1904	1637	}
1905	1638
1906		static constexpr IrInstGenId ir_inst_id(IrInstGenTagName *) {
1907		return IrInstGenIdTagName;
1908		}
	1639	static IrInstGen ir_build_err_name_gen(IrAnalyze ira, IrInst source_instr, IrInstGen value,
	1640	ZigType *str_type)
	1641	{
	1642	IrInstGenErrName *instruction = ir_build_inst_gen<IrInstGenErrName>(&ira->new_irb,
	1643	source_instr->scope, source_instr->source_node);
	1644	instruction->base.value->type = str_type;
	1645	instruction->value = value;
1909	1646
1910		static constexpr IrInstGenId ir_inst_id(IrInstGenFieldParentPtr *) {
1911		return IrInstGenIdFieldParentPtr;
1912		}
	1647	ir_ref_inst_gen(value);
1913	1648
1914		static constexpr IrInstGenId ir_inst_id(IrInstGenAlignCast *) {
1915		return IrInstGenIdAlignCast;
	1649	return &instruction->base;
1916	1650	}
1917	1651
1918		static constexpr IrInstGenId ir_inst_id(IrInstGenErrorReturnTrace *) {
1919		return IrInstGenIdErrorReturnTrace;
1920		}
	1652	static IrInstGen ir_build_cmpxchg_gen(IrAnalyze ira, IrInst source_instruction, ZigType result_type,
	1653	IrInstGen ptr, IrInstGen cmp_value, IrInstGen *new_value,
	1654	AtomicOrder success_order, AtomicOrder failure_order, bool is_weak, IrInstGen *result_loc)
	1655	{
	1656	IrInstGenCmpxchg *instruction = ir_build_inst_gen<IrInstGenCmpxchg>(&ira->new_irb,
	1657	source_instruction->scope, source_instruction->source_node);
	1658	instruction->base.value->type = result_type;
	1659	instruction->ptr = ptr;
	1660	instruction->cmp_value = cmp_value;
	1661	instruction->new_value = new_value;
	1662	instruction->success_order = success_order;
	1663	instruction->failure_order = failure_order;
	1664	instruction->is_weak = is_weak;
	1665	instruction->result_loc = result_loc;
1921	1666
1922		static constexpr IrInstGenId ir_inst_id(IrInstGenAtomicRmw *) {
1923		return IrInstGenIdAtomicRmw;
1924		}
	1667	ir_ref_inst_gen(ptr);
	1668	ir_ref_inst_gen(cmp_value);
	1669	ir_ref_inst_gen(new_value);
	1670	if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
1925	1671
1926		static constexpr IrInstGenId ir_inst_id(IrInstGenAtomicLoad *) {
1927		return IrInstGenIdAtomicLoad;
	1672	return &instruction->base;
1928	1673	}
1929	1674
1930		static constexpr IrInstGenId ir_inst_id(IrInstGenAtomicStore *) {
1931		return IrInstGenIdAtomicStore;
1932		}
	1675	static IrInstGen ir_build_fence_gen(IrAnalyze ira, IrInst *source_instr, AtomicOrder order) {
	1676	IrInstGenFence *instruction = ir_build_inst_void<IrInstGenFence>(&ira->new_irb,
	1677	source_instr->scope, source_instr->source_node);
	1678	instruction->order = order;
1933	1679
1934		static constexpr IrInstGenId ir_inst_id(IrInstGenSaveErrRetAddr *) {
1935		return IrInstGenIdSaveErrRetAddr;
	1680	return &instruction->base;
1936	1681	}
1937	1682
1938		static constexpr IrInstGenId ir_inst_id(IrInstGenVectorToArray *) {
1939		return IrInstGenIdVectorToArray;
1940		}
1941
1942		static constexpr IrInstGenId ir_inst_id(IrInstGenArrayToVector *) {
1943		return IrInstGenIdArrayToVector;
1944		}
	1683	static IrInstGen ir_build_reduce_gen(IrAnalyze ira, IrInst source_instruction, ReduceOp op, IrInstGen value, ZigType *result_type) {
	1684	IrInstGenReduce *instruction = ir_build_inst_gen<IrInstGenReduce>(&ira->new_irb,
	1685	source_instruction->scope, source_instruction->source_node);
	1686	instruction->base.value->type = result_type;
	1687	instruction->op = op;
	1688	instruction->value = value;
1945	1689
1946		static constexpr IrInstGenId ir_inst_id(IrInstGenAssertZero *) {
1947		return IrInstGenIdAssertZero;
1948		}
	1690	ir_ref_inst_gen(value);
1949	1691
1950		static constexpr IrInstGenId ir_inst_id(IrInstGenAssertNonNull *) {
1951		return IrInstGenIdAssertNonNull;
	1692	return &instruction->base;
1952	1693	}
1953	1694
1954		static constexpr IrInstGenId ir_inst_id(IrInstGenPtrOfArrayToSlice *) {
1955		return IrInstGenIdPtrOfArrayToSlice;
	1695	static void ir_set_cursor_at_end_gen(IrBuilderGen irb, IrBasicBlockGen basic_block) {
	1696	assert(basic_block);
	1697	irb->current_basic_block = basic_block;
1956	1698	}
1957	1699
1958		static constexpr IrInstGenId ir_inst_id(IrInstGenSuspendBegin *) {
1959		return IrInstGenIdSuspendBegin;
	1700	static void ir_append_basic_block_gen(IrBuilderGen irb, IrBasicBlockGen bb) {
	1701	assert(!bb->already_appended);
	1702	bb->already_appended = true;
	1703	irb->exec->basic_block_list.append(bb);
1960	1704	}
1961	1705
1962		static constexpr IrInstGenId ir_inst_id(IrInstGenSuspendFinish *) {
1963		return IrInstGenIdSuspendFinish;
	1706	static void ir_set_cursor_at_end_and_append_block_gen(IrBuilderGen irb, IrBasicBlockGen basic_block) {
	1707	ir_append_basic_block_gen(irb, basic_block);
	1708	ir_set_cursor_at_end_gen(irb, basic_block);
1964	1709	}
1965	1710
1966		static constexpr IrInstGenId ir_inst_id(IrInstGenAwait *) {
1967		return IrInstGenIdAwait;
	1711	static IrInstGen ir_build_suspend_begin_gen(IrAnalyze ira, IrInst *source_instr) {
	1712	IrInstGenSuspendBegin *inst = ir_build_inst_void<IrInstGenSuspendBegin>(&ira->new_irb,
	1713	source_instr->scope, source_instr->source_node);
	1714	return &inst->base;
1968	1715	}
1969	1716
1970		static constexpr IrInstGenId ir_inst_id(IrInstGenResume *) {
1971		return IrInstGenIdResume;
	1717	static IrInstGen ir_build_save_err_ret_addr_gen(IrAnalyze ira, IrInst *source_instr) {
	1718	IrInstGenSaveErrRetAddr *inst = ir_build_inst_void<IrInstGenSaveErrRetAddr>(&ira->new_irb,
	1719	source_instr->scope, source_instr->source_node);
	1720	return &inst->base;
1972	1721	}
1973	1722
1974		static constexpr IrInstGenId ir_inst_id(IrInstGenSpillBegin *) {
1975		return IrInstGenIdSpillBegin;
1976		}
	1723	static IrInstGen ir_build_truncate_gen(IrAnalyze ira, IrInst source_instr, ZigType dest_type,
	1724	IrInstGen *target)
	1725	{
	1726	IrInstGenTruncate *instruction = ir_build_inst_gen<IrInstGenTruncate>(&ira->new_irb,
	1727	source_instr->scope, source_instr->source_node);
	1728	instruction->base.value->type = dest_type;
	1729	instruction->target = target;
1977	1730
1978		static constexpr IrInstGenId ir_inst_id(IrInstGenSpillEnd *) {
1979		return IrInstGenIdSpillEnd;
1980		}
	1731	ir_ref_inst_gen(target);
1981	1732
1982		static constexpr IrInstGenId ir_inst_id(IrInstGenVectorExtractElem *) {
1983		return IrInstGenIdVectorExtractElem;
	1733	return &instruction->base;
1984	1734	}
1985	1735
1986		static constexpr IrInstGenId ir_inst_id(IrInstGenAlloca *) {
1987		return IrInstGenIdAlloca;
1988		}
	1736	static IrInstGen ir_build_shuffle_vector_gen(IrAnalyze ira, Scope scope, AstNode source_node,
	1737	ZigType result_type, IrInstGen a, IrInstGen b, IrInstGen mask)
	1738	{
	1739	IrInstGenShuffleVector *inst = ir_build_inst_gen<IrInstGenShuffleVector>(&ira->new_irb, scope, source_node);
	1740	inst->base.value->type = result_type;
	1741	inst->a = a;
	1742	inst->b = b;
	1743	inst->mask = mask;
1989	1744
1990		static constexpr IrInstGenId ir_inst_id(IrInstGenConst *) {
1991		return IrInstGenIdConst;
1992		}
	1745	ir_ref_inst_gen(a);
	1746	ir_ref_inst_gen(b);
	1747	ir_ref_inst_gen(mask);
1993	1748
1994		static constexpr IrInstGenId ir_inst_id(IrInstGenWasmMemorySize *) {
1995		return IrInstGenIdWasmMemorySize;
	1749	return &inst->base;
1996	1750	}
1997	1751
1998		static constexpr IrInstGenId ir_inst_id(IrInstGenWasmMemoryGrow *) {
1999		return IrInstGenIdWasmMemoryGrow;
2000		}
	1752	static IrInstGen ir_build_splat_gen(IrAnalyze ira, IrInst source_instruction, ZigType result_type,
	1753	IrInstGen *scalar)
	1754	{
	1755	IrInstGenSplat *instruction = ir_build_inst_gen<IrInstGenSplat>(
	1756	&ira->new_irb, source_instruction->scope, source_instruction->source_node);
	1757	instruction->base.value->type = result_type;
	1758	instruction->scalar = scalar;
2001	1759
2002		static constexpr IrInstGenId ir_inst_id(IrInstGenExtern *) {
2003		return IrInstGenIdExtern;
2004		}
	1760	ir_ref_inst_gen(scalar);
2005	1761
2006		template<typename T>
2007		static T ir_create_instruction(IrBuilderSrc irb, Scope scope, AstNode source_node) {
2008		T *special_instruction = heap::c_allocator.create<T>();
2009		special_instruction->base.id = ir_inst_id(special_instruction);
2010		special_instruction->base.base.scope = scope;
2011		special_instruction->base.base.source_node = source_node;
2012		special_instruction->base.base.debug_id = exec_next_debug_id(irb->exec);
2013		special_instruction->base.owner_bb = irb->current_basic_block;
2014		return special_instruction;
	1762	return &instruction->base;
2015	1763	}
2016	1764
2017		template<typename T>
2018		static T ir_create_inst_gen(IrBuilderGen irb, Scope scope, AstNode source_node) {
2019		T *special_instruction = heap::c_allocator.create<T>();
2020		special_instruction->base.id = ir_inst_id(special_instruction);
2021		special_instruction->base.base.scope = scope;
2022		special_instruction->base.base.source_node = source_node;
2023		special_instruction->base.base.debug_id = exec_next_debug_id_gen(irb->exec);
2024		special_instruction->base.owner_bb = irb->current_basic_block;
2025		special_instruction->base.value = irb->codegen->pass1_arena->create<ZigValue>();
2026		return special_instruction;
2027		}
	1765	static IrInstGen ir_build_bool_not_gen(IrAnalyze ira, IrInst source_instr, IrInstGen value) {
	1766	IrInstGenBoolNot *instruction = ir_build_inst_gen<IrInstGenBoolNot>(&ira->new_irb,
	1767	source_instr->scope, source_instr->source_node);
	1768	instruction->base.value->type = ira->codegen->builtin_types.entry_bool;
	1769	instruction->value = value;
2028	1770
2029		template<typename T>
2030		static T ir_create_inst_noval(IrBuilderGen irb, Scope scope, AstNode source_node) {
2031		T *special_instruction = heap::c_allocator.create<T>();
2032		special_instruction->base.id = ir_inst_id(special_instruction);
2033		special_instruction->base.base.scope = scope;
2034		special_instruction->base.base.source_node = source_node;
2035		special_instruction->base.base.debug_id = exec_next_debug_id_gen(irb->exec);
2036		special_instruction->base.owner_bb = irb->current_basic_block;
2037		return special_instruction;
2038		}
	1771	ir_ref_inst_gen(value);
2039	1772
2040		template<typename T>
2041		static T ir_build_instruction(IrBuilderSrc irb, Scope scope, AstNode source_node) {
2042		T *special_instruction = ir_create_instruction<T>(irb, scope, source_node);
2043		ir_instruction_append(irb->current_basic_block, &special_instruction->base);
2044		return special_instruction;
	1773	return &instruction->base;
2045	1774	}
2046	1775
2047		template<typename T>
2048		static T ir_build_inst_gen(IrBuilderGen irb, Scope scope, AstNode source_node) {
2049		T *special_instruction = ir_create_inst_gen<T>(irb, scope, source_node);
2050		ir_inst_gen_append(irb->current_basic_block, &special_instruction->base);
2051		return special_instruction;
2052		}
	1776	static IrInstGen ir_build_memset_gen(IrAnalyze ira, IrInst *source_instr,
	1777	IrInstGen dest_ptr, IrInstGen byte, IrInstGen *count)
	1778	{
	1779	IrInstGenMemset *instruction = ir_build_inst_void<IrInstGenMemset>(&ira->new_irb,
	1780	source_instr->scope, source_instr->source_node);
	1781	instruction->dest_ptr = dest_ptr;
	1782	instruction->byte = byte;
	1783	instruction->count = count;
2053	1784
2054		template<typename T>
2055		static T ir_build_inst_noreturn(IrBuilderGen irb, Scope scope, AstNode source_node) {
2056		T *special_instruction = ir_create_inst_noval<T>(irb, scope, source_node);
2057		special_instruction->base.value = irb->codegen->intern.for_unreachable();
2058		ir_inst_gen_append(irb->current_basic_block, &special_instruction->base);
2059		return special_instruction;
2060		}
	1785	ir_ref_inst_gen(dest_ptr);
	1786	ir_ref_inst_gen(byte);
	1787	ir_ref_inst_gen(count);
2061	1788
2062		template<typename T>
2063		static T ir_build_inst_void(IrBuilderGen irb, Scope scope, AstNode source_node) {
2064		T *special_instruction = ir_create_inst_noval<T>(irb, scope, source_node);
2065		special_instruction->base.value = irb->codegen->intern.for_void();
2066		ir_inst_gen_append(irb->current_basic_block, &special_instruction->base);
2067		return special_instruction;
	1789	return &instruction->base;
2068	1790	}
2069	1791
2070		IrInstGen ir_create_alloca(CodeGen g, Scope scope, AstNode source_node, ZigFn *fn,
2071		ZigType var_type, const char name_hint)
	1792	static IrInstGen ir_build_memcpy_gen(IrAnalyze ira, IrInst *source_instr,
	1793	IrInstGen dest_ptr, IrInstGen src_ptr, IrInstGen *count)
2072	1794	{
2073		IrInstGenAlloca *alloca_gen = heap::c_allocator.create<IrInstGenAlloca>();
2074		alloca_gen->base.id = IrInstGenIdAlloca;
2075		alloca_gen->base.base.source_node = source_node;
2076		alloca_gen->base.base.scope = scope;
2077		alloca_gen->base.value = g->pass1_arena->create<ZigValue>();
2078		alloca_gen->base.value->type = get_pointer_to_type(g, var_type, false);
2079		alloca_gen->base.base.ref_count = 1;
2080		alloca_gen->name_hint = name_hint;
2081		fn->alloca_gen_list.append(alloca_gen);
2082		return &alloca_gen->base;
	1795	IrInstGenMemcpy *instruction = ir_build_inst_void<IrInstGenMemcpy>(&ira->new_irb,
	1796	source_instr->scope, source_instr->source_node);
	1797	instruction->dest_ptr = dest_ptr;
	1798	instruction->src_ptr = src_ptr;
	1799	instruction->count = count;
	1800
	1801	ir_ref_inst_gen(dest_ptr);
	1802	ir_ref_inst_gen(src_ptr);
	1803	ir_ref_inst_gen(count);
	1804
	1805	return &instruction->base;
2083	1806	}
2084	1807
2085		static IrInstGen ir_build_cast(IrAnalyze ira, IrInst source_instr,ZigType dest_type,
2086		IrInstGen *value, CastOp cast_op)
	1808	static IrInstGen ir_build_slice_gen(IrAnalyze ira, IrInst source_instruction, ZigType slice_type,
	1809	IrInstGen ptr, IrInstGen start, IrInstGen end, bool safety_check_on, IrInstGen result_loc,
	1810	ZigValue *sentinel)
2087	1811	{
2088		IrInstGenCast *inst = ir_build_inst_gen<IrInstGenCast>(&ira->new_irb, source_instr->scope, source_instr->source_node);
2089		inst->base.value->type = dest_type;
2090		inst->value = value;
2091		inst->cast_op = cast_op;
	1812	IrInstGenSlice *instruction = ir_build_inst_gen<IrInstGenSlice>(
	1813	&ira->new_irb, source_instruction->scope, source_instruction->source_node);
	1814	instruction->base.value->type = slice_type;
	1815	instruction->ptr = ptr;
	1816	instruction->start = start;
	1817	instruction->end = end;
	1818	instruction->safety_check_on = safety_check_on;
	1819	instruction->result_loc = result_loc;
	1820	instruction->sentinel = sentinel;
2092	1821
2093		ir_ref_inst_gen(value);
	1822	ir_ref_inst_gen(ptr);
	1823	ir_ref_inst_gen(start);
	1824	if (end != nullptr) ir_ref_inst_gen(end);
	1825	if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
2094	1826
2095		return &inst->base;
	1827	return &instruction->base;
2096	1828	}
2097	1829
2098		static IrInstSrc ir_build_cond_br(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *condition,
2099		IrBasicBlockSrc then_block, IrBasicBlockSrc else_block, IrInstSrc *is_comptime)
2100		{
2101		IrInstSrcCondBr *inst = ir_build_instruction<IrInstSrcCondBr>(irb, scope, source_node);
2102		inst->base.is_noreturn = true;
2103		inst->condition = condition;
2104		inst->then_block = then_block;
2105		inst->else_block = else_block;
2106		inst->is_comptime = is_comptime;
2107
2108		ir_ref_instruction(condition, irb->current_basic_block);
2109		ir_ref_bb(then_block);
2110		ir_ref_bb(else_block);
2111		if (is_comptime != nullptr) ir_ref_instruction(is_comptime, irb->current_basic_block);
	1830	static IrInstGen ir_build_breakpoint_gen(IrAnalyze ira, IrInst *source_instr) {
	1831	IrInstGenBreakpoint *instruction = ir_build_inst_void<IrInstGenBreakpoint>(&ira->new_irb,
	1832	source_instr->scope, source_instr->source_node);
	1833	return &instruction->base;
	1834	}
2112	1835
	1836	static IrInstGen ir_build_return_address_gen(IrAnalyze ira, IrInst *source_instr) {
	1837	IrInstGenReturnAddress *inst = ir_build_inst_gen<IrInstGenReturnAddress>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	1838	inst->base.value->type = ira->codegen->builtin_types.entry_usize;
2113	1839	return &inst->base;
2114	1840	}
2115	1841
2116		static IrInstGen ir_build_cond_br_gen(IrAnalyze ira, IrInst source_instr, IrInstGen condition,
2117		IrBasicBlockGen then_block, IrBasicBlockGen else_block)
2118		{
2119		IrInstGenCondBr *inst = ir_build_inst_noreturn<IrInstGenCondBr>(&ira->new_irb, source_instr->scope, source_instr->source_node);
2120		inst->condition = condition;
2121		inst->then_block = then_block;
2122		inst->else_block = else_block;
2123
2124		ir_ref_inst_gen(condition);
	1842	static IrInstGen ir_build_frame_address_gen(IrAnalyze ira, IrInst *source_instr) {
	1843	IrInstGenFrameAddress *inst = ir_build_inst_gen<IrInstGenFrameAddress>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	1844	inst->base.value->type = ira->codegen->builtin_types.entry_usize;
	1845	return &inst->base;
	1846	}
2125	1847
	1848	static IrInstGen ir_build_handle_gen(IrAnalyze ira, IrInst source_instr, ZigType ty) {
	1849	IrInstGenFrameHandle *inst = ir_build_inst_gen<IrInstGenFrameHandle>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	1850	inst->base.value->type = ty;
2126	1851	return &inst->base;
2127	1852	}
2128	1853
2129		static IrInstSrc ir_build_return_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *operand) {
2130		IrInstSrcReturn *inst = ir_build_instruction<IrInstSrcReturn>(irb, scope, source_node);
2131		inst->base.is_noreturn = true;
2132		inst->operand = operand;
	1854	static IrInstGen ir_build_frame_size_gen(IrAnalyze ira, IrInst source_instr, IrInstGen fn)
	1855	{
	1856	IrInstGenFrameSize *inst = ir_build_inst_gen<IrInstGenFrameSize>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	1857	inst->base.value->type = ira->codegen->builtin_types.entry_usize;
	1858	inst->fn = fn;
2133	1859
2134		if (operand != nullptr) ir_ref_instruction(operand, irb->current_basic_block);
	1860	ir_ref_inst_gen(fn);
2135	1861
2136	1862	return &inst->base;
2137	1863	}
2138	1864
2139		static IrInstGen ir_build_return_gen(IrAnalyze ira, IrInst source_inst, IrInstGen operand) {
2140		IrInstGenReturn *inst = ir_build_inst_noreturn<IrInstGenReturn>(&ira->new_irb,
2141		source_inst->scope, source_inst->source_node);
2142		inst->operand = operand;
2143
2144		if (operand != nullptr) ir_ref_inst_gen(operand);
2145
2146		return &inst->base;
2147		}
2148
2149		static IrInstSrc ir_build_const_void(IrBuilderSrc irb, Scope scope, AstNode source_node) {
2150		IrInstSrcConst *const_instruction = ir_create_instruction<IrInstSrcConst>(irb, scope, source_node);
2151		ir_instruction_append(irb->current_basic_block, &const_instruction->base);
2152		const_instruction->value = irb->codegen->intern.for_void();
2153		return &const_instruction->base;
2154		}
2155
2156		static IrInstSrc ir_build_const_undefined(IrBuilderSrc irb, Scope scope, AstNode source_node) {
2157		IrInstSrcConst *const_instruction = ir_create_instruction<IrInstSrcConst>(irb, scope, source_node);
2158		ir_instruction_append(irb->current_basic_block, &const_instruction->base);
2159		const_instruction->value = irb->codegen->intern.for_undefined();
2160		const_instruction->value->special = ConstValSpecialUndef;
2161		return &const_instruction->base;
2162		}
2163
2164		static IrInstSrc ir_build_const_uint(IrBuilderSrc irb, Scope scope, AstNode source_node, uint64_t value) {
2165		IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
2166		const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
2167		const_instruction->value->type = irb->codegen->builtin_types.entry_num_lit_int;
2168		const_instruction->value->special = ConstValSpecialStatic;
2169		bigint_init_unsigned(&const_instruction->value->data.x_bigint, value);
2170		return &const_instruction->base;
2171		}
	1865	static IrInstGen ir_build_overflow_op_gen(IrAnalyze ira, IrInst *source_instr,
	1866	IrOverflowOp op, IrInstGen op1, IrInstGen op2, IrInstGen *result_ptr,
	1867	ZigType *result_ptr_type)
	1868	{
	1869	IrInstGenOverflowOp *instruction = ir_build_inst_gen<IrInstGenOverflowOp>(&ira->new_irb,
	1870	source_instr->scope, source_instr->source_node);
	1871	instruction->base.value->type = ira->codegen->builtin_types.entry_bool;
	1872	instruction->op = op;
	1873	instruction->op1 = op1;
	1874	instruction->op2 = op2;
	1875	instruction->result_ptr = result_ptr;
	1876	instruction->result_ptr_type = result_ptr_type;
2172	1877
2173		static IrInstSrc ir_build_const_bigint(IrBuilderSrc irb, Scope scope, AstNode source_node, BigInt *bigint) {
2174		IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
2175		const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
2176		const_instruction->value->type = irb->codegen->builtin_types.entry_num_lit_int;
2177		const_instruction->value->special = ConstValSpecialStatic;
2178		bigint_init_bigint(&const_instruction->value->data.x_bigint, bigint);
2179		return &const_instruction->base;
2180		}
	1878	ir_ref_inst_gen(op1);
	1879	ir_ref_inst_gen(op2);
	1880	ir_ref_inst_gen(result_ptr);
2181	1881
2182		static IrInstSrc ir_build_const_bigfloat(IrBuilderSrc irb, Scope scope, AstNode source_node, BigFloat *bigfloat) {
2183		IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
2184		const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
2185		const_instruction->value->type = irb->codegen->builtin_types.entry_num_lit_float;
2186		const_instruction->value->special = ConstValSpecialStatic;
2187		bigfloat_init_bigfloat(&const_instruction->value->data.x_bigfloat, bigfloat);
2188		return &const_instruction->base;
	1882	return &instruction->base;
2189	1883	}
2190	1884
2191		static IrInstSrc ir_build_const_null(IrBuilderSrc irb, Scope scope, AstNode source_node) {
2192		IrInstSrcConst *const_instruction = ir_create_instruction<IrInstSrcConst>(irb, scope, source_node);
2193		ir_instruction_append(irb->current_basic_block, &const_instruction->base);
2194		const_instruction->value = irb->codegen->intern.for_null();
2195		return &const_instruction->base;
2196		}
	1885	static IrInstGen ir_build_float_op_gen(IrAnalyze ira, IrInst source_instr, IrInstGen operand,
	1886	BuiltinFnId fn_id, ZigType *operand_type)
	1887	{
	1888	IrInstGenFloatOp *instruction = ir_build_inst_gen<IrInstGenFloatOp>(&ira->new_irb,
	1889	source_instr->scope, source_instr->source_node);
	1890	instruction->base.value->type = operand_type;
	1891	instruction->operand = operand;
	1892	instruction->fn_id = fn_id;
2197	1893
2198		static IrInstSrc ir_build_const_usize(IrBuilderSrc irb, Scope scope, AstNode source_node, uint64_t value) {
2199		IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
2200		const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
2201		const_instruction->value->type = irb->codegen->builtin_types.entry_usize;
2202		const_instruction->value->special = ConstValSpecialStatic;
2203		bigint_init_unsigned(&const_instruction->value->data.x_bigint, value);
2204		return &const_instruction->base;
2205		}
	1894	ir_ref_inst_gen(operand);
2206	1895
2207		static IrInstSrc ir_create_const_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
2208		ZigType *type_entry)
2209		{
2210		IrInstSrcConst *const_instruction = ir_create_instruction<IrInstSrcConst>(irb, scope, source_node);
2211		const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
2212		const_instruction->value->type = irb->codegen->builtin_types.entry_type;
2213		const_instruction->value->special = ConstValSpecialStatic;
2214		const_instruction->value->data.x_type = type_entry;
2215		return &const_instruction->base;
	1896	return &instruction->base;
2216	1897	}
2217	1898
2218		static IrInstSrc ir_build_const_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
2219		ZigType *type_entry)
	1899	static IrInstGen ir_build_mul_add_gen(IrAnalyze ira, IrInst source_instr, IrInstGen op1, IrInstGen *op2,
	1900	IrInstGen op3, ZigType expr_type)
2220	1901	{
2221		IrInstSrc *instruction = ir_create_const_type(irb, scope, source_node, type_entry);
2222		ir_instruction_append(irb->current_basic_block, instruction);
2223		return instruction;
2224		}
2225
2226		static IrInstSrc ir_build_const_import(IrBuilderSrc irb, Scope scope, AstNode source_node, ZigType *import) {
2227		IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
2228		const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
2229		const_instruction->value->type = irb->codegen->builtin_types.entry_type;
2230		const_instruction->value->special = ConstValSpecialStatic;
2231		const_instruction->value->data.x_type = import;
2232		return &const_instruction->base;
2233		}
	1902	IrInstGenMulAdd *instruction = ir_build_inst_gen<IrInstGenMulAdd>(&ira->new_irb,
	1903	source_instr->scope, source_instr->source_node);
	1904	instruction->base.value->type = expr_type;
	1905	instruction->op1 = op1;
	1906	instruction->op2 = op2;
	1907	instruction->op3 = op3;
2234	1908
2235		static IrInstSrc ir_build_const_bool(IrBuilderSrc irb, Scope scope, AstNode source_node, bool value) {
2236		IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
2237		const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
2238		const_instruction->value->type = irb->codegen->builtin_types.entry_bool;
2239		const_instruction->value->special = ConstValSpecialStatic;
2240		const_instruction->value->data.x_bool = value;
2241		return &const_instruction->base;
2242		}
	1909	ir_ref_inst_gen(op1);
	1910	ir_ref_inst_gen(op2);
	1911	ir_ref_inst_gen(op3);
2243	1912
2244		static IrInstSrc ir_build_const_enum_literal(IrBuilderSrc irb, Scope scope, AstNode source_node, Buf *name) {
2245		IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, source_node);
2246		const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
2247		const_instruction->value->type = irb->codegen->builtin_types.entry_enum_literal;
2248		const_instruction->value->special = ConstValSpecialStatic;
2249		const_instruction->value->data.x_enum_literal = name;
2250		return &const_instruction->base;
	1913	return &instruction->base;
2251	1914	}
2252	1915
2253		static IrInstSrc ir_create_const_str_lit(IrBuilderSrc irb, Scope scope, AstNode source_node, Buf *str) {
2254		IrInstSrcConst *const_instruction = ir_create_instruction<IrInstSrcConst>(irb, scope, source_node);
2255		const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
2256		init_const_str_lit(irb->codegen, const_instruction->value, str);
	1916	static IrInstGen ir_build_test_err_gen(IrAnalyze ira, IrInst source_instruction, IrInstGen err_union) {
	1917	IrInstGenTestErr *instruction = ir_build_inst_gen<IrInstGenTestErr>(
	1918	&ira->new_irb, source_instruction->scope, source_instruction->source_node);
	1919	instruction->base.value->type = ira->codegen->builtin_types.entry_bool;
	1920	instruction->err_union = err_union;
2257	1921
2258		return &const_instruction->base;
2259		}
	1922	ir_ref_inst_gen(err_union);
2260	1923
2261		static IrInstSrc ir_build_const_str_lit(IrBuilderSrc irb, Scope scope, AstNode source_node, Buf *str) {
2262		IrInstSrc *instruction = ir_create_const_str_lit(irb, scope, source_node, str);
2263		ir_instruction_append(irb->current_basic_block, instruction);
2264		return instruction;
	1924	return &instruction->base;
2265	1925	}
2266	1926
2267		static IrInstSrc ir_build_bin_op(IrBuilderSrc irb, Scope scope, AstNode source_node, IrBinOp op_id,
2268		IrInstSrc op1, IrInstSrc op2, bool safety_check_on)
	1927	static IrInstGen ir_build_unwrap_err_code_gen(IrAnalyze ira, Scope scope, AstNode source_node,
	1928	IrInstGen err_union_ptr, ZigType result_type)
2269	1929	{
2270		IrInstSrcBinOp *inst = ir_build_instruction<IrInstSrcBinOp>(irb, scope, source_node);
2271		inst->op_id = op_id;
2272		inst->op1 = op1;
2273		inst->op2 = op2;
2274		inst->safety_check_on = safety_check_on;
	1930	IrInstGenUnwrapErrCode *inst = ir_build_inst_gen<IrInstGenUnwrapErrCode>(&ira->new_irb, scope, source_node);
	1931	inst->base.value->type = result_type;
	1932	inst->err_union_ptr = err_union_ptr;
2275	1933
2276		ir_ref_instruction(op1, irb->current_basic_block);
2277		ir_ref_instruction(op2, irb->current_basic_block);
	1934	ir_ref_inst_gen(err_union_ptr);
2278	1935
2279	1936	return &inst->base;
2280	1937	}
2281	1938
2282		static IrInstGen ir_build_bin_op_gen(IrAnalyze ira, IrInst source_instr, ZigType res_type,
2283		IrBinOp op_id, IrInstGen op1, IrInstGen op2, bool safety_check_on)
	1939	static IrInstGen ir_build_unwrap_err_payload_gen(IrAnalyze ira, Scope scope, AstNode source_node,
	1940	IrInstGen value, bool safety_check_on, bool initializing, ZigType result_type)
2284	1941	{
2285		IrInstGenBinOp *inst = ir_build_inst_gen<IrInstGenBinOp>(&ira->new_irb,
2286		source_instr->scope, source_instr->source_node);
2287		inst->base.value->type = res_type;
2288		inst->op_id = op_id;
2289		inst->op1 = op1;
2290		inst->op2 = op2;
	1942	IrInstGenUnwrapErrPayload *inst = ir_build_inst_gen<IrInstGenUnwrapErrPayload>(&ira->new_irb, scope, source_node);
	1943	inst->base.value->type = result_type;
	1944	inst->value = value;
2291	1945	inst->safety_check_on = safety_check_on;
	1946	inst->initializing = initializing;
2292	1947
2293		ir_ref_inst_gen(op1);
2294		ir_ref_inst_gen(op2);
	1948	ir_ref_inst_gen(value);
2295	1949
2296	1950	return &inst->base;
2297	1951	}
2298	1952
2299
2300		static IrInstSrc ir_build_merge_err_sets(IrBuilderSrc irb, Scope scope, AstNode source_node,
2301		IrInstSrc op1, IrInstSrc op2, Buf *type_name)
	1953	static IrInstGen ir_build_ptr_cast_gen(IrAnalyze ira, IrInst *source_instruction,
	1954	ZigType ptr_type, IrInstGen ptr, bool safety_check_on)
2302	1955	{
2303		IrInstSrcMergeErrSets *inst = ir_build_instruction<IrInstSrcMergeErrSets>(irb, scope, source_node);
2304		inst->op1 = op1;
2305		inst->op2 = op2;
2306		inst->type_name = type_name;
	1956	IrInstGenPtrCast *instruction = ir_build_inst_gen<IrInstGenPtrCast>(
	1957	&ira->new_irb, source_instruction->scope, source_instruction->source_node);
	1958	instruction->base.value->type = ptr_type;
	1959	instruction->ptr = ptr;
	1960	instruction->safety_check_on = safety_check_on;
2307	1961
2308		ir_ref_instruction(op1, irb->current_basic_block);
2309		ir_ref_instruction(op2, irb->current_basic_block);
	1962	ir_ref_inst_gen(ptr);
2310	1963
2311		return &inst->base;
	1964	return &instruction->base;
2312	1965	}
2313	1966
2314		static IrInstSrc ir_build_var_ptr_x(IrBuilderSrc irb, Scope scope, AstNode source_node, ZigVar *var,
2315		ScopeFnDef *crossed_fndef_scope)
	1967	static IrInstGen ir_build_bit_cast_gen(IrAnalyze ira, IrInst *source_instruction,
	1968	IrInstGen operand, ZigType ty)
2316	1969	{
2317		IrInstSrcVarPtr *instruction = ir_build_instruction<IrInstSrcVarPtr>(irb, scope, source_node);
2318		instruction->var = var;
2319		instruction->crossed_fndef_scope = crossed_fndef_scope;
	1970	IrInstGenBitCast *instruction = ir_build_inst_gen<IrInstGenBitCast>(
	1971	&ira->new_irb, source_instruction->scope, source_instruction->source_node);
	1972	instruction->base.value->type = ty;
	1973	instruction->operand = operand;
2320	1974
2321		ir_ref_var(var);
	1975	ir_ref_inst_gen(operand);
2322	1976
2323	1977	return &instruction->base;
2324	1978	}
2325	1979
2326		static IrInstSrc ir_build_var_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node, ZigVar *var) {
2327		return ir_build_var_ptr_x(irb, scope, source_node, var, nullptr);
2328		}
2329
2330		static IrInstGen ir_build_var_ptr_gen(IrAnalyze ira, IrInst source_instr, ZigVar var) {
2331		IrInstGenVarPtr *instruction = ir_build_inst_gen<IrInstGenVarPtr>(&ira->new_irb, source_instr->scope, source_instr->source_node);
2332		instruction->var = var;
	1980	static IrInstGen ir_build_widen_or_shorten(IrAnalyze ira, Scope scope, AstNode source_node, IrInstGen *target,
	1981	ZigType *result_type)
	1982	{
	1983	IrInstGenWidenOrShorten *inst = ir_build_inst_gen<IrInstGenWidenOrShorten>(&ira->new_irb, scope, source_node);
	1984	inst->base.value->type = result_type;
	1985	inst->target = target;
2333	1986
2334		ir_ref_var(var);
	1987	ir_ref_inst_gen(target);
2335	1988
2336		return &instruction->base;
	1989	return &inst->base;
2337	1990	}
2338	1991
2339		static IrInstGen ir_build_return_ptr(IrAnalyze ira, Scope scope, AstNode source_node, ZigType *ty) {
2340		IrInstGenReturnPtr *instruction = ir_build_inst_gen<IrInstGenReturnPtr>(&ira->new_irb, scope, source_node);
2341		instruction->base.value->type = ty;
	1992	static IrInstGen ir_build_int_to_ptr_gen(IrAnalyze ira, Scope scope, AstNode source_node,
	1993	IrInstGen target, ZigType ptr_type)
	1994	{
	1995	IrInstGenIntToPtr *instruction = ir_build_inst_gen<IrInstGenIntToPtr>(&ira->new_irb, scope, source_node);
	1996	instruction->base.value->type = ptr_type;
	1997	instruction->target = target;
	1998
	1999	ir_ref_inst_gen(target);
	2000
2342	2001	return &instruction->base;
2343	2002	}
2344	2003
2345		static IrInstSrc ir_build_elem_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node,
2346		IrInstSrc array_ptr, IrInstSrc elem_index, bool safety_check_on, PtrLen ptr_len,
2347		AstNode *init_array_type_source_node)
2348		{
2349		IrInstSrcElemPtr *instruction = ir_build_instruction<IrInstSrcElemPtr>(irb, scope, source_node);
2350		instruction->array_ptr = array_ptr;
2351		instruction->elem_index = elem_index;
2352		instruction->safety_check_on = safety_check_on;
2353		instruction->ptr_len = ptr_len;
2354		instruction->init_array_type_source_node = init_array_type_source_node;
	2004	static IrInstGen ir_build_ptr_to_int_gen(IrAnalyze ira, IrInst source_instr, IrInstGen target) {
	2005	IrInstGenPtrToInt *inst = ir_build_inst_gen<IrInstGenPtrToInt>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	2006	inst->base.value->type = ira->codegen->builtin_types.entry_usize;
	2007	inst->target = target;
2355	2008
2356		ir_ref_instruction(array_ptr, irb->current_basic_block);
2357		ir_ref_instruction(elem_index, irb->current_basic_block);
	2009	ir_ref_inst_gen(target);
2358	2010
2359		return &instruction->base;
	2011	return &inst->base;
2360	2012	}
2361	2013
2362		static IrInstGen ir_build_elem_ptr_gen(IrAnalyze ira, Scope scope, AstNode source_node,
2363		IrInstGen array_ptr, IrInstGen elem_index, bool safety_check_on, ZigType *return_type)
	2014	static IrInstGen ir_build_int_to_enum_gen(IrAnalyze ira, Scope scope, AstNode source_node,
	2015	ZigType dest_type, IrInstGen target)
2364	2016	{
2365		IrInstGenElemPtr *instruction = ir_build_inst_gen<IrInstGenElemPtr>(&ira->new_irb, scope, source_node);
2366		instruction->base.value->type = return_type;
2367		instruction->array_ptr = array_ptr;
2368		instruction->elem_index = elem_index;
2369		instruction->safety_check_on = safety_check_on;
	2017	IrInstGenIntToEnum *instruction = ir_build_inst_gen<IrInstGenIntToEnum>(&ira->new_irb, scope, source_node);
	2018	instruction->base.value->type = dest_type;
	2019	instruction->target = target;
2370	2020
2371		ir_ref_inst_gen(array_ptr);
2372		ir_ref_inst_gen(elem_index);
	2021	ir_ref_inst_gen(target);
2373	2022
2374	2023	return &instruction->base;
2375	2024	}
2376	2025
2377		static IrInstSrc ir_build_field_ptr_instruction(IrBuilderSrc irb, Scope scope, AstNode source_node,
2378		IrInstSrc container_ptr, IrInstSrc field_name_expr, bool initializing)
	2026	static IrInstGen ir_build_int_to_err_gen(IrAnalyze ira, Scope scope, AstNode source_node, IrInstGen *target,
	2027	ZigType *wanted_type)
2379	2028	{
2380		IrInstSrcFieldPtr *instruction = ir_build_instruction<IrInstSrcFieldPtr>(irb, scope, source_node);
2381		instruction->container_ptr = container_ptr;
2382		instruction->field_name_buffer = nullptr;
2383		instruction->field_name_expr = field_name_expr;
2384		instruction->initializing = initializing;
	2029	IrInstGenIntToErr *instruction = ir_build_inst_gen<IrInstGenIntToErr>(&ira->new_irb, scope, source_node);
	2030	instruction->base.value->type = wanted_type;
	2031	instruction->target = target;
2385	2032
2386		ir_ref_instruction(container_ptr, irb->current_basic_block);
2387		ir_ref_instruction(field_name_expr, irb->current_basic_block);
	2033	ir_ref_inst_gen(target);
2388	2034
2389	2035	return &instruction->base;
2390	2036	}
2391	2037
2392		static IrInstSrc ir_build_field_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node,
2393		IrInstSrc container_ptr, Buf field_name, bool initializing)
	2038	static IrInstGen ir_build_err_to_int_gen(IrAnalyze ira, Scope scope, AstNode source_node, IrInstGen *target,
	2039	ZigType *wanted_type)
2394	2040	{
2395		IrInstSrcFieldPtr *instruction = ir_build_instruction<IrInstSrcFieldPtr>(irb, scope, source_node);
2396		instruction->container_ptr = container_ptr;
2397		instruction->field_name_buffer = field_name;
2398		instruction->field_name_expr = nullptr;
2399		instruction->initializing = initializing;
	2041	IrInstGenErrToInt *instruction = ir_build_inst_gen<IrInstGenErrToInt>(&ira->new_irb, scope, source_node);
	2042	instruction->base.value->type = wanted_type;
	2043	instruction->target = target;
2400	2044
2401		ir_ref_instruction(container_ptr, irb->current_basic_block);
	2045	ir_ref_inst_gen(target);
2402	2046
2403	2047	return &instruction->base;
2404	2048	}
2405	2049
2406		static IrInstSrc ir_build_has_field(IrBuilderSrc irb, Scope scope, AstNode source_node,
2407		IrInstSrc container_type, IrInstSrc field_name)
2408		{
2409		IrInstSrcHasField *instruction = ir_build_instruction<IrInstSrcHasField>(irb, scope, source_node);
2410		instruction->container_type = container_type;
2411		instruction->field_name = field_name;
	2050	static IrInstGen ir_build_panic_gen(IrAnalyze ira, IrInst source_instr, IrInstGen msg) {
	2051	IrInstGenPanic *instruction = ir_build_inst_noreturn<IrInstGenPanic>(&ira->new_irb,
	2052	source_instr->scope, source_instr->source_node);
	2053	instruction->msg = msg;
2412	2054
2413		ir_ref_instruction(container_type, irb->current_basic_block);
2414		ir_ref_instruction(field_name, irb->current_basic_block);
	2055	ir_ref_inst_gen(msg);
2415	2056
2416	2057	return &instruction->base;
2417	2058	}
2418	2059
2419		static IrInstGen ir_build_struct_field_ptr(IrAnalyze ira, IrInst *source_instr,
2420		IrInstGen struct_ptr, TypeStructField field, ZigType *ptr_type)
	2060	static IrInstGen ir_build_tag_name_gen(IrAnalyze ira, IrInst source_instr, IrInstGen target,
	2061	ZigType *result_type)
2421	2062	{
2422		IrInstGenStructFieldPtr *inst = ir_build_inst_gen<IrInstGenStructFieldPtr>(&ira->new_irb, source_instr->scope, source_instr->source_node);
2423		inst->base.value->type = ptr_type;
2424		inst->struct_ptr = struct_ptr;
2425		inst->field = field;
	2063	IrInstGenTagName *instruction = ir_build_inst_gen<IrInstGenTagName>(&ira->new_irb,
	2064	source_instr->scope, source_instr->source_node);
	2065	instruction->base.value->type = result_type;
	2066	instruction->target = target;
2426	2067
2427		ir_ref_inst_gen(struct_ptr);
	2068	ir_ref_inst_gen(target);
2428	2069
2429		return &inst->base;
	2070	return &instruction->base;
2430	2071	}
2431	2072
2432		static IrInstGen ir_build_union_field_ptr(IrAnalyze ira, IrInst *source_instr,
2433		IrInstGen union_ptr, TypeUnionField field, bool safety_check_on, bool initializing, ZigType *ptr_type)
	2073	static IrInstGen ir_build_field_parent_ptr_gen(IrAnalyze ira, IrInst *source_instr,
	2074	IrInstGen field_ptr, TypeStructField field, ZigType *result_type)
2434	2075	{
2435		IrInstGenUnionFieldPtr *inst = ir_build_inst_gen<IrInstGenUnionFieldPtr>(&ira->new_irb,
	2076	IrInstGenFieldParentPtr *inst = ir_build_inst_gen<IrInstGenFieldParentPtr>(&ira->new_irb,
2436	2077	source_instr->scope, source_instr->source_node);
2437		inst->base.value->type = ptr_type;
2438		inst->initializing = initializing;
2439		inst->safety_check_on = safety_check_on;
2440		inst->union_ptr = union_ptr;
	2078	inst->base.value->type = result_type;
	2079	inst->field_ptr = field_ptr;
2441	2080	inst->field = field;
2442	2081
2443		ir_ref_inst_gen(union_ptr);
	2082	ir_ref_inst_gen(field_ptr);
2444	2083
2445	2084	return &inst->base;
2446	2085	}
2447	2086
2448		static IrInstSrc ir_build_call_extra(IrBuilderSrc irb, Scope scope, AstNode source_node,
2449		IrInstSrc options, IrInstSrc fn_ref, IrInstSrc args, ResultLoc result_loc)
	2087	static IrInstGen ir_build_align_cast_gen(IrAnalyze ira, Scope scope, AstNode source_node, IrInstGen *target,
	2088	ZigType *result_type)
2450	2089	{
2451		IrInstSrcCallExtra *call_instruction = ir_build_instruction<IrInstSrcCallExtra>(irb, scope, source_node);
2452		call_instruction->options = options;
2453		call_instruction->fn_ref = fn_ref;
2454		call_instruction->args = args;
2455		call_instruction->result_loc = result_loc;
	2090	IrInstGenAlignCast *instruction = ir_build_inst_gen<IrInstGenAlignCast>(&ira->new_irb, scope, source_node);
	2091	instruction->base.value->type = result_type;
	2092	instruction->target = target;
2456	2093
2457		ir_ref_instruction(options, irb->current_basic_block);
2458		ir_ref_instruction(fn_ref, irb->current_basic_block);
2459		ir_ref_instruction(args, irb->current_basic_block);
	2094	ir_ref_inst_gen(target);
2460	2095
2461		return &call_instruction->base;
	2096	return &instruction->base;
2462	2097	}
2463	2098
2464		static IrInstSrc ir_build_async_call_extra(IrBuilderSrc irb, Scope scope, AstNode source_node,
2465		CallModifier modifier, IrInstSrc fn_ref, IrInstSrc ret_ptr, IrInstSrc new_stack, IrInstSrc args, ResultLoc *result_loc)
	2099	static IrInstGen ir_build_error_return_trace_gen(IrAnalyze ira, Scope scope, AstNode source_node,
	2100	IrInstErrorReturnTraceOptional optional, ZigType *result_type)
2466	2101	{
2467		IrInstSrcAsyncCallExtra *call_instruction = ir_build_instruction<IrInstSrcAsyncCallExtra>(irb, scope, source_node);
2468		call_instruction->modifier = modifier;
2469		call_instruction->fn_ref = fn_ref;
2470		call_instruction->ret_ptr = ret_ptr;
2471		call_instruction->new_stack = new_stack;
2472		call_instruction->args = args;
2473		call_instruction->result_loc = result_loc;
2474
2475		ir_ref_instruction(fn_ref, irb->current_basic_block);
2476		if (ret_ptr != nullptr) ir_ref_instruction(ret_ptr, irb->current_basic_block);
2477		ir_ref_instruction(new_stack, irb->current_basic_block);
2478		ir_ref_instruction(args, irb->current_basic_block);
	2102	IrInstGenErrorReturnTrace *inst = ir_build_inst_gen<IrInstGenErrorReturnTrace>(&ira->new_irb, scope, source_node);
	2103	inst->base.value->type = result_type;
	2104	inst->optional = optional;
2479	2105
2480		return &call_instruction->base;
	2106	return &inst->base;
2481	2107	}
2482	2108
2483		static IrInstSrc ir_build_call_args(IrBuilderSrc irb, Scope scope, AstNode source_node,
2484		IrInstSrc options, IrInstSrc fn_ref, IrInstSrc **args_ptr, size_t args_len,
2485		ResultLoc *result_loc)
	2109	static IrInstGen ir_build_atomic_rmw_gen(IrAnalyze ira, IrInst *source_instr,
	2110	IrInstGen ptr, IrInstGen operand, AtomicRmwOp op, AtomicOrder ordering, ZigType *operand_type)
2486	2111	{
2487		IrInstSrcCallArgs *call_instruction = ir_build_instruction<IrInstSrcCallArgs>(irb, scope, source_node);
2488		call_instruction->options = options;
2489		call_instruction->fn_ref = fn_ref;
2490		call_instruction->args_ptr = args_ptr;
2491		call_instruction->args_len = args_len;
2492		call_instruction->result_loc = result_loc;
2493
2494		ir_ref_instruction(options, irb->current_basic_block);
2495		ir_ref_instruction(fn_ref, irb->current_basic_block);
2496		for (size_t i = 0; i < args_len; i += 1)
2497		ir_ref_instruction(args_ptr[i], irb->current_basic_block);
2498
2499		return &call_instruction->base;
2500		}
2501
2502		static IrInstSrc ir_build_call_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
2503		ZigFn fn_entry, IrInstSrc fn_ref, size_t arg_count, IrInstSrc **args,
2504		IrInstSrc *ret_ptr, CallModifier modifier, bool is_async_call_builtin,
2505		IrInstSrc new_stack, ResultLoc result_loc)
2506		{
2507		IrInstSrcCall *call_instruction = ir_build_instruction<IrInstSrcCall>(irb, scope, source_node);
2508		call_instruction->fn_entry = fn_entry;
2509		call_instruction->fn_ref = fn_ref;
2510		call_instruction->args = args;
2511		call_instruction->arg_count = arg_count;
2512		call_instruction->modifier = modifier;
2513		call_instruction->is_async_call_builtin = is_async_call_builtin;
2514		call_instruction->new_stack = new_stack;
2515		call_instruction->result_loc = result_loc;
2516		call_instruction->ret_ptr = ret_ptr;
2517
2518		if (fn_ref != nullptr) ir_ref_instruction(fn_ref, irb->current_basic_block);
2519		for (size_t i = 0; i < arg_count; i += 1)
2520		ir_ref_instruction(args[i], irb->current_basic_block);
2521		if (ret_ptr != nullptr) ir_ref_instruction(ret_ptr, irb->current_basic_block);
2522		if (new_stack != nullptr) ir_ref_instruction(new_stack, irb->current_basic_block);
2523
2524		return &call_instruction->base;
2525		}
2526
2527		static IrInstGenCall ir_build_call_gen(IrAnalyze ira, IrInst *source_instruction,
2528		ZigFn fn_entry, IrInstGen fn_ref, size_t arg_count, IrInstGen **args,
2529		CallModifier modifier, IrInstGen *new_stack, bool is_async_call_builtin,
2530		IrInstGen result_loc, ZigType return_type)
2531		{
2532		IrInstGenCall *call_instruction = ir_build_inst_gen<IrInstGenCall>(&ira->new_irb,
2533		source_instruction->scope, source_instruction->source_node);
2534		call_instruction->base.value->type = return_type;
2535		call_instruction->fn_entry = fn_entry;
2536		call_instruction->fn_ref = fn_ref;
2537		call_instruction->args = args;
2538		call_instruction->arg_count = arg_count;
2539		call_instruction->modifier = modifier;
2540		call_instruction->is_async_call_builtin = is_async_call_builtin;
2541		call_instruction->new_stack = new_stack;
2542		call_instruction->result_loc = result_loc;
2543
2544		if (fn_ref != nullptr) ir_ref_inst_gen(fn_ref);
2545		for (size_t i = 0; i < arg_count; i += 1)
2546		ir_ref_inst_gen(args[i]);
2547		if (new_stack != nullptr) ir_ref_inst_gen(new_stack);
2548		if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
2549
2550		return call_instruction;
2551		}
2552
2553		static IrInstSrc ir_build_phi(IrBuilderSrc irb, Scope scope, AstNode source_node,
2554		size_t incoming_count, IrBasicBlockSrc incoming_blocks, IrInstSrc incoming_values,
2555		ResultLocPeerParent *peer_parent)
2556		{
2557		assert(incoming_count != 0);
2558		assert(incoming_count != SIZE_MAX);
2559
2560		IrInstSrcPhi *phi_instruction = ir_build_instruction<IrInstSrcPhi>(irb, scope, source_node);
2561		phi_instruction->incoming_count = incoming_count;
2562		phi_instruction->incoming_blocks = incoming_blocks;
2563		phi_instruction->incoming_values = incoming_values;
2564		phi_instruction->peer_parent = peer_parent;
2565
2566		for (size_t i = 0; i < incoming_count; i += 1) {
2567		ir_ref_bb(incoming_blocks[i]);
2568		ir_ref_instruction(incoming_values[i], irb->current_basic_block);
2569		}
2570
2571		return &phi_instruction->base;
2572		}
2573
2574		static IrInstGen ir_build_phi_gen(IrAnalyze ira, IrInst *source_instr, size_t incoming_count,
2575		IrBasicBlockGen incoming_blocks, IrInstGen incoming_values, ZigType *result_type)
2576		{
2577		assert(incoming_count != 0);
2578		assert(incoming_count != SIZE_MAX);
2579
2580		IrInstGenPhi *phi_instruction = ir_build_inst_gen<IrInstGenPhi>(&ira->new_irb,
2581		source_instr->scope, source_instr->source_node);
2582		phi_instruction->base.value->type = result_type;
2583		phi_instruction->incoming_count = incoming_count;
2584		phi_instruction->incoming_blocks = incoming_blocks;
2585		phi_instruction->incoming_values = incoming_values;
2586
2587		for (size_t i = 0; i < incoming_count; i += 1) {
2588		ir_ref_inst_gen(incoming_values[i]);
2589		}
2590
2591		return &phi_instruction->base;
2592		}
2593
2594		static IrInstSrc ir_build_br(IrBuilderSrc irb, Scope scope, AstNode source_node,
2595		IrBasicBlockSrc dest_block, IrInstSrc is_comptime)
2596		{
2597		IrInstSrcBr *inst = ir_build_instruction<IrInstSrcBr>(irb, scope, source_node);
2598		inst->base.is_noreturn = true;
2599		inst->dest_block = dest_block;
2600		inst->is_comptime = is_comptime;
2601
2602		ir_ref_bb(dest_block);
2603		if (is_comptime) ir_ref_instruction(is_comptime, irb->current_basic_block);
2604
2605		return &inst->base;
2606		}
2607
2608		static IrInstGen ir_build_br_gen(IrAnalyze ira, IrInst source_instr, IrBasicBlockGen dest_block) {
2609		IrInstGenBr *inst = ir_build_inst_noreturn<IrInstGenBr>(&ira->new_irb, source_instr->scope, source_instr->source_node);
2610		inst->dest_block = dest_block;
2611
2612		return &inst->base;
2613		}
2614
2615		static IrInstSrc ir_build_ptr_type_simple(IrBuilderSrc irb, Scope scope, AstNode source_node,
2616		IrInstSrc *child_type, bool is_const)
2617		{
2618		IrInstSrcPtrTypeSimple *inst = heap::c_allocator.create<IrInstSrcPtrTypeSimple>();
2619		inst->base.id = is_const ? IrInstSrcIdPtrTypeSimpleConst : IrInstSrcIdPtrTypeSimple;
2620		inst->base.base.scope = scope;
2621		inst->base.base.source_node = source_node;
2622		inst->base.base.debug_id = exec_next_debug_id(irb->exec);
2623		inst->base.owner_bb = irb->current_basic_block;
2624		ir_instruction_append(irb->current_basic_block, &inst->base);
2625
2626		inst->child_type = child_type;
2627
2628		ir_ref_instruction(child_type, irb->current_basic_block);
2629
2630		return &inst->base;
2631		}
2632
2633		static IrInstSrc ir_build_ptr_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
2634		IrInstSrc *child_type, bool is_const, bool is_volatile, PtrLen ptr_len,
2635		IrInstSrc sentinel, IrInstSrc align_value,
2636		uint32_t bit_offset_start, uint32_t host_int_bytes, bool is_allow_zero)
2637		{
2638		if (!is_volatile && ptr_len == PtrLenSingle && sentinel == nullptr && align_value == nullptr &&
2639		bit_offset_start == 0 && host_int_bytes == 0 && is_allow_zero == 0)
2640		{
2641		return ir_build_ptr_type_simple(irb, scope, source_node, child_type, is_const);
2642		}
2643
2644		IrInstSrcPtrType *inst = ir_build_instruction<IrInstSrcPtrType>(irb, scope, source_node);
2645		inst->sentinel = sentinel;
2646		inst->align_value = align_value;
2647		inst->child_type = child_type;
2648		inst->is_const = is_const;
2649		inst->is_volatile = is_volatile;
2650		inst->ptr_len = ptr_len;
2651		inst->bit_offset_start = bit_offset_start;
2652		inst->host_int_bytes = host_int_bytes;
2653		inst->is_allow_zero = is_allow_zero;
2654
2655		if (sentinel) ir_ref_instruction(sentinel, irb->current_basic_block);
2656		if (align_value) ir_ref_instruction(align_value, irb->current_basic_block);
2657		ir_ref_instruction(child_type, irb->current_basic_block);
2658
2659		return &inst->base;
2660		}
2661
2662		static IrInstSrc ir_build_un_op_lval(IrBuilderSrc irb, Scope scope, AstNode source_node, IrUnOp op_id,
2663		IrInstSrc value, LVal lval, ResultLoc result_loc)
2664		{
2665		IrInstSrcUnOp *instruction = ir_build_instruction<IrInstSrcUnOp>(irb, scope, source_node);
2666		instruction->op_id = op_id;
2667		instruction->value = value;
2668		instruction->lval = lval;
2669		instruction->result_loc = result_loc;
2670
2671		ir_ref_instruction(value, irb->current_basic_block);
2672
2673		return &instruction->base;
2674		}
2675
2676		static IrInstSrc ir_build_un_op(IrBuilderSrc irb, Scope scope, AstNode source_node, IrUnOp op_id,
2677		IrInstSrc *value)
2678		{
2679		return ir_build_un_op_lval(irb, scope, source_node, op_id, value, LValNone, nullptr);
2680		}
2681
2682		static IrInstGen ir_build_negation(IrAnalyze ira, IrInst source_instr, IrInstGen operand, ZigType *expr_type, bool wrapping) {
2683		IrInstGenNegation *instruction = ir_build_inst_gen<IrInstGenNegation>(&ira->new_irb,
2684		source_instr->scope, source_instr->source_node);
2685		instruction->base.value->type = expr_type;
2686		instruction->operand = operand;
2687		instruction->wrapping = wrapping;
2688
2689		ir_ref_inst_gen(operand);
2690
2691		return &instruction->base;
2692		}
2693
2694		static IrInstGen ir_build_binary_not(IrAnalyze ira, IrInst source_instr, IrInstGen operand,
2695		ZigType *expr_type)
2696		{
2697		IrInstGenBinaryNot *instruction = ir_build_inst_gen<IrInstGenBinaryNot>(&ira->new_irb,
2698		source_instr->scope, source_instr->source_node);
2699		instruction->base.value->type = expr_type;
2700		instruction->operand = operand;
2701
2702		ir_ref_inst_gen(operand);
2703
2704		return &instruction->base;
2705		}
2706
2707		static IrInstSrc ir_build_container_init_list(IrBuilderSrc irb, Scope scope, AstNode source_node,
2708		size_t item_count, IrInstSrc *elem_result_loc_list, IrInstSrc result_loc,
2709		AstNode *init_array_type_source_node)
2710		{
2711		IrInstSrcContainerInitList *container_init_list_instruction =
2712		ir_build_instruction<IrInstSrcContainerInitList>(irb, scope, source_node);
2713		container_init_list_instruction->item_count = item_count;
2714		container_init_list_instruction->elem_result_loc_list = elem_result_loc_list;
2715		container_init_list_instruction->result_loc = result_loc;
2716		container_init_list_instruction->init_array_type_source_node = init_array_type_source_node;
2717
2718		for (size_t i = 0; i < item_count; i += 1) {
2719		ir_ref_instruction(elem_result_loc_list[i], irb->current_basic_block);
2720		}
2721		if (result_loc != nullptr) ir_ref_instruction(result_loc, irb->current_basic_block);
2722
2723		return &container_init_list_instruction->base;
2724		}
2725
2726		static IrInstSrc ir_build_container_init_fields(IrBuilderSrc irb, Scope scope, AstNode source_node,
2727		size_t field_count, IrInstSrcContainerInitFieldsField fields, IrInstSrc result_loc)
2728		{
2729		IrInstSrcContainerInitFields *container_init_fields_instruction =
2730		ir_build_instruction<IrInstSrcContainerInitFields>(irb, scope, source_node);
2731		container_init_fields_instruction->field_count = field_count;
2732		container_init_fields_instruction->fields = fields;
2733		container_init_fields_instruction->result_loc = result_loc;
2734
2735		for (size_t i = 0; i < field_count; i += 1) {
2736		ir_ref_instruction(fields[i].result_loc, irb->current_basic_block);
2737		}
2738		if (result_loc != nullptr) ir_ref_instruction(result_loc, irb->current_basic_block);
2739
2740		return &container_init_fields_instruction->base;
2741		}
2742
2743		static IrInstSrc ir_build_unreachable(IrBuilderSrc irb, Scope scope, AstNode source_node) {
2744		IrInstSrcUnreachable *inst = ir_build_instruction<IrInstSrcUnreachable>(irb, scope, source_node);
2745		inst->base.is_noreturn = true;
2746		return &inst->base;
2747		}
2748
2749		static IrInstGen ir_build_unreachable_gen(IrAnalyze ira, IrInst *source_instr) {
2750		IrInstGenUnreachable *inst = ir_build_inst_noreturn<IrInstGenUnreachable>(&ira->new_irb, source_instr->scope, source_instr->source_node);
2751		return &inst->base;
2752		}
2753
2754		static IrInstSrcStorePtr ir_build_store_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node,
2755		IrInstSrc ptr, IrInstSrc value)
2756		{
2757		IrInstSrcStorePtr *instruction = ir_build_instruction<IrInstSrcStorePtr>(irb, scope, source_node);
2758		instruction->ptr = ptr;
2759		instruction->value = value;
2760
2761		ir_ref_instruction(ptr, irb->current_basic_block);
2762		ir_ref_instruction(value, irb->current_basic_block);
2763
2764		return instruction;
2765		}
2766
2767		static IrInstGen ir_build_store_ptr_gen(IrAnalyze ira, IrInst source_instr, IrInstGen ptr, IrInstGen *value) {
2768		IrInstGenStorePtr *instruction = ir_build_inst_void<IrInstGenStorePtr>(&ira->new_irb,
2769		source_instr->scope, source_instr->source_node);
	2112	IrInstGenAtomicRmw *instruction = ir_build_inst_gen<IrInstGenAtomicRmw>(&ira->new_irb, source_instr->scope, source_instr->source_node);
	2113	instruction->base.value->type = operand_type;
2770	2114	instruction->ptr = ptr;
2771		instruction->value = value;
	2115	instruction->op = op;
	2116	instruction->operand = operand;
	2117	instruction->ordering = ordering;
2772	2118
2773	2119	ir_ref_inst_gen(ptr);
2774		ir_ref_inst_gen(value);
	2120	ir_ref_inst_gen(operand);
2775	2121
2776	2122	return &instruction->base;
2777	2123	}
2778	2124
2779		static IrInstGen ir_build_vector_store_elem(IrAnalyze ira, IrInst *src_inst,
2780		IrInstGen vector_ptr, IrInstGen index, IrInstGen *value)
2781		{
2782		IrInstGenVectorStoreElem *inst = ir_build_inst_void<IrInstGenVectorStoreElem>(
2783		&ira->new_irb, src_inst->scope, src_inst->source_node);
2784		inst->vector_ptr = vector_ptr;
2785		inst->index = index;
2786		inst->value = value;
2787
2788		ir_ref_inst_gen(vector_ptr);
2789		ir_ref_inst_gen(index);
2790		ir_ref_inst_gen(value);
2791
2792		return &inst->base;
2793		}
2794
2795		static IrInstSrc ir_build_var_decl_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
2796		ZigVar var, IrInstSrc align_value, IrInstSrc *ptr)
2797		{
2798		IrInstSrcDeclVar *inst = ir_build_instruction<IrInstSrcDeclVar>(irb, scope, source_node);
2799		inst->var = var;
2800		inst->align_value = align_value;
2801		inst->ptr = ptr;
2802
2803		if (align_value != nullptr) ir_ref_instruction(align_value, irb->current_basic_block);
2804		ir_ref_instruction(ptr, irb->current_basic_block);
2805
2806		return &inst->base;
2807		}
2808
2809		static IrInstGen ir_build_var_decl_gen(IrAnalyze ira, IrInst *source_instruction,
2810		ZigVar var, IrInstGen var_ptr)
2811		{
2812		IrInstGenDeclVar *inst = ir_build_inst_gen<IrInstGenDeclVar>(&ira->new_irb,
2813		source_instruction->scope, source_instruction->source_node);
2814		inst->base.value->special = ConstValSpecialStatic;
2815		inst->base.value->type = ira->codegen->builtin_types.entry_void;
2816		inst->var = var;
2817		inst->var_ptr = var_ptr;
2818
2819		ir_ref_inst_gen(var_ptr);
2820
2821		return &inst->base;
2822		}
2823
2824		static IrInstSrc ir_build_export(IrBuilderSrc irb, Scope scope, AstNode source_node,
2825		IrInstSrc target, IrInstSrc options)
2826		{
2827		IrInstSrcExport *export_instruction = ir_build_instruction<IrInstSrcExport>(
2828		irb, scope, source_node);
2829		export_instruction->target = target;
2830		export_instruction->options = options;
2831
2832		ir_ref_instruction(target, irb->current_basic_block);
2833		ir_ref_instruction(options, irb->current_basic_block);
2834
2835		return &export_instruction->base;
2836		}
2837
2838		static IrInstSrc ir_build_extern(IrBuilderSrc irb, Scope scope, AstNode source_node,
2839		IrInstSrc type, IrInstSrc options)
2840		{
2841		IrInstSrcExtern *extern_instruction = ir_build_instruction<IrInstSrcExtern>(
2842		irb, scope, source_node);
2843		extern_instruction->type = type;
2844		extern_instruction->options = options;
2845
2846		ir_ref_instruction(type, irb->current_basic_block);
2847		ir_ref_instruction(options, irb->current_basic_block);
2848
2849		return &extern_instruction->base;
2850		}
2851
2852		static IrInstGen ir_build_extern_gen(IrAnalyze ira, IrInst source_instr, Buf name,
2853		GlobalLinkageId linkage, bool is_thread_local, ZigType *expr_type)
	2125	static IrInstGen ir_build_atomic_load_gen(IrAnalyze ira, IrInst *source_instr,
	2126	IrInstGen ptr, AtomicOrder ordering, ZigType operand_type)
2854	2127	{
2855		IrInstGenExtern *instruction = ir_build_inst_gen<IrInstGenExtern>(&ira->new_irb,
	2128	IrInstGenAtomicLoad *instruction = ir_build_inst_gen<IrInstGenAtomicLoad>(&ira->new_irb,
2856	2129	source_instr->scope, source_instr->source_node);
2857		instruction->base.value->type = expr_type;
2858		instruction->name = name;
2859		instruction->linkage = linkage;
2860		instruction->is_thread_local = is_thread_local;
2861
2862		return &instruction->base;
2863		}
2864
2865		static IrInstSrc ir_build_load_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *ptr) {
2866		IrInstSrcLoadPtr *instruction = ir_build_instruction<IrInstSrcLoadPtr>(irb, scope, source_node);
2867		instruction->ptr = ptr;
2868
2869		ir_ref_instruction(ptr, irb->current_basic_block);
2870
2871		return &instruction->base;
2872		}
2873
2874		static IrInstGen ir_build_load_ptr_gen(IrAnalyze ira, IrInst *source_instruction,
2875		IrInstGen ptr, ZigType ty, IrInstGen *result_loc)
2876		{
2877		IrInstGenLoadPtr *instruction = ir_build_inst_gen<IrInstGenLoadPtr>(
2878		&ira->new_irb, source_instruction->scope, source_instruction->source_node);
2879		instruction->base.value->type = ty;
2880		instruction->ptr = ptr;
2881		instruction->result_loc = result_loc;
2882
2883		ir_ref_inst_gen(ptr);
2884		if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
2885
2886		return &instruction->base;
2887		}
2888
2889		static IrInstSrc ir_build_typeof_n(IrBuilderSrc irb, Scope scope, AstNode source_node,
2890		IrInstSrc **values, size_t value_count)
2891		{
2892		assert(value_count >= 2);
2893
2894		IrInstSrcTypeOf *instruction = ir_build_instruction<IrInstSrcTypeOf>(irb, scope, source_node);
2895		instruction->value.list = values;
2896		instruction->value_count = value_count;
2897
2898		for (size_t i = 0; i < value_count; i++)
2899		ir_ref_instruction(values[i], irb->current_basic_block);
2900
2901		return &instruction->base;
2902		}
2903
2904		static IrInstSrc ir_build_typeof_1(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *value) {
2905		IrInstSrcTypeOf *instruction = ir_build_instruction<IrInstSrcTypeOf>(irb, scope, source_node);
2906		instruction->value.scalar = value;
2907
2908		ir_ref_instruction(value, irb->current_basic_block);
2909
2910		return &instruction->base;
2911		}
2912
2913		static IrInstSrc ir_build_set_cold(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *is_cold) {
2914		IrInstSrcSetCold *instruction = ir_build_instruction<IrInstSrcSetCold>(irb, scope, source_node);
2915		instruction->is_cold = is_cold;
2916
2917		ir_ref_instruction(is_cold, irb->current_basic_block);
2918
2919		return &instruction->base;
2920		}
2921
2922		static IrInstSrc ir_build_set_runtime_safety(IrBuilderSrc irb, Scope scope, AstNode source_node,
2923		IrInstSrc *safety_on)
2924		{
2925		IrInstSrcSetRuntimeSafety *inst = ir_build_instruction<IrInstSrcSetRuntimeSafety>(irb, scope, source_node);
2926		inst->safety_on = safety_on;
2927
2928		ir_ref_instruction(safety_on, irb->current_basic_block);
2929
2930		return &inst->base;
2931		}
2932
2933		static IrInstSrc ir_build_set_float_mode(IrBuilderSrc irb, Scope scope, AstNode source_node,
2934		IrInstSrc *mode_value)
2935		{
2936		IrInstSrcSetFloatMode *instruction = ir_build_instruction<IrInstSrcSetFloatMode>(irb, scope, source_node);
2937		instruction->mode_value = mode_value;
2938
2939		ir_ref_instruction(mode_value, irb->current_basic_block);
2940
2941		return &instruction->base;
2942		}
2943
2944		static IrInstSrc ir_build_array_type(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *size,
2945		IrInstSrc sentinel, IrInstSrc child_type)
2946		{
2947		IrInstSrcArrayType *instruction = ir_build_instruction<IrInstSrcArrayType>(irb, scope, source_node);
2948		instruction->size = size;
2949		instruction->sentinel = sentinel;
2950		instruction->child_type = child_type;
2951
2952		ir_ref_instruction(size, irb->current_basic_block);
2953		if (sentinel != nullptr) ir_ref_instruction(sentinel, irb->current_basic_block);
2954		ir_ref_instruction(child_type, irb->current_basic_block);
2955
2956		return &instruction->base;
2957		}
2958
2959		static IrInstSrc ir_build_anyframe_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
2960		IrInstSrc *payload_type)
2961		{
2962		IrInstSrcAnyFrameType *instruction = ir_build_instruction<IrInstSrcAnyFrameType>(irb, scope, source_node);
2963		instruction->payload_type = payload_type;
2964
2965		if (payload_type != nullptr) ir_ref_instruction(payload_type, irb->current_basic_block);
2966
2967		return &instruction->base;
2968		}
2969
2970		static IrInstSrc ir_build_slice_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
2971		IrInstSrc *child_type, bool is_const, bool is_volatile,
2972		IrInstSrc sentinel, IrInstSrc align_value, bool is_allow_zero)
2973		{
2974		IrInstSrcSliceType *instruction = ir_build_instruction<IrInstSrcSliceType>(irb, scope, source_node);
2975		instruction->is_const = is_const;
2976		instruction->is_volatile = is_volatile;
2977		instruction->child_type = child_type;
2978		instruction->sentinel = sentinel;
2979		instruction->align_value = align_value;
2980		instruction->is_allow_zero = is_allow_zero;
2981
2982		if (sentinel != nullptr) ir_ref_instruction(sentinel, irb->current_basic_block);
2983		if (align_value != nullptr) ir_ref_instruction(align_value, irb->current_basic_block);
2984		ir_ref_instruction(child_type, irb->current_basic_block);
2985
2986		return &instruction->base;
2987		}
2988
2989		static IrInstSrc ir_build_asm_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
2990		IrInstSrc asm_template, IrInstSrc input_list, IrInstSrc *output_types,
2991		ZigVar **output_vars, size_t return_count, bool has_side_effects, bool is_global)
2992		{
2993		IrInstSrcAsm *instruction = ir_build_instruction<IrInstSrcAsm>(irb, scope, source_node);
2994		instruction->asm_template = asm_template;
2995		instruction->input_list = input_list;
2996		instruction->output_types = output_types;
2997		instruction->output_vars = output_vars;
2998		instruction->return_count = return_count;
2999		instruction->has_side_effects = has_side_effects;
3000		instruction->is_global = is_global;
3001
3002		assert(source_node->type == NodeTypeAsmExpr);
3003		for (size_t i = 0; i < source_node->data.asm_expr.output_list.length; i += 1) {
3004		IrInstSrc *output_type = output_types[i];
3005		if (output_type) ir_ref_instruction(output_type, irb->current_basic_block);
3006		}
3007
3008		for (size_t i = 0; i < source_node->data.asm_expr.input_list.length; i += 1) {
3009		IrInstSrc *input_value = input_list[i];
3010		ir_ref_instruction(input_value, irb->current_basic_block);
3011		}
3012
3013		return &instruction->base;
3014		}
3015
3016		static IrInstGen ir_build_asm_gen(IrAnalyze ira, IrInst *source_instr,
3017		Buf asm_template, AsmToken token_list, size_t token_list_len,
3018		IrInstGen input_list, IrInstGen output_types, ZigVar **output_vars, size_t return_count,
3019		bool has_side_effects, ZigType *return_type)
3020		{
3021		IrInstGenAsm *instruction = ir_build_inst_gen<IrInstGenAsm>(&ira->new_irb, source_instr->scope, source_instr->source_node);
3022		instruction->base.value->type = return_type;
3023		instruction->asm_template = asm_template;
3024		instruction->token_list = token_list;
3025		instruction->token_list_len = token_list_len;
3026		instruction->input_list = input_list;
3027		instruction->output_types = output_types;
3028		instruction->output_vars = output_vars;
3029		instruction->return_count = return_count;
3030		instruction->has_side_effects = has_side_effects;
3031
3032		assert(source_instr->source_node->type == NodeTypeAsmExpr);
3033		for (size_t i = 0; i < source_instr->source_node->data.asm_expr.output_list.length; i += 1) {
3034		IrInstGen *output_type = output_types[i];
3035		if (output_type) ir_ref_inst_gen(output_type);
3036		}
3037
3038		for (size_t i = 0; i < source_instr->source_node->data.asm_expr.input_list.length; i += 1) {
3039		IrInstGen *input_value = input_list[i];
3040		ir_ref_inst_gen(input_value);
3041		}
3042
3043		return &instruction->base;
3044		}
3045
3046		static IrInstSrc ir_build_size_of(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type_value,
3047		bool bit_size)
3048		{
3049		IrInstSrcSizeOf *instruction = ir_build_instruction<IrInstSrcSizeOf>(irb, scope, source_node);
3050		instruction->type_value = type_value;
3051		instruction->bit_size = bit_size;
3052
3053		ir_ref_instruction(type_value, irb->current_basic_block);
3054
3055		return &instruction->base;
3056		}
3057
3058		static IrInstSrc ir_build_test_non_null_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
3059		IrInstSrc *value)
3060		{
3061		IrInstSrcTestNonNull *instruction = ir_build_instruction<IrInstSrcTestNonNull>(irb, scope, source_node);
3062		instruction->value = value;
3063
3064		ir_ref_instruction(value, irb->current_basic_block);
3065
3066		return &instruction->base;
3067		}
3068
3069		static IrInstGen ir_build_test_non_null_gen(IrAnalyze ira, IrInst source_instr, IrInstGen value) {
3070		IrInstGenTestNonNull *inst = ir_build_inst_gen<IrInstGenTestNonNull>(&ira->new_irb,
3071		source_instr->scope, source_instr->source_node);
3072		inst->base.value->type = ira->codegen->builtin_types.entry_bool;
3073		inst->value = value;
3074
3075		ir_ref_inst_gen(value);
3076
3077		return &inst->base;
3078		}
3079
3080		static IrInstSrc ir_build_optional_unwrap_ptr(IrBuilderSrc irb, Scope scope, AstNode source_node,
3081		IrInstSrc *base_ptr, bool safety_check_on)
3082		{
3083		IrInstSrcOptionalUnwrapPtr *instruction = ir_build_instruction<IrInstSrcOptionalUnwrapPtr>(irb, scope, source_node);
3084		instruction->base_ptr = base_ptr;
3085		instruction->safety_check_on = safety_check_on;
3086
3087		ir_ref_instruction(base_ptr, irb->current_basic_block);
3088
3089		return &instruction->base;
3090		}
3091
3092		static IrInstGen ir_build_optional_unwrap_ptr_gen(IrAnalyze ira, IrInst *source_instr,
3093		IrInstGen base_ptr, bool safety_check_on, bool initializing, ZigType result_type)
3094		{
3095		IrInstGenOptionalUnwrapPtr *inst = ir_build_inst_gen<IrInstGenOptionalUnwrapPtr>(&ira->new_irb,
3096		source_instr->scope, source_instr->source_node);
3097		inst->base.value->type = result_type;
3098		inst->base_ptr = base_ptr;
3099		inst->safety_check_on = safety_check_on;
3100		inst->initializing = initializing;
3101
3102		ir_ref_inst_gen(base_ptr);
3103
3104		return &inst->base;
3105		}
3106
3107		static IrInstGen ir_build_optional_wrap(IrAnalyze ira, IrInst source_instruction, ZigType result_ty,
3108		IrInstGen operand, IrInstGen result_loc)
3109		{
3110		IrInstGenOptionalWrap *instruction = ir_build_inst_gen<IrInstGenOptionalWrap>(
3111		&ira->new_irb, source_instruction->scope, source_instruction->source_node);
3112		instruction->base.value->type = result_ty;
3113		instruction->operand = operand;
3114		instruction->result_loc = result_loc;
3115
3116		ir_ref_inst_gen(operand);
3117		if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
3118
3119		return &instruction->base;
3120		}
3121
3122		static IrInstGen ir_build_err_wrap_payload(IrAnalyze ira, IrInst *source_instruction,
3123		ZigType result_type, IrInstGen operand, IrInstGen *result_loc)
3124		{
3125		IrInstGenErrWrapPayload *instruction = ir_build_inst_gen<IrInstGenErrWrapPayload>(
3126		&ira->new_irb, source_instruction->scope, source_instruction->source_node);
3127		instruction->base.value->type = result_type;
3128		instruction->operand = operand;
3129		instruction->result_loc = result_loc;
3130
3131		ir_ref_inst_gen(operand);
3132		if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
3133
3134		return &instruction->base;
3135		}
3136
3137		static IrInstGen ir_build_err_wrap_code(IrAnalyze ira, IrInst *source_instruction,
3138		ZigType result_type, IrInstGen operand, IrInstGen *result_loc)
3139		{
3140		IrInstGenErrWrapCode *instruction = ir_build_inst_gen<IrInstGenErrWrapCode>(
3141		&ira->new_irb, source_instruction->scope, source_instruction->source_node);
3142		instruction->base.value->type = result_type;
3143		instruction->operand = operand;
3144		instruction->result_loc = result_loc;
3145
3146		ir_ref_inst_gen(operand);
3147		if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
3148
3149		return &instruction->base;
3150		}
3151
3152		static IrInstSrc ir_build_clz(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type,
3153		IrInstSrc *op)
3154		{
3155		IrInstSrcClz *instruction = ir_build_instruction<IrInstSrcClz>(irb, scope, source_node);
3156		instruction->type = type;
3157		instruction->op = op;
3158
3159		ir_ref_instruction(type, irb->current_basic_block);
3160		ir_ref_instruction(op, irb->current_basic_block);
3161
3162		return &instruction->base;
3163		}
3164
3165		static IrInstGen ir_build_clz_gen(IrAnalyze ira, IrInst source_instr, ZigType result_type, IrInstGen *op) {
3166		IrInstGenClz *instruction = ir_build_inst_gen<IrInstGenClz>(&ira->new_irb,
3167		source_instr->scope, source_instr->source_node);
3168		instruction->base.value->type = result_type;
3169		instruction->op = op;
3170
3171		ir_ref_inst_gen(op);
3172
3173		return &instruction->base;
3174		}
3175
3176		static IrInstSrc ir_build_ctz(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type,
3177		IrInstSrc *op)
3178		{
3179		IrInstSrcCtz *instruction = ir_build_instruction<IrInstSrcCtz>(irb, scope, source_node);
3180		instruction->type = type;
3181		instruction->op = op;
3182
3183		ir_ref_instruction(type, irb->current_basic_block);
3184		ir_ref_instruction(op, irb->current_basic_block);
3185
3186		return &instruction->base;
3187		}
3188
3189		static IrInstGen ir_build_ctz_gen(IrAnalyze ira, IrInst source_instr, ZigType result_type, IrInstGen *op) {
3190		IrInstGenCtz *instruction = ir_build_inst_gen<IrInstGenCtz>(&ira->new_irb,
3191		source_instr->scope, source_instr->source_node);
3192		instruction->base.value->type = result_type;
3193		instruction->op = op;
3194
3195		ir_ref_inst_gen(op);
3196
3197		return &instruction->base;
3198		}
3199
3200		static IrInstSrc ir_build_pop_count(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type,
3201		IrInstSrc *op)
3202		{
3203		IrInstSrcPopCount *instruction = ir_build_instruction<IrInstSrcPopCount>(irb, scope, source_node);
3204		instruction->type = type;
3205		instruction->op = op;
3206
3207		ir_ref_instruction(type, irb->current_basic_block);
3208		ir_ref_instruction(op, irb->current_basic_block);
3209
3210		return &instruction->base;
3211		}
3212
3213		static IrInstGen ir_build_pop_count_gen(IrAnalyze ira, IrInst source_instr, ZigType result_type,
3214		IrInstGen *op)
3215		{
3216		IrInstGenPopCount *instruction = ir_build_inst_gen<IrInstGenPopCount>(&ira->new_irb,
3217		source_instr->scope, source_instr->source_node);
3218		instruction->base.value->type = result_type;
3219		instruction->op = op;
3220
3221		ir_ref_inst_gen(op);
3222
3223		return &instruction->base;
3224		}
3225
3226		static IrInstSrc ir_build_bswap(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type,
3227		IrInstSrc *op)
3228		{
3229		IrInstSrcBswap *instruction = ir_build_instruction<IrInstSrcBswap>(irb, scope, source_node);
3230		instruction->type = type;
3231		instruction->op = op;
3232
3233		ir_ref_instruction(type, irb->current_basic_block);
3234		ir_ref_instruction(op, irb->current_basic_block);
3235
3236		return &instruction->base;
3237		}
3238
3239		static IrInstGen ir_build_bswap_gen(IrAnalyze ira, IrInst source_instr, ZigType op_type,
3240		IrInstGen *op)
3241		{
3242		IrInstGenBswap *instruction = ir_build_inst_gen<IrInstGenBswap>(&ira->new_irb,
3243		source_instr->scope, source_instr->source_node);
3244		instruction->base.value->type = op_type;
3245		instruction->op = op;
3246
3247		ir_ref_inst_gen(op);
3248
3249		return &instruction->base;
3250		}
3251
3252		static IrInstSrc ir_build_bit_reverse(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type,
3253		IrInstSrc *op)
3254		{
3255		IrInstSrcBitReverse *instruction = ir_build_instruction<IrInstSrcBitReverse>(irb, scope, source_node);
3256		instruction->type = type;
3257		instruction->op = op;
3258
3259		ir_ref_instruction(type, irb->current_basic_block);
3260		ir_ref_instruction(op, irb->current_basic_block);
3261
3262		return &instruction->base;
3263		}
3264
3265		static IrInstGen ir_build_bit_reverse_gen(IrAnalyze ira, IrInst source_instr, ZigType int_type,
3266		IrInstGen *op)
3267		{
3268		IrInstGenBitReverse *instruction = ir_build_inst_gen<IrInstGenBitReverse>(&ira->new_irb,
3269		source_instr->scope, source_instr->source_node);
3270		instruction->base.value->type = int_type;
3271		instruction->op = op;
3272
3273		ir_ref_inst_gen(op);
3274
3275		return &instruction->base;
3276		}
3277
3278		static IrInstSrcSwitchBr ir_build_switch_br_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
3279		IrInstSrc target_value, IrBasicBlockSrc else_block, size_t case_count, IrInstSrcSwitchBrCase *cases,
3280		IrInstSrc is_comptime, IrInstSrc switch_prongs_void)
3281		{
3282		IrInstSrcSwitchBr *instruction = ir_build_instruction<IrInstSrcSwitchBr>(irb, scope, source_node);
3283		instruction->base.is_noreturn = true;
3284		instruction->target_value = target_value;
3285		instruction->else_block = else_block;
3286		instruction->case_count = case_count;
3287		instruction->cases = cases;
3288		instruction->is_comptime = is_comptime;
3289		instruction->switch_prongs_void = switch_prongs_void;
3290
3291		ir_ref_instruction(target_value, irb->current_basic_block);
3292		ir_ref_instruction(is_comptime, irb->current_basic_block);
3293		ir_ref_bb(else_block);
3294		ir_ref_instruction(switch_prongs_void, irb->current_basic_block);
3295
3296		for (size_t i = 0; i < case_count; i += 1) {
3297		ir_ref_instruction(cases[i].value, irb->current_basic_block);
3298		ir_ref_bb(cases[i].block);
3299		}
3300
3301		return instruction;
3302		}
3303
3304		static IrInstGenSwitchBr ir_build_switch_br_gen(IrAnalyze ira, IrInst *source_instr,
3305		IrInstGen target_value, IrBasicBlockGen else_block, size_t case_count, IrInstGenSwitchBrCase *cases)
3306		{
3307		IrInstGenSwitchBr *instruction = ir_build_inst_noreturn<IrInstGenSwitchBr>(&ira->new_irb,
3308		source_instr->scope, source_instr->source_node);
3309		instruction->target_value = target_value;
3310		instruction->else_block = else_block;
3311		instruction->case_count = case_count;
3312		instruction->cases = cases;
3313
3314		ir_ref_inst_gen(target_value);
3315
3316		for (size_t i = 0; i < case_count; i += 1) {
3317		ir_ref_inst_gen(cases[i].value);
3318		}
3319
3320		return instruction;
3321		}
3322
3323		static IrInstSrc ir_build_switch_target(IrBuilderSrc irb, Scope scope, AstNode source_node,
3324		IrInstSrc *target_value_ptr)
3325		{
3326		IrInstSrcSwitchTarget *instruction = ir_build_instruction<IrInstSrcSwitchTarget>(irb, scope, source_node);
3327		instruction->target_value_ptr = target_value_ptr;
3328
3329		ir_ref_instruction(target_value_ptr, irb->current_basic_block);
3330
3331		return &instruction->base;
3332		}
3333
3334		static IrInstSrc ir_build_switch_var(IrBuilderSrc irb, Scope scope, AstNode source_node,
3335		IrInstSrc target_value_ptr, IrInstSrc *prongs_ptr, size_t prongs_len)
3336		{
3337		IrInstSrcSwitchVar *instruction = ir_build_instruction<IrInstSrcSwitchVar>(irb, scope, source_node);
3338		instruction->target_value_ptr = target_value_ptr;
3339		instruction->prongs_ptr = prongs_ptr;
3340		instruction->prongs_len = prongs_len;
3341
3342		ir_ref_instruction(target_value_ptr, irb->current_basic_block);
3343		for (size_t i = 0; i < prongs_len; i += 1) {
3344		ir_ref_instruction(prongs_ptr[i], irb->current_basic_block);
3345		}
3346
3347		return &instruction->base;
3348		}
3349
3350		// For this instruction the switch_br must be set later.
3351		static IrInstSrcSwitchElseVar ir_build_switch_else_var(IrBuilderSrc irb, Scope scope, AstNode source_node,
3352		IrInstSrc *target_value_ptr)
3353		{
3354		IrInstSrcSwitchElseVar *instruction = ir_build_instruction<IrInstSrcSwitchElseVar>(irb, scope, source_node);
3355		instruction->target_value_ptr = target_value_ptr;
3356
3357		ir_ref_instruction(target_value_ptr, irb->current_basic_block);
3358
3359		return instruction;
3360		}
3361
3362		static IrInstGen ir_build_union_tag(IrAnalyze ira, IrInst source_instr, IrInstGen value,
3363		ZigType *tag_type)
3364		{
3365		IrInstGenUnionTag *instruction = ir_build_inst_gen<IrInstGenUnionTag>(&ira->new_irb,
3366		source_instr->scope, source_instr->source_node);
3367		instruction->value = value;
3368		instruction->base.value->type = tag_type;
3369
3370		ir_ref_inst_gen(value);
3371
3372		return &instruction->base;
3373		}
3374
3375		static IrInstSrc ir_build_import(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *name) {
3376		IrInstSrcImport *instruction = ir_build_instruction<IrInstSrcImport>(irb, scope, source_node);
3377		instruction->name = name;
3378
3379		ir_ref_instruction(name, irb->current_basic_block);
3380
3381		return &instruction->base;
3382		}
3383
3384		static IrInstSrc ir_build_ref_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *value) {
3385		IrInstSrcRef *instruction = ir_build_instruction<IrInstSrcRef>(irb, scope, source_node);
3386		instruction->value = value;
3387
3388		ir_ref_instruction(value, irb->current_basic_block);
3389
3390		return &instruction->base;
3391		}
3392
3393		static IrInstGen ir_build_ref_gen(IrAnalyze ira, IrInst source_instruction, ZigType result_type,
3394		IrInstGen operand, IrInstGen result_loc)
3395		{
3396		IrInstGenRef *instruction = ir_build_inst_gen<IrInstGenRef>(&ira->new_irb,
3397		source_instruction->scope, source_instruction->source_node);
3398		instruction->base.value->type = result_type;
3399		instruction->operand = operand;
3400		instruction->result_loc = result_loc;
3401
3402		ir_ref_inst_gen(operand);
3403		if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
3404
3405		return &instruction->base;
3406		}
3407
3408		static IrInstSrc ir_build_compile_err(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *msg) {
3409		IrInstSrcCompileErr *instruction = ir_build_instruction<IrInstSrcCompileErr>(irb, scope, source_node);
3410		instruction->msg = msg;
3411
3412		ir_ref_instruction(msg, irb->current_basic_block);
3413
3414		return &instruction->base;
3415		}
3416
3417		static IrInstSrc ir_build_compile_log(IrBuilderSrc irb, Scope scope, AstNode source_node,
3418		size_t msg_count, IrInstSrc **msg_list)
3419		{
3420		IrInstSrcCompileLog *instruction = ir_build_instruction<IrInstSrcCompileLog>(irb, scope, source_node);
3421		instruction->msg_count = msg_count;
3422		instruction->msg_list = msg_list;
3423
3424		for (size_t i = 0; i < msg_count; i += 1) {
3425		ir_ref_instruction(msg_list[i], irb->current_basic_block);
3426		}
3427
3428		return &instruction->base;
3429		}
3430
3431		static IrInstSrc ir_build_err_name(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *value) {
3432		IrInstSrcErrName *instruction = ir_build_instruction<IrInstSrcErrName>(irb, scope, source_node);
3433		instruction->value = value;
3434
3435		ir_ref_instruction(value, irb->current_basic_block);
3436
3437		return &instruction->base;
3438		}
3439
3440		static IrInstGen ir_build_err_name_gen(IrAnalyze ira, IrInst source_instr, IrInstGen value,
3441		ZigType *str_type)
3442		{
3443		IrInstGenErrName *instruction = ir_build_inst_gen<IrInstGenErrName>(&ira->new_irb,
3444		source_instr->scope, source_instr->source_node);
3445		instruction->base.value->type = str_type;
3446		instruction->value = value;
3447
3448		ir_ref_inst_gen(value);
3449
3450		return &instruction->base;
3451		}
3452
3453		static IrInstSrc ir_build_c_import(IrBuilderSrc irb, Scope scope, AstNode source_node) {
3454		IrInstSrcCImport *instruction = ir_build_instruction<IrInstSrcCImport>(irb, scope, source_node);
3455		return &instruction->base;
3456		}
3457
3458		static IrInstSrc ir_build_c_include(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *name) {
3459		IrInstSrcCInclude *instruction = ir_build_instruction<IrInstSrcCInclude>(irb, scope, source_node);
3460		instruction->name = name;
3461
3462		ir_ref_instruction(name, irb->current_basic_block);
3463
3464		return &instruction->base;
3465		}
3466
3467		static IrInstSrc ir_build_c_define(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc name, IrInstSrc value) {
3468		IrInstSrcCDefine *instruction = ir_build_instruction<IrInstSrcCDefine>(irb, scope, source_node);
3469		instruction->name = name;
3470		instruction->value = value;
3471
3472		ir_ref_instruction(name, irb->current_basic_block);
3473		ir_ref_instruction(value, irb->current_basic_block);
3474
3475		return &instruction->base;
3476		}
3477
3478		static IrInstSrc ir_build_c_undef(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *name) {
3479		IrInstSrcCUndef *instruction = ir_build_instruction<IrInstSrcCUndef>(irb, scope, source_node);
3480		instruction->name = name;
3481
3482		ir_ref_instruction(name, irb->current_basic_block);
3483
3484		return &instruction->base;
3485		}
3486
3487		static IrInstSrc ir_build_embed_file(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *name) {
3488		IrInstSrcEmbedFile *instruction = ir_build_instruction<IrInstSrcEmbedFile>(irb, scope, source_node);
3489		instruction->name = name;
3490
3491		ir_ref_instruction(name, irb->current_basic_block);
3492
3493		return &instruction->base;
3494		}
3495
3496		static IrInstSrc ir_build_cmpxchg_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
3497		IrInstSrc type_value, IrInstSrc ptr, IrInstSrc cmp_value, IrInstSrc new_value,
3498		IrInstSrc success_order_value, IrInstSrc failure_order_value, bool is_weak, ResultLoc *result_loc)
3499		{
3500		IrInstSrcCmpxchg *instruction = ir_build_instruction<IrInstSrcCmpxchg>(irb, scope, source_node);
3501		instruction->type_value = type_value;
3502		instruction->ptr = ptr;
3503		instruction->cmp_value = cmp_value;
3504		instruction->new_value = new_value;
3505		instruction->success_order_value = success_order_value;
3506		instruction->failure_order_value = failure_order_value;
3507		instruction->is_weak = is_weak;
3508		instruction->result_loc = result_loc;
3509
3510		ir_ref_instruction(type_value, irb->current_basic_block);
3511		ir_ref_instruction(ptr, irb->current_basic_block);
3512		ir_ref_instruction(cmp_value, irb->current_basic_block);
3513		ir_ref_instruction(new_value, irb->current_basic_block);
3514		ir_ref_instruction(success_order_value, irb->current_basic_block);
3515		ir_ref_instruction(failure_order_value, irb->current_basic_block);
3516
3517		return &instruction->base;
3518		}
3519
3520		static IrInstGen ir_build_cmpxchg_gen(IrAnalyze ira, IrInst source_instruction, ZigType result_type,
3521		IrInstGen ptr, IrInstGen cmp_value, IrInstGen *new_value,
3522		AtomicOrder success_order, AtomicOrder failure_order, bool is_weak, IrInstGen *result_loc)
3523		{
3524		IrInstGenCmpxchg *instruction = ir_build_inst_gen<IrInstGenCmpxchg>(&ira->new_irb,
3525		source_instruction->scope, source_instruction->source_node);
3526		instruction->base.value->type = result_type;
3527		instruction->ptr = ptr;
3528		instruction->cmp_value = cmp_value;
3529		instruction->new_value = new_value;
3530		instruction->success_order = success_order;
3531		instruction->failure_order = failure_order;
3532		instruction->is_weak = is_weak;
3533		instruction->result_loc = result_loc;
3534
3535		ir_ref_inst_gen(ptr);
3536		ir_ref_inst_gen(cmp_value);
3537		ir_ref_inst_gen(new_value);
3538		if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
3539
3540		return &instruction->base;
3541		}
3542
3543		static IrInstSrc ir_build_fence(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *order) {
3544		IrInstSrcFence *instruction = ir_build_instruction<IrInstSrcFence>(irb, scope, source_node);
3545		instruction->order = order;
3546
3547		ir_ref_instruction(order, irb->current_basic_block);
3548
3549		return &instruction->base;
3550		}
3551
3552		static IrInstGen ir_build_fence_gen(IrAnalyze ira, IrInst *source_instr, AtomicOrder order) {
3553		IrInstGenFence *instruction = ir_build_inst_void<IrInstGenFence>(&ira->new_irb,
3554		source_instr->scope, source_instr->source_node);
3555		instruction->order = order;
3556
3557		return &instruction->base;
3558		}
3559
3560		static IrInstSrc ir_build_reduce(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc op, IrInstSrc value) {
3561		IrInstSrcReduce *instruction = ir_build_instruction<IrInstSrcReduce>(irb, scope, source_node);
3562		instruction->op = op;
3563		instruction->value = value;
3564
3565		ir_ref_instruction(op, irb->current_basic_block);
3566		ir_ref_instruction(value, irb->current_basic_block);
3567
3568		return &instruction->base;
3569		}
3570
3571		static IrInstGen ir_build_reduce_gen(IrAnalyze ira, IrInst source_instruction, ReduceOp op, IrInstGen value, ZigType *result_type) {
3572		IrInstGenReduce *instruction = ir_build_inst_gen<IrInstGenReduce>(&ira->new_irb,
3573		source_instruction->scope, source_instruction->source_node);
3574		instruction->base.value->type = result_type;
3575		instruction->op = op;
3576		instruction->value = value;
3577
3578		ir_ref_inst_gen(value);
3579
3580		return &instruction->base;
3581		}
3582
3583		static IrInstSrc ir_build_truncate(IrBuilderSrc irb, Scope scope, AstNode source_node,
3584		IrInstSrc dest_type, IrInstSrc target)
3585		{
3586		IrInstSrcTruncate *instruction = ir_build_instruction<IrInstSrcTruncate>(irb, scope, source_node);
3587		instruction->dest_type = dest_type;
3588		instruction->target = target;
3589
3590		ir_ref_instruction(dest_type, irb->current_basic_block);
3591		ir_ref_instruction(target, irb->current_basic_block);
3592
3593		return &instruction->base;
3594		}
3595
3596		static IrInstGen ir_build_truncate_gen(IrAnalyze ira, IrInst source_instr, ZigType dest_type,
3597		IrInstGen *target)
3598		{
3599		IrInstGenTruncate *instruction = ir_build_inst_gen<IrInstGenTruncate>(&ira->new_irb,
3600		source_instr->scope, source_instr->source_node);
3601		instruction->base.value->type = dest_type;
3602		instruction->target = target;
3603
3604		ir_ref_inst_gen(target);
3605
3606		return &instruction->base;
3607		}
3608
3609		static IrInstSrc ir_build_int_cast(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *dest_type,
3610		IrInstSrc *target)
3611		{
3612		IrInstSrcIntCast *instruction = ir_build_instruction<IrInstSrcIntCast>(irb, scope, source_node);
3613		instruction->dest_type = dest_type;
3614		instruction->target = target;
3615
3616		ir_ref_instruction(dest_type, irb->current_basic_block);
3617		ir_ref_instruction(target, irb->current_basic_block);
3618
3619		return &instruction->base;
3620		}
3621
3622		static IrInstSrc ir_build_float_cast(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *dest_type,
3623		IrInstSrc *target)
3624		{
3625		IrInstSrcFloatCast *instruction = ir_build_instruction<IrInstSrcFloatCast>(irb, scope, source_node);
3626		instruction->dest_type = dest_type;
3627		instruction->target = target;
3628
3629		ir_ref_instruction(dest_type, irb->current_basic_block);
3630		ir_ref_instruction(target, irb->current_basic_block);
3631
3632		return &instruction->base;
3633		}
3634
3635		static IrInstSrc ir_build_err_set_cast(IrBuilderSrc irb, Scope scope, AstNode source_node,
3636		IrInstSrc dest_type, IrInstSrc target)
3637		{
3638		IrInstSrcErrSetCast *instruction = ir_build_instruction<IrInstSrcErrSetCast>(irb, scope, source_node);
3639		instruction->dest_type = dest_type;
3640		instruction->target = target;
3641
3642		ir_ref_instruction(dest_type, irb->current_basic_block);
3643		ir_ref_instruction(target, irb->current_basic_block);
3644
3645		return &instruction->base;
3646		}
3647
3648		static IrInstSrc ir_build_int_to_float(IrBuilderSrc irb, Scope scope, AstNode source_node,
3649		IrInstSrc dest_type, IrInstSrc target)
3650		{
3651		IrInstSrcIntToFloat *instruction = ir_build_instruction<IrInstSrcIntToFloat>(irb, scope, source_node);
3652		instruction->dest_type = dest_type;
3653		instruction->target = target;
3654
3655		ir_ref_instruction(dest_type, irb->current_basic_block);
3656		ir_ref_instruction(target, irb->current_basic_block);
3657
3658		return &instruction->base;
3659		}
3660
3661		static IrInstSrc ir_build_float_to_int(IrBuilderSrc irb, Scope scope, AstNode source_node,
3662		IrInstSrc dest_type, IrInstSrc target)
3663		{
3664		IrInstSrcFloatToInt *instruction = ir_build_instruction<IrInstSrcFloatToInt>(irb, scope, source_node);
3665		instruction->dest_type = dest_type;
3666		instruction->target = target;
3667
3668		ir_ref_instruction(dest_type, irb->current_basic_block);
3669		ir_ref_instruction(target, irb->current_basic_block);
3670
3671		return &instruction->base;
3672		}
3673
3674		static IrInstSrc ir_build_bool_to_int(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *target) {
3675		IrInstSrcBoolToInt *instruction = ir_build_instruction<IrInstSrcBoolToInt>(irb, scope, source_node);
3676		instruction->target = target;
3677
3678		ir_ref_instruction(target, irb->current_basic_block);
3679
3680		return &instruction->base;
3681		}
3682
3683		static IrInstSrc ir_build_vector_type(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *len,
3684		IrInstSrc *elem_type)
3685		{
3686		IrInstSrcVectorType *instruction = ir_build_instruction<IrInstSrcVectorType>(irb, scope, source_node);
3687		instruction->len = len;
3688		instruction->elem_type = elem_type;
3689
3690		ir_ref_instruction(len, irb->current_basic_block);
3691		ir_ref_instruction(elem_type, irb->current_basic_block);
3692
3693		return &instruction->base;
3694		}
3695
3696		static IrInstSrc ir_build_shuffle_vector(IrBuilderSrc irb, Scope scope, AstNode source_node,
3697		IrInstSrc scalar_type, IrInstSrc a, IrInstSrc b, IrInstSrc mask)
3698		{
3699		IrInstSrcShuffleVector *instruction = ir_build_instruction<IrInstSrcShuffleVector>(irb, scope, source_node);
3700		instruction->scalar_type = scalar_type;
3701		instruction->a = a;
3702		instruction->b = b;
3703		instruction->mask = mask;
3704
3705		if (scalar_type != nullptr) ir_ref_instruction(scalar_type, irb->current_basic_block);
3706		ir_ref_instruction(a, irb->current_basic_block);
3707		ir_ref_instruction(b, irb->current_basic_block);
3708		ir_ref_instruction(mask, irb->current_basic_block);
3709
3710		return &instruction->base;
3711		}
3712
3713		static IrInstGen ir_build_shuffle_vector_gen(IrAnalyze ira, Scope scope, AstNode source_node,
3714		ZigType result_type, IrInstGen a, IrInstGen b, IrInstGen mask)
3715		{
3716		IrInstGenShuffleVector *inst = ir_build_inst_gen<IrInstGenShuffleVector>(&ira->new_irb, scope, source_node);
3717		inst->base.value->type = result_type;
3718		inst->a = a;
3719		inst->b = b;
3720		inst->mask = mask;
3721
3722		ir_ref_inst_gen(a);
3723		ir_ref_inst_gen(b);
3724		ir_ref_inst_gen(mask);
3725
3726		return &inst->base;
3727		}
3728
3729		static IrInstSrc ir_build_splat_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
3730		IrInstSrc len, IrInstSrc scalar)
3731		{
3732		IrInstSrcSplat *instruction = ir_build_instruction<IrInstSrcSplat>(irb, scope, source_node);
3733		instruction->len = len;
3734		instruction->scalar = scalar;
3735
3736		ir_ref_instruction(len, irb->current_basic_block);
3737		ir_ref_instruction(scalar, irb->current_basic_block);
3738
3739		return &instruction->base;
3740		}
3741
3742		static IrInstGen ir_build_splat_gen(IrAnalyze ira, IrInst source_instruction, ZigType result_type,
3743		IrInstGen *scalar)
3744		{
3745		IrInstGenSplat *instruction = ir_build_inst_gen<IrInstGenSplat>(
3746		&ira->new_irb, source_instruction->scope, source_instruction->source_node);
3747		instruction->base.value->type = result_type;
3748		instruction->scalar = scalar;
3749
3750		ir_ref_inst_gen(scalar);
3751
3752		return &instruction->base;
3753		}
3754
3755		static IrInstSrc ir_build_bool_not(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *value) {
3756		IrInstSrcBoolNot *instruction = ir_build_instruction<IrInstSrcBoolNot>(irb, scope, source_node);
3757		instruction->value = value;
3758
3759		ir_ref_instruction(value, irb->current_basic_block);
3760
3761		return &instruction->base;
3762		}
3763
3764		static IrInstGen ir_build_bool_not_gen(IrAnalyze ira, IrInst source_instr, IrInstGen value) {
3765		IrInstGenBoolNot *instruction = ir_build_inst_gen<IrInstGenBoolNot>(&ira->new_irb,
3766		source_instr->scope, source_instr->source_node);
3767		instruction->base.value->type = ira->codegen->builtin_types.entry_bool;
3768		instruction->value = value;
3769
3770		ir_ref_inst_gen(value);
3771
3772		return &instruction->base;
3773		}
3774
3775		static IrInstSrc ir_build_memset_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
3776		IrInstSrc dest_ptr, IrInstSrc byte, IrInstSrc *count)
3777		{
3778		IrInstSrcMemset *instruction = ir_build_instruction<IrInstSrcMemset>(irb, scope, source_node);
3779		instruction->dest_ptr = dest_ptr;
3780		instruction->byte = byte;
3781		instruction->count = count;
3782
3783		ir_ref_instruction(dest_ptr, irb->current_basic_block);
3784		ir_ref_instruction(byte, irb->current_basic_block);
3785		ir_ref_instruction(count, irb->current_basic_block);
3786
3787		return &instruction->base;
3788		}
3789
3790		static IrInstGen ir_build_memset_gen(IrAnalyze ira, IrInst *source_instr,
3791		IrInstGen dest_ptr, IrInstGen byte, IrInstGen *count)
3792		{
3793		IrInstGenMemset *instruction = ir_build_inst_void<IrInstGenMemset>(&ira->new_irb,
3794		source_instr->scope, source_instr->source_node);
3795		instruction->dest_ptr = dest_ptr;
3796		instruction->byte = byte;
3797		instruction->count = count;
3798
3799		ir_ref_inst_gen(dest_ptr);
3800		ir_ref_inst_gen(byte);
3801		ir_ref_inst_gen(count);
3802
3803		return &instruction->base;
3804		}
3805
3806		static IrInstSrc ir_build_memcpy_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
3807		IrInstSrc dest_ptr, IrInstSrc src_ptr, IrInstSrc *count)
3808		{
3809		IrInstSrcMemcpy *instruction = ir_build_instruction<IrInstSrcMemcpy>(irb, scope, source_node);
3810		instruction->dest_ptr = dest_ptr;
3811		instruction->src_ptr = src_ptr;
3812		instruction->count = count;
3813
3814		ir_ref_instruction(dest_ptr, irb->current_basic_block);
3815		ir_ref_instruction(src_ptr, irb->current_basic_block);
3816		ir_ref_instruction(count, irb->current_basic_block);
3817
3818		return &instruction->base;
3819		}
3820
3821		static IrInstGen ir_build_memcpy_gen(IrAnalyze ira, IrInst *source_instr,
3822		IrInstGen dest_ptr, IrInstGen src_ptr, IrInstGen *count)
3823		{
3824		IrInstGenMemcpy *instruction = ir_build_inst_void<IrInstGenMemcpy>(&ira->new_irb,
3825		source_instr->scope, source_instr->source_node);
3826		instruction->dest_ptr = dest_ptr;
3827		instruction->src_ptr = src_ptr;
3828		instruction->count = count;
3829
3830		ir_ref_inst_gen(dest_ptr);
3831		ir_ref_inst_gen(src_ptr);
3832		ir_ref_inst_gen(count);
3833
3834		return &instruction->base;
3835		}
3836
3837		static IrInstSrc ir_build_slice_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
3838		IrInstSrc ptr, IrInstSrc start, IrInstSrc end, IrInstSrc sentinel,
3839		bool safety_check_on, ResultLoc *result_loc)
3840		{
3841		IrInstSrcSlice *instruction = ir_build_instruction<IrInstSrcSlice>(irb, scope, source_node);
3842		instruction->ptr = ptr;
3843		instruction->start = start;
3844		instruction->end = end;
3845		instruction->sentinel = sentinel;
3846		instruction->safety_check_on = safety_check_on;
3847		instruction->result_loc = result_loc;
3848
3849		ir_ref_instruction(ptr, irb->current_basic_block);
3850		ir_ref_instruction(start, irb->current_basic_block);
3851		if (end) ir_ref_instruction(end, irb->current_basic_block);
3852		if (sentinel) ir_ref_instruction(sentinel, irb->current_basic_block);
3853
3854		return &instruction->base;
3855		}
3856
3857		static IrInstGen ir_build_slice_gen(IrAnalyze ira, IrInst source_instruction, ZigType slice_type,
3858		IrInstGen ptr, IrInstGen start, IrInstGen end, bool safety_check_on, IrInstGen result_loc,
3859		ZigValue *sentinel)
3860		{
3861		IrInstGenSlice *instruction = ir_build_inst_gen<IrInstGenSlice>(
3862		&ira->new_irb, source_instruction->scope, source_instruction->source_node);
3863		instruction->base.value->type = slice_type;
3864		instruction->ptr = ptr;
3865		instruction->start = start;
3866		instruction->end = end;
3867		instruction->safety_check_on = safety_check_on;
3868		instruction->result_loc = result_loc;
3869		instruction->sentinel = sentinel;
3870
3871		ir_ref_inst_gen(ptr);
3872		ir_ref_inst_gen(start);
3873		if (end != nullptr) ir_ref_inst_gen(end);
3874		if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
3875
3876		return &instruction->base;
3877		}
3878
3879		static IrInstSrc ir_build_breakpoint(IrBuilderSrc irb, Scope scope, AstNode source_node) {
3880		IrInstSrcBreakpoint *instruction = ir_build_instruction<IrInstSrcBreakpoint>(irb, scope, source_node);
3881		return &instruction->base;
3882		}
3883
3884		static IrInstGen ir_build_breakpoint_gen(IrAnalyze ira, IrInst *source_instr) {
3885		IrInstGenBreakpoint *instruction = ir_build_inst_void<IrInstGenBreakpoint>(&ira->new_irb,
3886		source_instr->scope, source_instr->source_node);
3887		return &instruction->base;
3888		}
3889
3890		static IrInstSrc ir_build_return_address_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
3891		IrInstSrcReturnAddress *instruction = ir_build_instruction<IrInstSrcReturnAddress>(irb, scope, source_node);
3892		return &instruction->base;
3893		}
3894
3895		static IrInstGen ir_build_return_address_gen(IrAnalyze ira, IrInst *source_instr) {
3896		IrInstGenReturnAddress *inst = ir_build_inst_gen<IrInstGenReturnAddress>(&ira->new_irb, source_instr->scope, source_instr->source_node);
3897		inst->base.value->type = ira->codegen->builtin_types.entry_usize;
3898		return &inst->base;
3899		}
3900
3901		static IrInstSrc ir_build_frame_address_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
3902		IrInstSrcFrameAddress *inst = ir_build_instruction<IrInstSrcFrameAddress>(irb, scope, source_node);
3903		return &inst->base;
3904		}
3905
3906		static IrInstGen ir_build_frame_address_gen(IrAnalyze ira, IrInst *source_instr) {
3907		IrInstGenFrameAddress *inst = ir_build_inst_gen<IrInstGenFrameAddress>(&ira->new_irb, source_instr->scope, source_instr->source_node);
3908		inst->base.value->type = ira->codegen->builtin_types.entry_usize;
3909		return &inst->base;
3910		}
3911
3912		static IrInstSrc ir_build_handle_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
3913		IrInstSrcFrameHandle *inst = ir_build_instruction<IrInstSrcFrameHandle>(irb, scope, source_node);
3914		return &inst->base;
3915		}
3916
3917		static IrInstGen ir_build_handle_gen(IrAnalyze ira, IrInst source_instr, ZigType ty) {
3918		IrInstGenFrameHandle *inst = ir_build_inst_gen<IrInstGenFrameHandle>(&ira->new_irb, source_instr->scope, source_instr->source_node);
3919		inst->base.value->type = ty;
3920		return &inst->base;
3921		}
3922
3923		static IrInstSrc ir_build_frame_type(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *fn) {
3924		IrInstSrcFrameType *inst = ir_build_instruction<IrInstSrcFrameType>(irb, scope, source_node);
3925		inst->fn = fn;
3926
3927		ir_ref_instruction(fn, irb->current_basic_block);
3928
3929		return &inst->base;
3930		}
3931
3932		static IrInstSrc ir_build_frame_size_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *fn) {
3933		IrInstSrcFrameSize *inst = ir_build_instruction<IrInstSrcFrameSize>(irb, scope, source_node);
3934		inst->fn = fn;
3935
3936		ir_ref_instruction(fn, irb->current_basic_block);
3937
3938		return &inst->base;
3939		}
3940
3941		static IrInstGen ir_build_frame_size_gen(IrAnalyze ira, IrInst source_instr, IrInstGen fn)
3942		{
3943		IrInstGenFrameSize *inst = ir_build_inst_gen<IrInstGenFrameSize>(&ira->new_irb, source_instr->scope, source_instr->source_node);
3944		inst->base.value->type = ira->codegen->builtin_types.entry_usize;
3945		inst->fn = fn;
3946
3947		ir_ref_inst_gen(fn);
3948
3949		return &inst->base;
3950		}
3951
3952		static IrInstSrc ir_build_overflow_op_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
3953		IrOverflowOp op, IrInstSrc type_value, IrInstSrc op1, IrInstSrc op2, IrInstSrc result_ptr)
3954		{
3955		IrInstSrcOverflowOp *instruction = ir_build_instruction<IrInstSrcOverflowOp>(irb, scope, source_node);
3956		instruction->op = op;
3957		instruction->type_value = type_value;
3958		instruction->op1 = op1;
3959		instruction->op2 = op2;
3960		instruction->result_ptr = result_ptr;
3961
3962		ir_ref_instruction(type_value, irb->current_basic_block);
3963		ir_ref_instruction(op1, irb->current_basic_block);
3964		ir_ref_instruction(op2, irb->current_basic_block);
3965		ir_ref_instruction(result_ptr, irb->current_basic_block);
3966
3967		return &instruction->base;
3968		}
3969
3970		static IrInstGen ir_build_overflow_op_gen(IrAnalyze ira, IrInst *source_instr,
3971		IrOverflowOp op, IrInstGen op1, IrInstGen op2, IrInstGen *result_ptr,
3972		ZigType *result_ptr_type)
3973		{
3974		IrInstGenOverflowOp *instruction = ir_build_inst_gen<IrInstGenOverflowOp>(&ira->new_irb,
3975		source_instr->scope, source_instr->source_node);
3976		instruction->base.value->type = ira->codegen->builtin_types.entry_bool;
3977		instruction->op = op;
3978		instruction->op1 = op1;
3979		instruction->op2 = op2;
3980		instruction->result_ptr = result_ptr;
3981		instruction->result_ptr_type = result_ptr_type;
3982
3983		ir_ref_inst_gen(op1);
3984		ir_ref_inst_gen(op2);
3985		ir_ref_inst_gen(result_ptr);
3986
3987		return &instruction->base;
3988		}
3989
3990		static IrInstSrc ir_build_float_op_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *operand,
3991		BuiltinFnId fn_id)
3992		{
3993		IrInstSrcFloatOp *instruction = ir_build_instruction<IrInstSrcFloatOp>(irb, scope, source_node);
3994		instruction->operand = operand;
3995		instruction->fn_id = fn_id;
3996
3997		ir_ref_instruction(operand, irb->current_basic_block);
3998
3999		return &instruction->base;
4000		}
4001
4002		static IrInstGen ir_build_float_op_gen(IrAnalyze ira, IrInst source_instr, IrInstGen operand,
4003		BuiltinFnId fn_id, ZigType *operand_type)
4004		{
4005		IrInstGenFloatOp *instruction = ir_build_inst_gen<IrInstGenFloatOp>(&ira->new_irb,
4006		source_instr->scope, source_instr->source_node);
4007		instruction->base.value->type = operand_type;
4008		instruction->operand = operand;
4009		instruction->fn_id = fn_id;
4010
4011		ir_ref_inst_gen(operand);
4012
4013		return &instruction->base;
4014		}
4015
4016		static IrInstSrc ir_build_mul_add_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4017		IrInstSrc type_value, IrInstSrc op1, IrInstSrc op2, IrInstSrc op3)
4018		{
4019		IrInstSrcMulAdd *instruction = ir_build_instruction<IrInstSrcMulAdd>(irb, scope, source_node);
4020		instruction->type_value = type_value;
4021		instruction->op1 = op1;
4022		instruction->op2 = op2;
4023		instruction->op3 = op3;
4024
4025		ir_ref_instruction(type_value, irb->current_basic_block);
4026		ir_ref_instruction(op1, irb->current_basic_block);
4027		ir_ref_instruction(op2, irb->current_basic_block);
4028		ir_ref_instruction(op3, irb->current_basic_block);
4029
4030		return &instruction->base;
4031		}
4032
4033		static IrInstGen ir_build_mul_add_gen(IrAnalyze ira, IrInst source_instr, IrInstGen op1, IrInstGen *op2,
4034		IrInstGen op3, ZigType expr_type)
4035		{
4036		IrInstGenMulAdd *instruction = ir_build_inst_gen<IrInstGenMulAdd>(&ira->new_irb,
4037		source_instr->scope, source_instr->source_node);
4038		instruction->base.value->type = expr_type;
4039		instruction->op1 = op1;
4040		instruction->op2 = op2;
4041		instruction->op3 = op3;
4042
4043		ir_ref_inst_gen(op1);
4044		ir_ref_inst_gen(op2);
4045		ir_ref_inst_gen(op3);
4046
4047		return &instruction->base;
4048		}
4049
4050		static IrInstSrc ir_build_align_of(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type_value) {
4051		IrInstSrcAlignOf *instruction = ir_build_instruction<IrInstSrcAlignOf>(irb, scope, source_node);
4052		instruction->type_value = type_value;
4053
4054		ir_ref_instruction(type_value, irb->current_basic_block);
4055
4056		return &instruction->base;
4057		}
4058
4059		static IrInstSrc ir_build_test_err_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4060		IrInstSrc *base_ptr, bool resolve_err_set, bool base_ptr_is_payload)
4061		{
4062		IrInstSrcTestErr *instruction = ir_build_instruction<IrInstSrcTestErr>(irb, scope, source_node);
4063		instruction->base_ptr = base_ptr;
4064		instruction->resolve_err_set = resolve_err_set;
4065		instruction->base_ptr_is_payload = base_ptr_is_payload;
4066
4067		ir_ref_instruction(base_ptr, irb->current_basic_block);
4068
4069		return &instruction->base;
4070		}
4071
4072		static IrInstGen ir_build_test_err_gen(IrAnalyze ira, IrInst source_instruction, IrInstGen err_union) {
4073		IrInstGenTestErr *instruction = ir_build_inst_gen<IrInstGenTestErr>(
4074		&ira->new_irb, source_instruction->scope, source_instruction->source_node);
4075		instruction->base.value->type = ira->codegen->builtin_types.entry_bool;
4076		instruction->err_union = err_union;
4077
4078		ir_ref_inst_gen(err_union);
4079
4080		return &instruction->base;
4081		}
4082
4083		static IrInstSrc ir_build_unwrap_err_code_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4084		IrInstSrc *err_union_ptr)
4085		{
4086		IrInstSrcUnwrapErrCode *inst = ir_build_instruction<IrInstSrcUnwrapErrCode>(irb, scope, source_node);
4087		inst->err_union_ptr = err_union_ptr;
4088
4089		ir_ref_instruction(err_union_ptr, irb->current_basic_block);
4090
4091		return &inst->base;
4092		}
4093
4094		static IrInstGen ir_build_unwrap_err_code_gen(IrAnalyze ira, Scope scope, AstNode source_node,
4095		IrInstGen err_union_ptr, ZigType result_type)
4096		{
4097		IrInstGenUnwrapErrCode *inst = ir_build_inst_gen<IrInstGenUnwrapErrCode>(&ira->new_irb, scope, source_node);
4098		inst->base.value->type = result_type;
4099		inst->err_union_ptr = err_union_ptr;
4100
4101		ir_ref_inst_gen(err_union_ptr);
4102
4103		return &inst->base;
4104		}
4105
4106		static IrInstSrc ir_build_unwrap_err_payload_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4107		IrInstSrc *value, bool safety_check_on, bool initializing)
4108		{
4109		IrInstSrcUnwrapErrPayload *inst = ir_build_instruction<IrInstSrcUnwrapErrPayload>(irb, scope, source_node);
4110		inst->value = value;
4111		inst->safety_check_on = safety_check_on;
4112		inst->initializing = initializing;
4113
4114		ir_ref_instruction(value, irb->current_basic_block);
4115
4116		return &inst->base;
4117		}
4118
4119		static IrInstGen ir_build_unwrap_err_payload_gen(IrAnalyze ira, Scope scope, AstNode source_node,
4120		IrInstGen value, bool safety_check_on, bool initializing, ZigType result_type)
4121		{
4122		IrInstGenUnwrapErrPayload *inst = ir_build_inst_gen<IrInstGenUnwrapErrPayload>(&ira->new_irb, scope, source_node);
4123		inst->base.value->type = result_type;
4124		inst->value = value;
4125		inst->safety_check_on = safety_check_on;
4126		inst->initializing = initializing;
4127
4128		ir_ref_inst_gen(value);
4129
4130		return &inst->base;
4131		}
4132
4133		static IrInstSrc ir_build_fn_proto(IrBuilderSrc irb, Scope scope, AstNode source_node,
4134		IrInstSrc *param_types, IrInstSrc align_value, IrInstSrc *callconv_value,
4135		IrInstSrc *return_type, bool is_var_args)
4136		{
4137		IrInstSrcFnProto *instruction = ir_build_instruction<IrInstSrcFnProto>(irb, scope, source_node);
4138		instruction->param_types = param_types;
4139		instruction->align_value = align_value;
4140		instruction->callconv_value = callconv_value;
4141		instruction->return_type = return_type;
4142		instruction->is_var_args = is_var_args;
4143
4144		assert(source_node->type == NodeTypeFnProto);
4145		size_t param_count = source_node->data.fn_proto.params.length;
4146		if (is_var_args) param_count -= 1;
4147		for (size_t i = 0; i < param_count; i += 1) {
4148		if (param_types[i] != nullptr) ir_ref_instruction(param_types[i], irb->current_basic_block);
4149		}
4150		if (align_value != nullptr) ir_ref_instruction(align_value, irb->current_basic_block);
4151		if (callconv_value != nullptr) ir_ref_instruction(callconv_value, irb->current_basic_block);
4152		ir_ref_instruction(return_type, irb->current_basic_block);
4153
4154		return &instruction->base;
4155		}
4156
4157		static IrInstSrc ir_build_test_comptime(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *value) {
4158		IrInstSrcTestComptime *instruction = ir_build_instruction<IrInstSrcTestComptime>(irb, scope, source_node);
4159		instruction->value = value;
4160
4161		ir_ref_instruction(value, irb->current_basic_block);
4162
4163		return &instruction->base;
4164		}
4165
4166		static IrInstSrc ir_build_ptr_cast_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4167		IrInstSrc dest_type, IrInstSrc ptr, bool safety_check_on)
4168		{
4169		IrInstSrcPtrCast *instruction = ir_build_instruction<IrInstSrcPtrCast>(
4170		irb, scope, source_node);
4171		instruction->dest_type = dest_type;
4172		instruction->ptr = ptr;
4173		instruction->safety_check_on = safety_check_on;
4174
4175		ir_ref_instruction(dest_type, irb->current_basic_block);
4176		ir_ref_instruction(ptr, irb->current_basic_block);
4177
4178		return &instruction->base;
4179		}
4180
4181		static IrInstGen ir_build_ptr_cast_gen(IrAnalyze ira, IrInst *source_instruction,
4182		ZigType ptr_type, IrInstGen ptr, bool safety_check_on)
4183		{
4184		IrInstGenPtrCast *instruction = ir_build_inst_gen<IrInstGenPtrCast>(
4185		&ira->new_irb, source_instruction->scope, source_instruction->source_node);
4186		instruction->base.value->type = ptr_type;
4187		instruction->ptr = ptr;
4188		instruction->safety_check_on = safety_check_on;
4189
4190		ir_ref_inst_gen(ptr);
4191
4192		return &instruction->base;
4193		}
4194
4195		static IrInstSrc ir_build_implicit_cast(IrBuilderSrc irb, Scope scope, AstNode source_node,
4196		IrInstSrc operand, ResultLocCast result_loc_cast)
4197		{
4198		IrInstSrcImplicitCast *instruction = ir_build_instruction<IrInstSrcImplicitCast>(irb, scope, source_node);
4199		instruction->operand = operand;
4200		instruction->result_loc_cast = result_loc_cast;
4201
4202		ir_ref_instruction(operand, irb->current_basic_block);
4203
4204		return &instruction->base;
4205		}
4206
4207		static IrInstSrc ir_build_bit_cast_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4208		IrInstSrc operand, ResultLocBitCast result_loc_bit_cast)
4209		{
4210		IrInstSrcBitCast *instruction = ir_build_instruction<IrInstSrcBitCast>(irb, scope, source_node);
4211		instruction->operand = operand;
4212		instruction->result_loc_bit_cast = result_loc_bit_cast;
4213
4214		ir_ref_instruction(operand, irb->current_basic_block);
4215
4216		return &instruction->base;
4217		}
4218
4219		static IrInstGen ir_build_bit_cast_gen(IrAnalyze ira, IrInst *source_instruction,
4220		IrInstGen operand, ZigType ty)
4221		{
4222		IrInstGenBitCast *instruction = ir_build_inst_gen<IrInstGenBitCast>(
4223		&ira->new_irb, source_instruction->scope, source_instruction->source_node);
4224		instruction->base.value->type = ty;
4225		instruction->operand = operand;
4226
4227		ir_ref_inst_gen(operand);
4228
4229		return &instruction->base;
4230		}
4231
4232		static IrInstGen ir_build_widen_or_shorten(IrAnalyze ira, Scope scope, AstNode source_node, IrInstGen *target,
4233		ZigType *result_type)
4234		{
4235		IrInstGenWidenOrShorten *inst = ir_build_inst_gen<IrInstGenWidenOrShorten>(&ira->new_irb, scope, source_node);
4236		inst->base.value->type = result_type;
4237		inst->target = target;
4238
4239		ir_ref_inst_gen(target);
4240
4241		return &inst->base;
4242		}
4243
4244		static IrInstSrc ir_build_int_to_ptr_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4245		IrInstSrc dest_type, IrInstSrc target)
4246		{
4247		IrInstSrcIntToPtr *instruction = ir_build_instruction<IrInstSrcIntToPtr>(irb, scope, source_node);
4248		instruction->dest_type = dest_type;
4249		instruction->target = target;
4250
4251		ir_ref_instruction(dest_type, irb->current_basic_block);
4252		ir_ref_instruction(target, irb->current_basic_block);
4253
4254		return &instruction->base;
4255		}
4256
4257		static IrInstGen ir_build_int_to_ptr_gen(IrAnalyze ira, Scope scope, AstNode source_node,
4258		IrInstGen target, ZigType ptr_type)
4259		{
4260		IrInstGenIntToPtr *instruction = ir_build_inst_gen<IrInstGenIntToPtr>(&ira->new_irb, scope, source_node);
4261		instruction->base.value->type = ptr_type;
4262		instruction->target = target;
4263
4264		ir_ref_inst_gen(target);
4265
4266		return &instruction->base;
4267		}
4268
4269		static IrInstSrc ir_build_ptr_to_int_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4270		IrInstSrc *target)
4271		{
4272		IrInstSrcPtrToInt *inst = ir_build_instruction<IrInstSrcPtrToInt>(irb, scope, source_node);
4273		inst->target = target;
4274
4275		ir_ref_instruction(target, irb->current_basic_block);
4276
4277		return &inst->base;
4278		}
4279
4280		static IrInstGen ir_build_ptr_to_int_gen(IrAnalyze ira, IrInst source_instr, IrInstGen target) {
4281		IrInstGenPtrToInt *inst = ir_build_inst_gen<IrInstGenPtrToInt>(&ira->new_irb, source_instr->scope, source_instr->source_node);
4282		inst->base.value->type = ira->codegen->builtin_types.entry_usize;
4283		inst->target = target;
4284
4285		ir_ref_inst_gen(target);
4286
4287		return &inst->base;
4288		}
4289
4290		static IrInstSrc ir_build_int_to_enum_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4291		IrInstSrc dest_type, IrInstSrc target)
4292		{
4293		IrInstSrcIntToEnum *instruction = ir_build_instruction<IrInstSrcIntToEnum>(irb, scope, source_node);
4294		instruction->dest_type = dest_type;
4295		instruction->target = target;
4296
4297		if (dest_type) ir_ref_instruction(dest_type, irb->current_basic_block);
4298		ir_ref_instruction(target, irb->current_basic_block);
4299
4300		return &instruction->base;
4301		}
4302
4303		static IrInstGen ir_build_int_to_enum_gen(IrAnalyze ira, Scope scope, AstNode source_node,
4304		ZigType dest_type, IrInstGen target)
4305		{
4306		IrInstGenIntToEnum *instruction = ir_build_inst_gen<IrInstGenIntToEnum>(&ira->new_irb, scope, source_node);
4307		instruction->base.value->type = dest_type;
4308		instruction->target = target;
4309
4310		ir_ref_inst_gen(target);
4311
4312		return &instruction->base;
4313		}
4314
4315		static IrInstSrc ir_build_enum_to_int(IrBuilderSrc irb, Scope scope, AstNode source_node,
4316		IrInstSrc *target)
4317		{
4318		IrInstSrcEnumToInt *instruction = ir_build_instruction<IrInstSrcEnumToInt>(
4319		irb, scope, source_node);
4320		instruction->target = target;
4321
4322		ir_ref_instruction(target, irb->current_basic_block);
4323
4324		return &instruction->base;
4325		}
4326
4327		static IrInstSrc ir_build_int_to_err_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4328		IrInstSrc *target)
4329		{
4330		IrInstSrcIntToErr *instruction = ir_build_instruction<IrInstSrcIntToErr>(irb, scope, source_node);
4331		instruction->target = target;
4332
4333		ir_ref_instruction(target, irb->current_basic_block);
4334
4335		return &instruction->base;
4336		}
4337
4338		static IrInstGen ir_build_int_to_err_gen(IrAnalyze ira, Scope scope, AstNode source_node, IrInstGen *target,
4339		ZigType *wanted_type)
4340		{
4341		IrInstGenIntToErr *instruction = ir_build_inst_gen<IrInstGenIntToErr>(&ira->new_irb, scope, source_node);
4342		instruction->base.value->type = wanted_type;
4343		instruction->target = target;
4344
4345		ir_ref_inst_gen(target);
4346
4347		return &instruction->base;
4348		}
4349
4350		static IrInstSrc ir_build_err_to_int_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4351		IrInstSrc *target)
4352		{
4353		IrInstSrcErrToInt *instruction = ir_build_instruction<IrInstSrcErrToInt>(
4354		irb, scope, source_node);
4355		instruction->target = target;
4356
4357		ir_ref_instruction(target, irb->current_basic_block);
4358
4359		return &instruction->base;
4360		}
4361
4362		static IrInstGen ir_build_err_to_int_gen(IrAnalyze ira, Scope scope, AstNode source_node, IrInstGen *target,
4363		ZigType *wanted_type)
4364		{
4365		IrInstGenErrToInt *instruction = ir_build_inst_gen<IrInstGenErrToInt>(&ira->new_irb, scope, source_node);
4366		instruction->base.value->type = wanted_type;
4367		instruction->target = target;
4368
4369		ir_ref_inst_gen(target);
4370
4371		return &instruction->base;
4372		}
4373
4374		static IrInstSrc ir_build_check_switch_prongs(IrBuilderSrc irb, Scope scope, AstNode source_node,
4375		IrInstSrc target_value, IrInstSrcCheckSwitchProngsRange ranges, size_t range_count,
4376		AstNode* else_prong, bool have_underscore_prong)
4377		{
4378		IrInstSrcCheckSwitchProngs *instruction = heap::c_allocator.create<IrInstSrcCheckSwitchProngs>();
4379		instruction->base.id = have_underscore_prong ?
4380		IrInstSrcIdCheckSwitchProngsUnderYes : IrInstSrcIdCheckSwitchProngsUnderNo;
4381		instruction->base.base.scope = scope;
4382		instruction->base.base.source_node = source_node;
4383		instruction->base.base.debug_id = exec_next_debug_id(irb->exec);
4384		instruction->base.owner_bb = irb->current_basic_block;
4385		ir_instruction_append(irb->current_basic_block, &instruction->base);
4386
4387		instruction->target_value = target_value;
4388		instruction->ranges = ranges;
4389		instruction->range_count = range_count;
4390		instruction->else_prong = else_prong;
4391
4392		ir_ref_instruction(target_value, irb->current_basic_block);
4393		for (size_t i = 0; i < range_count; i += 1) {
4394		ir_ref_instruction(ranges[i].start, irb->current_basic_block);
4395		ir_ref_instruction(ranges[i].end, irb->current_basic_block);
4396		}
4397
4398		return &instruction->base;
4399		}
4400
4401		static IrInstSrc ir_build_check_statement_is_void(IrBuilderSrc irb, Scope scope, AstNode source_node,
4402		IrInstSrc* statement_value)
4403		{
4404		IrInstSrcCheckStatementIsVoid *instruction = ir_build_instruction<IrInstSrcCheckStatementIsVoid>(
4405		irb, scope, source_node);
4406		instruction->statement_value = statement_value;
4407
4408		ir_ref_instruction(statement_value, irb->current_basic_block);
4409
4410		return &instruction->base;
4411		}
4412
4413		static IrInstSrc ir_build_type_name(IrBuilderSrc irb, Scope scope, AstNode source_node,
4414		IrInstSrc *type_value)
4415		{
4416		IrInstSrcTypeName *instruction = ir_build_instruction<IrInstSrcTypeName>(irb, scope, source_node);
4417		instruction->type_value = type_value;
4418
4419		ir_ref_instruction(type_value, irb->current_basic_block);
4420
4421		return &instruction->base;
4422		}
4423
4424		static IrInstSrc ir_build_decl_ref(IrBuilderSrc irb, Scope scope, AstNode source_node, Tld *tld, LVal lval) {
4425		IrInstSrcDeclRef *instruction = ir_build_instruction<IrInstSrcDeclRef>(irb, scope, source_node);
4426		instruction->tld = tld;
4427		instruction->lval = lval;
4428
4429		return &instruction->base;
4430		}
4431
4432		static IrInstSrc ir_build_panic_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *msg) {
4433		IrInstSrcPanic *instruction = ir_build_instruction<IrInstSrcPanic>(irb, scope, source_node);
4434		instruction->base.is_noreturn = true;
4435		instruction->msg = msg;
4436
4437		ir_ref_instruction(msg, irb->current_basic_block);
4438
4439		return &instruction->base;
4440		}
4441
4442		static IrInstGen ir_build_panic_gen(IrAnalyze ira, IrInst source_instr, IrInstGen msg) {
4443		IrInstGenPanic *instruction = ir_build_inst_noreturn<IrInstGenPanic>(&ira->new_irb,
4444		source_instr->scope, source_instr->source_node);
4445		instruction->msg = msg;
4446
4447		ir_ref_inst_gen(msg);
4448
4449		return &instruction->base;
4450		}
4451
4452		static IrInstSrc ir_build_tag_name_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *target) {
4453		IrInstSrcTagName *instruction = ir_build_instruction<IrInstSrcTagName>(irb, scope, source_node);
4454		instruction->target = target;
4455
4456		ir_ref_instruction(target, irb->current_basic_block);
4457
4458		return &instruction->base;
4459		}
4460
4461		static IrInstGen ir_build_tag_name_gen(IrAnalyze ira, IrInst source_instr, IrInstGen target,
4462		ZigType *result_type)
4463		{
4464		IrInstGenTagName *instruction = ir_build_inst_gen<IrInstGenTagName>(&ira->new_irb,
4465		source_instr->scope, source_instr->source_node);
4466		instruction->base.value->type = result_type;
4467		instruction->target = target;
4468
4469		ir_ref_inst_gen(target);
4470
4471		return &instruction->base;
4472		}
4473
4474		static IrInstSrc ir_build_field_parent_ptr_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4475		IrInstSrc type_value, IrInstSrc field_name, IrInstSrc *field_ptr)
4476		{
4477		IrInstSrcFieldParentPtr *inst = ir_build_instruction<IrInstSrcFieldParentPtr>(
4478		irb, scope, source_node);
4479		inst->type_value = type_value;
4480		inst->field_name = field_name;
4481		inst->field_ptr = field_ptr;
4482
4483		ir_ref_instruction(type_value, irb->current_basic_block);
4484		ir_ref_instruction(field_name, irb->current_basic_block);
4485		ir_ref_instruction(field_ptr, irb->current_basic_block);
4486
4487		return &inst->base;
4488		}
4489
4490		static IrInstGen ir_build_field_parent_ptr_gen(IrAnalyze ira, IrInst *source_instr,
4491		IrInstGen field_ptr, TypeStructField field, ZigType *result_type)
4492		{
4493		IrInstGenFieldParentPtr *inst = ir_build_inst_gen<IrInstGenFieldParentPtr>(&ira->new_irb,
4494		source_instr->scope, source_instr->source_node);
4495		inst->base.value->type = result_type;
4496		inst->field_ptr = field_ptr;
4497		inst->field = field;
4498
4499		ir_ref_inst_gen(field_ptr);
4500
4501		return &inst->base;
4502		}
4503
4504		static IrInstSrc ir_build_byte_offset_of(IrBuilderSrc irb, Scope scope, AstNode source_node,
4505		IrInstSrc type_value, IrInstSrc field_name)
4506		{
4507		IrInstSrcByteOffsetOf *instruction = ir_build_instruction<IrInstSrcByteOffsetOf>(irb, scope, source_node);
4508		instruction->type_value = type_value;
4509		instruction->field_name = field_name;
4510
4511		ir_ref_instruction(type_value, irb->current_basic_block);
4512		ir_ref_instruction(field_name, irb->current_basic_block);
4513
4514		return &instruction->base;
4515		}
4516
4517		static IrInstSrc ir_build_bit_offset_of(IrBuilderSrc irb, Scope scope, AstNode source_node,
4518		IrInstSrc type_value, IrInstSrc field_name)
4519		{
4520		IrInstSrcBitOffsetOf *instruction = ir_build_instruction<IrInstSrcBitOffsetOf>(irb, scope, source_node);
4521		instruction->type_value = type_value;
4522		instruction->field_name = field_name;
4523
4524		ir_ref_instruction(type_value, irb->current_basic_block);
4525		ir_ref_instruction(field_name, irb->current_basic_block);
4526
4527		return &instruction->base;
4528		}
4529
4530		static IrInstSrc ir_build_type_info(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type_value) {
4531		IrInstSrcTypeInfo *instruction = ir_build_instruction<IrInstSrcTypeInfo>(irb, scope, source_node);
4532		instruction->type_value = type_value;
4533
4534		ir_ref_instruction(type_value, irb->current_basic_block);
4535
4536		return &instruction->base;
4537		}
4538
4539		static IrInstSrc ir_build_type(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *type_info) {
4540		IrInstSrcType *instruction = ir_build_instruction<IrInstSrcType>(irb, scope, source_node);
4541		instruction->type_info = type_info;
4542
4543		ir_ref_instruction(type_info, irb->current_basic_block);
4544
4545		return &instruction->base;
4546		}
4547
4548		static IrInstSrc ir_build_set_eval_branch_quota(IrBuilderSrc irb, Scope scope, AstNode source_node,
4549		IrInstSrc *new_quota)
4550		{
4551		IrInstSrcSetEvalBranchQuota *instruction = ir_build_instruction<IrInstSrcSetEvalBranchQuota>(irb, scope, source_node);
4552		instruction->new_quota = new_quota;
4553
4554		ir_ref_instruction(new_quota, irb->current_basic_block);
4555
4556		return &instruction->base;
4557		}
4558
4559		static IrInstSrc ir_build_align_cast_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4560		IrInstSrc align_bytes, IrInstSrc target)
4561		{
4562		IrInstSrcAlignCast *instruction = ir_build_instruction<IrInstSrcAlignCast>(irb, scope, source_node);
4563		instruction->align_bytes = align_bytes;
4564		instruction->target = target;
4565
4566		ir_ref_instruction(align_bytes, irb->current_basic_block);
4567		ir_ref_instruction(target, irb->current_basic_block);
4568
4569		return &instruction->base;
4570		}
4571
4572		static IrInstGen ir_build_align_cast_gen(IrAnalyze ira, Scope scope, AstNode source_node, IrInstGen *target,
4573		ZigType *result_type)
4574		{
4575		IrInstGenAlignCast *instruction = ir_build_inst_gen<IrInstGenAlignCast>(&ira->new_irb, scope, source_node);
4576		instruction->base.value->type = result_type;
4577		instruction->target = target;
4578
4579		ir_ref_inst_gen(target);
4580
4581		return &instruction->base;
4582		}
4583
4584		static IrInstSrc ir_build_resolve_result(IrBuilderSrc irb, Scope scope, AstNode source_node,
4585		ResultLoc result_loc, IrInstSrc ty)
4586		{
4587		IrInstSrcResolveResult *instruction = ir_build_instruction<IrInstSrcResolveResult>(irb, scope, source_node);
4588		instruction->result_loc = result_loc;
4589		instruction->ty = ty;
4590
4591		if (ty != nullptr) ir_ref_instruction(ty, irb->current_basic_block);
4592
4593		return &instruction->base;
4594		}
4595
4596		static IrInstSrc ir_build_reset_result(IrBuilderSrc irb, Scope scope, AstNode source_node,
4597		ResultLoc *result_loc)
4598		{
4599		IrInstSrcResetResult *instruction = ir_build_instruction<IrInstSrcResetResult>(irb, scope, source_node);
4600		instruction->result_loc = result_loc;
4601		instruction->base.is_gen = true;
4602
4603		return &instruction->base;
4604		}
4605
4606		static IrInstSrc ir_build_set_align_stack(IrBuilderSrc irb, Scope scope, AstNode source_node,
4607		IrInstSrc *align_bytes)
4608		{
4609		IrInstSrcSetAlignStack *instruction = ir_build_instruction<IrInstSrcSetAlignStack>(irb, scope, source_node);
4610		instruction->align_bytes = align_bytes;
4611
4612		ir_ref_instruction(align_bytes, irb->current_basic_block);
4613
4614		return &instruction->base;
4615		}
4616
4617		static IrInstSrc ir_build_arg_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
4618		IrInstSrc fn_type, IrInstSrc arg_index, bool allow_var)
4619		{
4620		IrInstSrcArgType *instruction = heap::c_allocator.create<IrInstSrcArgType>();
4621		instruction->base.id = allow_var ?
4622		IrInstSrcIdArgTypeAllowVarTrue : IrInstSrcIdArgTypeAllowVarFalse;
4623		instruction->base.base.scope = scope;
4624		instruction->base.base.source_node = source_node;
4625		instruction->base.base.debug_id = exec_next_debug_id(irb->exec);
4626		instruction->base.owner_bb = irb->current_basic_block;
4627		ir_instruction_append(irb->current_basic_block, &instruction->base);
4628
4629		instruction->fn_type = fn_type;
4630		instruction->arg_index = arg_index;
4631
4632		ir_ref_instruction(fn_type, irb->current_basic_block);
4633		ir_ref_instruction(arg_index, irb->current_basic_block);
4634
4635		return &instruction->base;
4636		}
4637
4638		static IrInstSrc ir_build_error_return_trace_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4639		IrInstErrorReturnTraceOptional optional)
4640		{
4641		IrInstSrcErrorReturnTrace *inst = ir_build_instruction<IrInstSrcErrorReturnTrace>(irb, scope, source_node);
4642		inst->optional = optional;
4643
4644		return &inst->base;
4645		}
4646
4647		static IrInstGen ir_build_error_return_trace_gen(IrAnalyze ira, Scope scope, AstNode source_node,
4648		IrInstErrorReturnTraceOptional optional, ZigType *result_type)
4649		{
4650		IrInstGenErrorReturnTrace *inst = ir_build_inst_gen<IrInstGenErrorReturnTrace>(&ira->new_irb, scope, source_node);
4651		inst->base.value->type = result_type;
4652		inst->optional = optional;
4653
4654		return &inst->base;
4655		}
4656
4657		static IrInstSrc ir_build_error_union(IrBuilderSrc irb, Scope scope, AstNode source_node,
4658		IrInstSrc err_set, IrInstSrc payload)
4659		{
4660		IrInstSrcErrorUnion *instruction = ir_build_instruction<IrInstSrcErrorUnion>(irb, scope, source_node);
4661		instruction->err_set = err_set;
4662		instruction->payload = payload;
4663
4664		ir_ref_instruction(err_set, irb->current_basic_block);
4665		ir_ref_instruction(payload, irb->current_basic_block);
4666
4667		return &instruction->base;
4668		}
4669
4670		static IrInstSrc ir_build_atomic_rmw_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4671		IrInstSrc operand_type, IrInstSrc ptr, IrInstSrc op, IrInstSrc operand,
4672		IrInstSrc *ordering)
4673		{
4674		IrInstSrcAtomicRmw *instruction = ir_build_instruction<IrInstSrcAtomicRmw>(irb, scope, source_node);
4675		instruction->operand_type = operand_type;
4676		instruction->ptr = ptr;
4677		instruction->op = op;
4678		instruction->operand = operand;
4679		instruction->ordering = ordering;
4680
4681		ir_ref_instruction(operand_type, irb->current_basic_block);
4682		ir_ref_instruction(ptr, irb->current_basic_block);
4683		ir_ref_instruction(op, irb->current_basic_block);
4684		ir_ref_instruction(operand, irb->current_basic_block);
4685		ir_ref_instruction(ordering, irb->current_basic_block);
4686
4687		return &instruction->base;
4688		}
4689
4690		static IrInstGen ir_build_atomic_rmw_gen(IrAnalyze ira, IrInst *source_instr,
4691		IrInstGen ptr, IrInstGen operand, AtomicRmwOp op, AtomicOrder ordering, ZigType *operand_type)
4692		{
4693		IrInstGenAtomicRmw *instruction = ir_build_inst_gen<IrInstGenAtomicRmw>(&ira->new_irb, source_instr->scope, source_instr->source_node);
4694		instruction->base.value->type = operand_type;
4695		instruction->ptr = ptr;
4696		instruction->op = op;
4697		instruction->operand = operand;
4698		instruction->ordering = ordering;
4699
4700		ir_ref_inst_gen(ptr);
4701		ir_ref_inst_gen(operand);
4702
4703		return &instruction->base;
4704		}
4705
4706		static IrInstSrc ir_build_atomic_load_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4707		IrInstSrc operand_type, IrInstSrc ptr, IrInstSrc *ordering)
4708		{
4709		IrInstSrcAtomicLoad *instruction = ir_build_instruction<IrInstSrcAtomicLoad>(irb, scope, source_node);
4710		instruction->operand_type = operand_type;
4711		instruction->ptr = ptr;
4712		instruction->ordering = ordering;
4713
4714		ir_ref_instruction(operand_type, irb->current_basic_block);
4715		ir_ref_instruction(ptr, irb->current_basic_block);
4716		ir_ref_instruction(ordering, irb->current_basic_block);
4717
4718		return &instruction->base;
4719		}
4720
4721		static IrInstGen ir_build_atomic_load_gen(IrAnalyze ira, IrInst *source_instr,
4722		IrInstGen ptr, AtomicOrder ordering, ZigType operand_type)
4723		{
4724		IrInstGenAtomicLoad *instruction = ir_build_inst_gen<IrInstGenAtomicLoad>(&ira->new_irb,
4725		source_instr->scope, source_instr->source_node);
4726		instruction->base.value->type = operand_type;
4727		instruction->ptr = ptr;
4728		instruction->ordering = ordering;
4729
4730		ir_ref_inst_gen(ptr);
4731
4732		return &instruction->base;
4733		}
4734
4735		static IrInstSrc ir_build_atomic_store_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4736		IrInstSrc operand_type, IrInstSrc ptr, IrInstSrc value, IrInstSrc ordering)
4737		{
4738		IrInstSrcAtomicStore *instruction = ir_build_instruction<IrInstSrcAtomicStore>(irb, scope, source_node);
4739		instruction->operand_type = operand_type;
4740		instruction->ptr = ptr;
4741		instruction->value = value;
4742		instruction->ordering = ordering;
4743
4744		ir_ref_instruction(operand_type, irb->current_basic_block);
4745		ir_ref_instruction(ptr, irb->current_basic_block);
4746		ir_ref_instruction(value, irb->current_basic_block);
4747		ir_ref_instruction(ordering, irb->current_basic_block);
4748
4749		return &instruction->base;
4750		}
4751
4752		static IrInstGen ir_build_atomic_store_gen(IrAnalyze ira, IrInst *source_instr,
4753		IrInstGen ptr, IrInstGen value, AtomicOrder ordering)
4754		{
4755		IrInstGenAtomicStore *instruction = ir_build_inst_void<IrInstGenAtomicStore>(&ira->new_irb,
4756		source_instr->scope, source_instr->source_node);
4757		instruction->ptr = ptr;
4758		instruction->value = value;
4759		instruction->ordering = ordering;
4760
4761		ir_ref_inst_gen(ptr);
4762		ir_ref_inst_gen(value);
4763
4764		return &instruction->base;
4765		}
4766
4767		static IrInstSrc ir_build_save_err_ret_addr_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
4768		IrInstSrcSaveErrRetAddr *inst = ir_build_instruction<IrInstSrcSaveErrRetAddr>(irb, scope, source_node);
4769		return &inst->base;
4770		}
4771
4772		static IrInstGen ir_build_save_err_ret_addr_gen(IrAnalyze ira, IrInst *source_instr) {
4773		IrInstGenSaveErrRetAddr *inst = ir_build_inst_void<IrInstGenSaveErrRetAddr>(&ira->new_irb,
4774		source_instr->scope, source_instr->source_node);
4775		return &inst->base;
4776		}
4777
4778		static IrInstSrc ir_build_add_implicit_return_type(IrBuilderSrc irb, Scope scope, AstNode source_node,
4779		IrInstSrc value, ResultLocReturn result_loc_ret)
4780		{
4781		IrInstSrcAddImplicitReturnType *inst = ir_build_instruction<IrInstSrcAddImplicitReturnType>(irb, scope, source_node);
4782		inst->value = value;
4783		inst->result_loc_ret = result_loc_ret;
4784
4785		ir_ref_instruction(value, irb->current_basic_block);
4786
4787		return &inst->base;
4788		}
4789
4790		static IrInstSrc ir_build_has_decl(IrBuilderSrc irb, Scope scope, AstNode source_node,
4791		IrInstSrc container, IrInstSrc name)
4792		{
4793		IrInstSrcHasDecl *instruction = ir_build_instruction<IrInstSrcHasDecl>(irb, scope, source_node);
4794		instruction->container = container;
4795		instruction->name = name;
4796
4797		ir_ref_instruction(container, irb->current_basic_block);
4798		ir_ref_instruction(name, irb->current_basic_block);
4799
4800		return &instruction->base;
4801		}
4802
4803		static IrInstSrc ir_build_undeclared_identifier(IrBuilderSrc irb, Scope scope, AstNode source_node, Buf *name) {
4804		IrInstSrcUndeclaredIdent *instruction = ir_build_instruction<IrInstSrcUndeclaredIdent>(irb, scope, source_node);
4805		instruction->name = name;
4806
4807		return &instruction->base;
4808		}
4809
4810		static IrInstSrc ir_build_check_runtime_scope(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc scope_is_comptime, IrInstSrc is_comptime) {
4811		IrInstSrcCheckRuntimeScope *instruction = ir_build_instruction<IrInstSrcCheckRuntimeScope>(irb, scope, source_node);
4812		instruction->scope_is_comptime = scope_is_comptime;
4813		instruction->is_comptime = is_comptime;
4814
4815		ir_ref_instruction(scope_is_comptime, irb->current_basic_block);
4816		ir_ref_instruction(is_comptime, irb->current_basic_block);
4817
4818		return &instruction->base;
4819		}
4820
4821		static IrInstSrc ir_build_union_init_named_field(IrBuilderSrc irb, Scope scope, AstNode source_node,
4822		IrInstSrc union_type, IrInstSrc field_name, IrInstSrc field_result_loc, IrInstSrc result_loc)
4823		{
4824		IrInstSrcUnionInitNamedField *instruction = ir_build_instruction<IrInstSrcUnionInitNamedField>(irb, scope, source_node);
4825		instruction->union_type = union_type;
4826		instruction->field_name = field_name;
4827		instruction->field_result_loc = field_result_loc;
4828		instruction->result_loc = result_loc;
4829
4830		ir_ref_instruction(union_type, irb->current_basic_block);
4831		ir_ref_instruction(field_name, irb->current_basic_block);
4832		ir_ref_instruction(field_result_loc, irb->current_basic_block);
4833		if (result_loc != nullptr) ir_ref_instruction(result_loc, irb->current_basic_block);
4834
4835		return &instruction->base;
4836		}
4837
4838
4839		static IrInstGen ir_build_vector_to_array(IrAnalyze ira, IrInst *source_instruction,
4840		ZigType result_type, IrInstGen vector, IrInstGen *result_loc)
4841		{
4842		IrInstGenVectorToArray *instruction = ir_build_inst_gen<IrInstGenVectorToArray>(&ira->new_irb,
4843		source_instruction->scope, source_instruction->source_node);
4844		instruction->base.value->type = result_type;
4845		instruction->vector = vector;
4846		instruction->result_loc = result_loc;
4847
4848		ir_ref_inst_gen(vector);
4849		ir_ref_inst_gen(result_loc);
4850
4851		return &instruction->base;
4852		}
4853
4854		static IrInstGen ir_build_ptr_of_array_to_slice(IrAnalyze ira, IrInst *source_instruction,
4855		ZigType result_type, IrInstGen operand, IrInstGen *result_loc)
4856		{
4857		IrInstGenPtrOfArrayToSlice *instruction = ir_build_inst_gen<IrInstGenPtrOfArrayToSlice>(&ira->new_irb,
4858		source_instruction->scope, source_instruction->source_node);
4859		instruction->base.value->type = result_type;
4860		instruction->operand = operand;
4861		instruction->result_loc = result_loc;
4862
4863		ir_ref_inst_gen(operand);
4864		ir_ref_inst_gen(result_loc);
4865
4866		return &instruction->base;
4867		}
4868
4869		static IrInstGen ir_build_array_to_vector(IrAnalyze ira, IrInst *source_instruction,
4870		IrInstGen array, ZigType result_type)
4871		{
4872		IrInstGenArrayToVector *instruction = ir_build_inst_gen<IrInstGenArrayToVector>(&ira->new_irb,
4873		source_instruction->scope, source_instruction->source_node);
4874		instruction->base.value->type = result_type;
4875		instruction->array = array;
4876
4877		ir_ref_inst_gen(array);
4878
4879		return &instruction->base;
4880		}
4881
4882		static IrInstGen ir_build_assert_zero(IrAnalyze ira, IrInst *source_instruction,
4883		IrInstGen *target)
4884		{
4885		IrInstGenAssertZero *instruction = ir_build_inst_gen<IrInstGenAssertZero>(&ira->new_irb,
4886		source_instruction->scope, source_instruction->source_node);
4887		instruction->base.value->type = ira->codegen->builtin_types.entry_void;
4888		instruction->target = target;
4889
4890		ir_ref_inst_gen(target);
4891
4892		return &instruction->base;
4893		}
4894
4895		static IrInstGen ir_build_assert_non_null(IrAnalyze ira, IrInst *source_instruction,
4896		IrInstGen *target)
4897		{
4898		IrInstGenAssertNonNull *instruction = ir_build_inst_gen<IrInstGenAssertNonNull>(&ira->new_irb,
4899		source_instruction->scope, source_instruction->source_node);
4900		instruction->base.value->type = ira->codegen->builtin_types.entry_void;
4901		instruction->target = target;
4902
4903		ir_ref_inst_gen(target);
4904
4905		return &instruction->base;
4906		}
4907
4908		static IrInstSrc ir_build_alloca_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4909		IrInstSrc align, const char name_hint, IrInstSrc *is_comptime)
4910		{
4911		IrInstSrcAlloca *instruction = ir_build_instruction<IrInstSrcAlloca>(irb, scope, source_node);
4912		instruction->base.is_gen = true;
4913		instruction->align = align;
4914		instruction->name_hint = name_hint;
4915		instruction->is_comptime = is_comptime;
4916
4917		if (align != nullptr) ir_ref_instruction(align, irb->current_basic_block);
4918		if (is_comptime != nullptr) ir_ref_instruction(is_comptime, irb->current_basic_block);
4919
4920		return &instruction->base;
4921		}
4922
4923		static IrInstGenAlloca ir_build_alloca_gen(IrAnalyze ira, IrInst *source_instruction,
4924		uint32_t align, const char *name_hint)
4925		{
4926		IrInstGenAlloca *instruction = ir_create_inst_gen<IrInstGenAlloca>(&ira->new_irb,
4927		source_instruction->scope, source_instruction->source_node);
4928		instruction->align = align;
4929		instruction->name_hint = name_hint;
4930
4931		return instruction;
4932		}
4933
4934		static IrInstSrc ir_build_end_expr(IrBuilderSrc irb, Scope scope, AstNode source_node,
4935		IrInstSrc value, ResultLoc result_loc)
4936		{
4937		IrInstSrcEndExpr *instruction = ir_build_instruction<IrInstSrcEndExpr>(irb, scope, source_node);
4938		instruction->base.is_gen = true;
4939		instruction->value = value;
4940		instruction->result_loc = result_loc;
4941
4942		ir_ref_instruction(value, irb->current_basic_block);
4943
4944		return &instruction->base;
4945		}
4946
4947		static IrInstSrcSuspendBegin ir_build_suspend_begin_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
4948		return ir_build_instruction<IrInstSrcSuspendBegin>(irb, scope, source_node);
4949		}
4950
4951		static IrInstGen ir_build_suspend_begin_gen(IrAnalyze ira, IrInst *source_instr) {
4952		IrInstGenSuspendBegin *inst = ir_build_inst_void<IrInstGenSuspendBegin>(&ira->new_irb,
4953		source_instr->scope, source_instr->source_node);
4954		return &inst->base;
4955		}
4956
4957		static IrInstSrc ir_build_suspend_finish_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4958		IrInstSrcSuspendBegin *begin)
4959		{
4960		IrInstSrcSuspendFinish *inst = ir_build_instruction<IrInstSrcSuspendFinish>(irb, scope, source_node);
4961		inst->begin = begin;
4962
4963		ir_ref_instruction(&begin->base, irb->current_basic_block);
4964
4965		return &inst->base;
4966		}
4967
4968		static IrInstGen ir_build_suspend_finish_gen(IrAnalyze ira, IrInst source_instr, IrInstGenSuspendBegin begin) {
4969		IrInstGenSuspendFinish *inst = ir_build_inst_void<IrInstGenSuspendFinish>(&ira->new_irb,
4970		source_instr->scope, source_instr->source_node);
4971		inst->begin = begin;
4972
4973		ir_ref_inst_gen(&begin->base);
4974
4975		return &inst->base;
4976		}
4977
4978		static IrInstSrc ir_build_await_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
4979		IrInstSrc frame, ResultLoc result_loc, bool is_nosuspend)
4980		{
4981		IrInstSrcAwait *instruction = ir_build_instruction<IrInstSrcAwait>(irb, scope, source_node);
4982		instruction->frame = frame;
4983		instruction->result_loc = result_loc;
4984		instruction->is_nosuspend = is_nosuspend;
4985
4986		ir_ref_instruction(frame, irb->current_basic_block);
4987
4988		return &instruction->base;
4989		}
4990
4991		static IrInstGenAwait ir_build_await_gen(IrAnalyze ira, IrInst *source_instruction,
4992		IrInstGen frame, ZigType result_type, IrInstGen *result_loc, bool is_nosuspend)
4993		{
4994		IrInstGenAwait *instruction = ir_build_inst_gen<IrInstGenAwait>(&ira->new_irb,
4995		source_instruction->scope, source_instruction->source_node);
4996		instruction->base.value->type = result_type;
4997		instruction->frame = frame;
4998		instruction->result_loc = result_loc;
4999		instruction->is_nosuspend = is_nosuspend;
5000
5001		ir_ref_inst_gen(frame);
5002		if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
5003
5004		return instruction;
5005		}
5006
5007		static IrInstSrc ir_build_resume_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *frame) {
5008		IrInstSrcResume *instruction = ir_build_instruction<IrInstSrcResume>(irb, scope, source_node);
5009		instruction->frame = frame;
5010
5011		ir_ref_instruction(frame, irb->current_basic_block);
5012
5013		return &instruction->base;
5014		}
5015
5016		static IrInstGen ir_build_resume_gen(IrAnalyze ira, IrInst source_instr, IrInstGen frame) {
5017		IrInstGenResume *instruction = ir_build_inst_void<IrInstGenResume>(&ira->new_irb,
5018		source_instr->scope, source_instr->source_node);
5019		instruction->frame = frame;
5020
5021		ir_ref_inst_gen(frame);
5022
5023		return &instruction->base;
5024		}
5025
5026		static IrInstSrcSpillBegin ir_build_spill_begin_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
5027		IrInstSrc *operand, SpillId spill_id)
5028		{
5029		IrInstSrcSpillBegin *instruction = ir_build_instruction<IrInstSrcSpillBegin>(irb, scope, source_node);
5030		instruction->operand = operand;
5031		instruction->spill_id = spill_id;
5032
5033		ir_ref_instruction(operand, irb->current_basic_block);
5034
5035		return instruction;
5036		}
5037
5038		static IrInstGen ir_build_spill_begin_gen(IrAnalyze ira, IrInst source_instr, IrInstGen operand,
5039		SpillId spill_id)
5040		{
5041		IrInstGenSpillBegin *instruction = ir_build_inst_void<IrInstGenSpillBegin>(&ira->new_irb,
5042		source_instr->scope, source_instr->source_node);
5043		instruction->operand = operand;
5044		instruction->spill_id = spill_id;
5045
5046		ir_ref_inst_gen(operand);
5047
5048		return &instruction->base;
5049		}
5050
5051		static IrInstSrc ir_build_spill_end_src(IrBuilderSrc irb, Scope scope, AstNode source_node,
5052		IrInstSrcSpillBegin *begin)
5053		{
5054		IrInstSrcSpillEnd *instruction = ir_build_instruction<IrInstSrcSpillEnd>(irb, scope, source_node);
5055		instruction->begin = begin;
5056
5057		ir_ref_instruction(&begin->base, irb->current_basic_block);
5058
5059		return &instruction->base;
5060		}
5061
5062		static IrInstGen ir_build_spill_end_gen(IrAnalyze ira, IrInst source_instr, IrInstGenSpillBegin begin,
5063		ZigType *result_type)
5064		{
5065		IrInstGenSpillEnd *instruction = ir_build_inst_gen<IrInstGenSpillEnd>(&ira->new_irb,
5066		source_instr->scope, source_instr->source_node);
5067		instruction->base.value->type = result_type;
5068		instruction->begin = begin;
5069
5070		ir_ref_inst_gen(&begin->base);
5071
5072		return &instruction->base;
5073		}
5074
5075		static IrInstGen ir_build_vector_extract_elem(IrAnalyze ira, IrInst *source_instruction,
5076		IrInstGen vector, IrInstGen index)
5077		{
5078		IrInstGenVectorExtractElem *instruction = ir_build_inst_gen<IrInstGenVectorExtractElem>(
5079		&ira->new_irb, source_instruction->scope, source_instruction->source_node);
5080		instruction->base.value->type = vector->value->type->data.vector.elem_type;
5081		instruction->vector = vector;
5082		instruction->index = index;
5083
5084		ir_ref_inst_gen(vector);
5085		ir_ref_inst_gen(index);
5086
5087		return &instruction->base;
5088		}
5089
5090		static IrInstSrc ir_build_wasm_memory_size_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc *index) {
5091		IrInstSrcWasmMemorySize *instruction = ir_build_instruction<IrInstSrcWasmMemorySize>(irb, scope, source_node);
5092		instruction->index = index;
5093
5094		ir_ref_instruction(index, irb->current_basic_block);
5095
5096		return &instruction->base;
5097		}
5098
5099		static IrInstGen ir_build_wasm_memory_size_gen(IrAnalyze ira, IrInst source_instr, IrInstGen index) {
5100		IrInstGenWasmMemorySize *instruction = ir_build_inst_gen<IrInstGenWasmMemorySize>(&ira->new_irb,
5101		source_instr->scope, source_instr->source_node);
5102		instruction->base.value->type = ira->codegen->builtin_types.entry_u32;
5103		instruction->index = index;
5104
5105		ir_ref_inst_gen(index);
5106
5107		return &instruction->base;
5108		}
5109
5110		static IrInstSrc ir_build_wasm_memory_grow_src(IrBuilderSrc irb, Scope scope, AstNode source_node, IrInstSrc index, IrInstSrc delta) {
5111		IrInstSrcWasmMemoryGrow *instruction = ir_build_instruction<IrInstSrcWasmMemoryGrow>(irb, scope, source_node);
5112		instruction->index = index;
5113		instruction->delta = delta;
5114
5115		ir_ref_instruction(index, irb->current_basic_block);
5116		ir_ref_instruction(delta, irb->current_basic_block);
5117
5118		return &instruction->base;
5119		}
5120
5121		static IrInstGen ir_build_wasm_memory_grow_gen(IrAnalyze ira, IrInst source_instr, IrInstGen index, IrInstGen *delta) {
5122		IrInstGenWasmMemoryGrow *instruction = ir_build_inst_gen<IrInstGenWasmMemoryGrow>(&ira->new_irb,
5123		source_instr->scope, source_instr->source_node);
5124		instruction->base.value->type = ira->codegen->builtin_types.entry_i32;
5125		instruction->index = index;
5126		instruction->delta = delta;
5127
5128		ir_ref_inst_gen(index);
5129		ir_ref_inst_gen(delta);
5130
5131		return &instruction->base;
5132		}
5133
5134		static IrInstSrc ir_build_src(IrBuilderSrc irb, Scope scope, AstNode source_node) {
5135		IrInstSrcSrc *instruction = ir_build_instruction<IrInstSrcSrc>(irb, scope, source_node);
5136
5137		return &instruction->base;
5138		}
5139
5140		static void ir_count_defers(IrBuilderSrc irb, Scope inner_scope, Scope outer_scope, size_t results) {
5141		results[ReturnKindUnconditional] = 0;
5142		results[ReturnKindError] = 0;
5143
5144		Scope *scope = inner_scope;
5145
5146		while (scope != outer_scope) {
5147		assert(scope);
5148		switch (scope->id) {
5149		case ScopeIdDefer: {
5150		AstNode *defer_node = scope->source_node;
5151		assert(defer_node->type == NodeTypeDefer);
5152		ReturnKind defer_kind = defer_node->data.defer.kind;
5153		results[defer_kind] += 1;
5154		scope = scope->parent;
5155		continue;
5156		}
5157		case ScopeIdDecls:
5158		case ScopeIdFnDef:
5159		return;
5160		case ScopeIdBlock:
5161		case ScopeIdVarDecl:
5162		case ScopeIdLoop:
5163		case ScopeIdSuspend:
5164		case ScopeIdCompTime:
5165		case ScopeIdNoSuspend:
5166		case ScopeIdRuntime:
5167		case ScopeIdTypeOf:
5168		case ScopeIdExpr:
5169		scope = scope->parent;
5170		continue;
5171		case ScopeIdDeferExpr:
5172		case ScopeIdCImport:
5173		zig_unreachable();
5174		}
5175		}
5176		}
5177
5178		static IrInstSrc ir_mark_gen(IrInstSrc instruction) {
5179		instruction->is_gen = true;
5180		return instruction;
5181		}
5182
5183		static bool ir_gen_defers_for_block(IrBuilderSrc irb, Scope inner_scope, Scope outer_scope, bool is_noreturn, IrInstSrc *err_value) {
5184		Scope *scope = inner_scope;
5185		if (is_noreturn != nullptr) *is_noreturn = false;
5186		while (scope != outer_scope) {
5187		if (!scope)
5188		return true;
5189
5190		switch (scope->id) {
5191		case ScopeIdDefer: {
5192		AstNode *defer_node = scope->source_node;
5193		assert(defer_node->type == NodeTypeDefer);
5194		ReturnKind defer_kind = defer_node->data.defer.kind;
5195		AstNode *defer_expr_node = defer_node->data.defer.expr;
5196		AstNode *defer_var_node = defer_node->data.defer.err_payload;
5197
5198		if (defer_kind == ReturnKindError && err_value == nullptr) {
5199		// This is an `errdefer` but we're generating code for a
5200		// `return` that doesn't return an error, skip it
5201		scope = scope->parent;
5202		continue;
5203		}
5204
5205		Scope *defer_expr_scope = defer_node->data.defer.expr_scope;
5206		if (defer_var_node != nullptr) {
5207		assert(defer_kind == ReturnKindError);
5208		assert(defer_var_node->type == NodeTypeSymbol);
5209		Buf *var_name = defer_var_node->data.symbol_expr.symbol;
5210
5211		if (defer_expr_node->type == NodeTypeUnreachable) {
5212		add_node_error(irb->codegen, defer_var_node,
5213		buf_sprintf("unused variable: '%s'", buf_ptr(var_name)));
5214		return false;
5215		}
5216
5217		IrInstSrc *is_comptime;
5218		if (ir_should_inline(irb->exec, defer_expr_scope)) {
5219		is_comptime = ir_build_const_bool(irb, defer_expr_scope,
5220		defer_expr_node, true);
5221		} else {
5222		is_comptime = ir_build_test_comptime(irb, defer_expr_scope,
5223		defer_expr_node, err_value);
5224		}
5225
5226		ZigVar *err_var = ir_create_var(irb, defer_var_node, defer_expr_scope,
5227		var_name, true, true, false, is_comptime);
5228		build_decl_var_and_init(irb, defer_expr_scope, defer_var_node, err_var, err_value,
5229		buf_ptr(var_name), is_comptime);
5230
5231		defer_expr_scope = err_var->child_scope;
5232		}
5233
5234		IrInstSrc *defer_expr_value = ir_gen_node(irb, defer_expr_node, defer_expr_scope);
5235		if (defer_expr_value == irb->codegen->invalid_inst_src)
5236		return irb->codegen->invalid_inst_src;
5237
5238		if (defer_expr_value->is_noreturn) {
5239		if (is_noreturn != nullptr) *is_noreturn = true;
5240		} else {
5241		ir_mark_gen(ir_build_check_statement_is_void(irb, defer_expr_scope, defer_expr_node,
5242		defer_expr_value));
5243		}
5244		scope = scope->parent;
5245		continue;
5246		}
5247		case ScopeIdDecls:
5248		case ScopeIdFnDef:
5249		return true;
5250		case ScopeIdBlock:
5251		case ScopeIdVarDecl:
5252		case ScopeIdLoop:
5253		case ScopeIdSuspend:
5254		case ScopeIdCompTime:
5255		case ScopeIdNoSuspend:
5256		case ScopeIdRuntime:
5257		case ScopeIdTypeOf:
5258		case ScopeIdExpr:
5259		scope = scope->parent;
5260		continue;
5261		case ScopeIdDeferExpr:
5262		case ScopeIdCImport:
5263		zig_unreachable();
5264		}
5265		}
5266		return true;
5267		}
5268
5269		static void ir_set_cursor_at_end_gen(IrBuilderGen irb, IrBasicBlockGen basic_block) {
5270		assert(basic_block);
5271		irb->current_basic_block = basic_block;
5272		}
5273
5274		static void ir_set_cursor_at_end(IrBuilderSrc irb, IrBasicBlockSrc basic_block) {
5275		assert(basic_block);
5276		irb->current_basic_block = basic_block;
5277		}
5278
5279		static void ir_append_basic_block_gen(IrBuilderGen irb, IrBasicBlockGen bb) {
5280		assert(!bb->already_appended);
5281		bb->already_appended = true;
5282		irb->exec->basic_block_list.append(bb);
5283		}
5284
5285		static void ir_set_cursor_at_end_and_append_block_gen(IrBuilderGen irb, IrBasicBlockGen basic_block) {
5286		ir_append_basic_block_gen(irb, basic_block);
5287		ir_set_cursor_at_end_gen(irb, basic_block);
5288		}
5289
5290		static void ir_set_cursor_at_end_and_append_block(IrBuilderSrc irb, IrBasicBlockSrc basic_block) {
5291		basic_block->index = irb->exec->basic_block_list.length;
5292		irb->exec->basic_block_list.append(basic_block);
5293		ir_set_cursor_at_end(irb, basic_block);
5294		}
5295
5296		static ScopeSuspend get_scope_suspend(Scope scope) {
5297		while (scope) {
5298		if (scope->id == ScopeIdSuspend)
5299		return (ScopeSuspend *)scope;
5300		if (scope->id == ScopeIdFnDef)
5301		return nullptr;
5302
5303		scope = scope->parent;
5304		}
5305		return nullptr;
5306		}
5307
5308		static ScopeDeferExpr get_scope_defer_expr(Scope scope) {
5309		while (scope) {
5310		if (scope->id == ScopeIdDeferExpr)
5311		return (ScopeDeferExpr *)scope;
5312		if (scope->id == ScopeIdFnDef)
5313		return nullptr;
5314
5315		scope = scope->parent;
5316		}
5317		return nullptr;
5318		}
5319
5320		static IrInstSrc ir_gen_return(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval, ResultLoc *result_loc) {
5321		assert(node->type == NodeTypeReturnExpr);
5322
5323		ScopeDeferExpr *scope_defer_expr = get_scope_defer_expr(scope);
5324		if (scope_defer_expr) {
5325		if (!scope_defer_expr->reported_err) {
5326		add_node_error(irb->codegen, node, buf_sprintf("cannot return from defer expression"));
5327		scope_defer_expr->reported_err = true;
5328		}
5329		return irb->codegen->invalid_inst_src;
5330		}
5331
5332		Scope *outer_scope = irb->exec->begin_scope;
5333
5334		AstNode *expr_node = node->data.return_expr.expr;
5335		switch (node->data.return_expr.kind) {
5336		case ReturnKindUnconditional:
5337		{
5338		ResultLocReturn *result_loc_ret = heap::c_allocator.create<ResultLocReturn>();
5339		result_loc_ret->base.id = ResultLocIdReturn;
5340		ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
5341
5342		IrInstSrc *return_value;
5343		if (expr_node) {
5344		// Temporarily set this so that if we return a type it gets the name of the function
5345		ZigFn *prev_name_fn = irb->exec->name_fn;
5346		irb->exec->name_fn = exec_fn_entry(irb->exec);
5347		return_value = ir_gen_node_extra(irb, expr_node, scope, LValNone, &result_loc_ret->base);
5348		irb->exec->name_fn = prev_name_fn;
5349		if (return_value == irb->codegen->invalid_inst_src)
5350		return irb->codegen->invalid_inst_src;
5351		} else {
5352		return_value = ir_build_const_void(irb, scope, node);
5353		ir_build_end_expr(irb, scope, node, return_value, &result_loc_ret->base);
5354		}
5355
5356		ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, return_value, result_loc_ret));
5357
5358		size_t defer_counts[2];
5359		ir_count_defers(irb, scope, outer_scope, defer_counts);
5360		bool have_err_defers = defer_counts[ReturnKindError] > 0;
5361		if (!have_err_defers && !irb->codegen->have_err_ret_tracing) {
5362		// only generate unconditional defers
5363		if (!ir_gen_defers_for_block(irb, scope, outer_scope, nullptr, nullptr))
5364		return irb->codegen->invalid_inst_src;
5365		IrInstSrc *result = ir_build_return_src(irb, scope, node, nullptr);
5366		result_loc_ret->base.source_instruction = result;
5367		return result;
5368		}
5369		bool should_inline = ir_should_inline(irb->exec, scope);
5370
5371		IrBasicBlockSrc *err_block = ir_create_basic_block(irb, scope, "ErrRetErr");
5372		IrBasicBlockSrc *ok_block = ir_create_basic_block(irb, scope, "ErrRetOk");
5373
5374		IrInstSrc *is_err = ir_build_test_err_src(irb, scope, node, return_value, false, true);
5375
5376		IrInstSrc *is_comptime;
5377		if (should_inline) {
5378		is_comptime = ir_build_const_bool(irb, scope, node, should_inline);
5379		} else {
5380		is_comptime = ir_build_test_comptime(irb, scope, node, is_err);
5381		}
5382
5383		ir_mark_gen(ir_build_cond_br(irb, scope, node, is_err, err_block, ok_block, is_comptime));
5384		IrBasicBlockSrc *ret_stmt_block = ir_create_basic_block(irb, scope, "RetStmt");
5385
5386		ir_set_cursor_at_end_and_append_block(irb, err_block);
5387		if (!ir_gen_defers_for_block(irb, scope, outer_scope, nullptr, return_value))
5388		return irb->codegen->invalid_inst_src;
5389		if (irb->codegen->have_err_ret_tracing && !should_inline) {
5390		ir_build_save_err_ret_addr_src(irb, scope, node);
5391		}
5392		ir_build_br(irb, scope, node, ret_stmt_block, is_comptime);
5393
5394		ir_set_cursor_at_end_and_append_block(irb, ok_block);
5395		if (!ir_gen_defers_for_block(irb, scope, outer_scope, nullptr, nullptr))
5396		return irb->codegen->invalid_inst_src;
5397		ir_build_br(irb, scope, node, ret_stmt_block, is_comptime);
5398
5399		ir_set_cursor_at_end_and_append_block(irb, ret_stmt_block);
5400		IrInstSrc *result = ir_build_return_src(irb, scope, node, nullptr);
5401		result_loc_ret->base.source_instruction = result;
5402		return result;
5403		}
5404		case ReturnKindError:
5405		{
5406		assert(expr_node);
5407		IrInstSrc *err_union_ptr = ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr);
5408		if (err_union_ptr == irb->codegen->invalid_inst_src)
5409		return irb->codegen->invalid_inst_src;
5410		IrInstSrc *is_err_val = ir_build_test_err_src(irb, scope, node, err_union_ptr, true, false);
5411
5412		IrBasicBlockSrc *return_block = ir_create_basic_block(irb, scope, "ErrRetReturn");
5413		IrBasicBlockSrc *continue_block = ir_create_basic_block(irb, scope, "ErrRetContinue");
5414		IrInstSrc *is_comptime;
5415		bool should_inline = ir_should_inline(irb->exec, scope);
5416		if (should_inline) {
5417		is_comptime = ir_build_const_bool(irb, scope, node, true);
5418		} else {
5419		is_comptime = ir_build_test_comptime(irb, scope, node, is_err_val);
5420		}
5421		ir_mark_gen(ir_build_cond_br(irb, scope, node, is_err_val, return_block, continue_block, is_comptime));
5422
5423		ir_set_cursor_at_end_and_append_block(irb, return_block);
5424		IrInstSrc *err_val_ptr = ir_build_unwrap_err_code_src(irb, scope, node, err_union_ptr);
5425		IrInstSrc *err_val = ir_build_load_ptr(irb, scope, node, err_val_ptr);
5426		ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, err_val, nullptr));
5427		IrInstSrcSpillBegin *spill_begin = ir_build_spill_begin_src(irb, scope, node, err_val,
5428		SpillIdRetErrCode);
5429		ResultLocReturn *result_loc_ret = heap::c_allocator.create<ResultLocReturn>();
5430		result_loc_ret->base.id = ResultLocIdReturn;
5431		ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
5432		ir_build_end_expr(irb, scope, node, err_val, &result_loc_ret->base);
5433
5434		bool is_noreturn = false;
5435		if (!ir_gen_defers_for_block(irb, scope, outer_scope, &is_noreturn, err_val)) {
5436		return irb->codegen->invalid_inst_src;
5437		}
5438		if (!is_noreturn) {
5439		if (irb->codegen->have_err_ret_tracing && !should_inline) {
5440		ir_build_save_err_ret_addr_src(irb, scope, node);
5441		}
5442		err_val = ir_build_spill_end_src(irb, scope, node, spill_begin);
5443		IrInstSrc *ret_inst = ir_build_return_src(irb, scope, node, err_val);
5444		result_loc_ret->base.source_instruction = ret_inst;
5445		}
5446
5447		ir_set_cursor_at_end_and_append_block(irb, continue_block);
5448		IrInstSrc *unwrapped_ptr = ir_build_unwrap_err_payload_src(irb, scope, node, err_union_ptr, false, false);
5449		if (lval == LValPtr)
5450		return unwrapped_ptr;
5451		else
5452		return ir_expr_wrap(irb, scope, ir_build_load_ptr(irb, scope, node, unwrapped_ptr), result_loc);
5453		}
5454		}
5455		zig_unreachable();
5456		}
5457
5458		static ZigVar create_local_var(CodeGen codegen, AstNode node, Scope parent_scope,
5459		Buf name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstSrc is_comptime,
5460		bool skip_name_check)
5461		{
5462		ZigVar *variable_entry = heap::c_allocator.create<ZigVar>();
5463		variable_entry->parent_scope = parent_scope;
5464		variable_entry->shadowable = is_shadowable;
5465		variable_entry->is_comptime = is_comptime;
5466		variable_entry->src_arg_index = SIZE_MAX;
5467		variable_entry->const_value = codegen->pass1_arena->create<ZigValue>();
5468
5469		if (is_comptime != nullptr) {
5470		is_comptime->base.ref_count += 1;
5471		}
5472
5473		if (name) {
5474		variable_entry->name = strdup(buf_ptr(name));
5475
5476		if (!skip_name_check) {
5477		ZigVar *existing_var = find_variable(codegen, parent_scope, name, nullptr);
5478		if (existing_var && !existing_var->shadowable) {
5479		if (existing_var->var_type == nullptr \|\| !type_is_invalid(existing_var->var_type)) {
5480		ErrorMsg *msg = add_node_error(codegen, node,
5481		buf_sprintf("redeclaration of variable '%s'", buf_ptr(name)));
5482		add_error_note(codegen, msg, existing_var->decl_node, buf_sprintf("previous declaration is here"));
5483		}
5484		variable_entry->var_type = codegen->builtin_types.entry_invalid;
5485		} else {
5486		ZigType *type;
5487		if (get_primitive_type(codegen, name, &type) != ErrorPrimitiveTypeNotFound) {
5488		add_node_error(codegen, node,
5489		buf_sprintf("variable shadows primitive type '%s'", buf_ptr(name)));
5490		variable_entry->var_type = codegen->builtin_types.entry_invalid;
5491		} else {
5492		Tld *tld = find_decl(codegen, parent_scope, name);
5493		if (tld != nullptr) {
5494		bool want_err_msg = true;
5495		if (tld->id == TldIdVar) {
5496		ZigVar var = reinterpret_cast<TldVar >(tld)->var;
5497		if (var != nullptr && var->var_type != nullptr && type_is_invalid(var->var_type)) {
5498		want_err_msg = false;
5499		}
5500		}
5501		if (want_err_msg) {
5502		ErrorMsg *msg = add_node_error(codegen, node,
5503		buf_sprintf("redefinition of '%s'", buf_ptr(name)));
5504		add_error_note(codegen, msg, tld->source_node, buf_sprintf("previous definition is here"));
5505		}
5506		variable_entry->var_type = codegen->builtin_types.entry_invalid;
5507		}
5508		}
5509		}
5510		}
5511		} else {
5512		assert(is_shadowable);
5513		// TODO make this name not actually be in scope. user should be able to make a variable called "_anon"
5514		// might already be solved, let's just make sure it has test coverage
5515		// maybe we put a prefix on this so the debug info doesn't clobber user debug info for same named variables
5516		variable_entry->name = "_anon";
5517		}
5518
5519		variable_entry->src_is_const = src_is_const;
5520		variable_entry->gen_is_const = gen_is_const;
5521		variable_entry->decl_node = node;
5522		variable_entry->child_scope = create_var_scope(codegen, node, parent_scope, variable_entry);
5523
5524		return variable_entry;
5525		}
5526
5527		// Set name to nullptr to make the variable anonymous (not visible to programmer).
5528		// After you call this function var->child_scope has the variable in scope
5529		static ZigVar ir_create_var(IrBuilderSrc irb, AstNode node, Scope scope, Buf *name,
5530		bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstSrc *is_comptime)
5531		{
5532		bool is_underscored = name ? buf_eql_str(name, "_") : false;
5533		ZigVar *var = create_local_var(irb->codegen, node, scope,
5534		(is_underscored ? nullptr : name), src_is_const, gen_is_const,
5535		(is_underscored ? true : is_shadowable), is_comptime, false);
5536		assert(var->child_scope);
5537		return var;
5538		}
5539
5540		static ResultLocPeer create_peer_result(ResultLocPeerParent peer_parent) {
5541		ResultLocPeer *result = heap::c_allocator.create<ResultLocPeer>();
5542		result->base.id = ResultLocIdPeer;
5543		result->base.source_instruction = peer_parent->base.source_instruction;
5544		result->parent = peer_parent;
5545		result->base.allow_write_through_const = peer_parent->parent->allow_write_through_const;
5546		return result;
5547		}
5548
5549		static bool is_duplicate_label(CodeGen g, Scope scope, AstNode node, Buf name) {
5550		if (name == nullptr) return false;
5551
5552		for (;;) {
5553		if (scope == nullptr \|\| scope->id == ScopeIdFnDef) {
5554		break;
5555		} else if (scope->id == ScopeIdBlock \|\| scope->id == ScopeIdLoop) {
5556		Buf this_block_name = scope->id == ScopeIdBlock ? ((ScopeBlock )scope)->name : ((ScopeLoop *)scope)->name;
5557		if (this_block_name != nullptr && buf_eql_buf(name, this_block_name)) {
5558		ErrorMsg *msg = add_node_error(g, node, buf_sprintf("redeclaration of label '%s'", buf_ptr(name)));
5559		add_error_note(g, msg, scope->source_node, buf_sprintf("previous declaration is here"));
5560		return true;
5561		}
5562		}
5563		scope = scope->parent;
5564		}
5565		return false;
5566		}
5567
5568		static IrInstSrc ir_gen_block(IrBuilderSrc irb, Scope parent_scope, AstNode block_node, LVal lval,
5569		ResultLoc *result_loc)
5570		{
5571		assert(block_node->type == NodeTypeBlock);
5572
5573		ZigList<IrInstSrc *> incoming_values = {0};
5574		ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
5575
5576		if (is_duplicate_label(irb->codegen, parent_scope, block_node, block_node->data.block.name))
5577		return irb->codegen->invalid_inst_src;
5578
5579		ScopeBlock *scope_block = create_block_scope(irb->codegen, block_node, parent_scope);
5580
5581		Scope *outer_block_scope = &scope_block->base;
5582		Scope *child_scope = outer_block_scope;
5583
5584		ZigFn *fn_entry = scope_fn_entry(parent_scope);
5585		if (fn_entry && fn_entry->child_scope == parent_scope) {
5586		fn_entry->def_scope = scope_block;
5587		}
5588
5589		if (block_node->data.block.statements.length == 0) {
5590		if (scope_block->name != nullptr) {
5591		add_node_error(irb->codegen, block_node, buf_sprintf("unused block label"));
5592		}
5593		// {}
5594		return ir_lval_wrap(irb, parent_scope, ir_build_const_void(irb, child_scope, block_node), lval, result_loc);
5595		}
5596
5597		if (block_node->data.block.name != nullptr) {
5598		scope_block->lval = lval;
5599		scope_block->incoming_blocks = &incoming_blocks;
5600		scope_block->incoming_values = &incoming_values;
5601		scope_block->end_block = ir_create_basic_block(irb, parent_scope, "BlockEnd");
5602		scope_block->is_comptime = ir_build_const_bool(irb, parent_scope, block_node,
5603		ir_should_inline(irb->exec, parent_scope));
5604
5605		scope_block->peer_parent = heap::c_allocator.create<ResultLocPeerParent>();
5606		scope_block->peer_parent->base.id = ResultLocIdPeerParent;
5607		scope_block->peer_parent->base.source_instruction = scope_block->is_comptime;
5608		scope_block->peer_parent->base.allow_write_through_const = result_loc->allow_write_through_const;
5609		scope_block->peer_parent->end_bb = scope_block->end_block;
5610		scope_block->peer_parent->is_comptime = scope_block->is_comptime;
5611		scope_block->peer_parent->parent = result_loc;
5612		ir_build_reset_result(irb, parent_scope, block_node, &scope_block->peer_parent->base);
5613		}
5614
5615		bool is_continuation_unreachable = false;
5616		bool found_invalid_inst = false;
5617		IrInstSrc *noreturn_return_value = nullptr;
5618		for (size_t i = 0; i < block_node->data.block.statements.length; i += 1) {
5619		AstNode *statement_node = block_node->data.block.statements.at(i);
5620
5621		IrInstSrc *statement_value = ir_gen_node(irb, statement_node, child_scope);
5622		if (statement_value == irb->codegen->invalid_inst_src) {
5623		// keep generating all the elements of the block in case of error,
5624		// we want to collect other compile errors
5625		found_invalid_inst = true;
5626		continue;
5627		}
5628
5629		is_continuation_unreachable = instr_is_unreachable(statement_value);
5630		if (is_continuation_unreachable) {
5631		// keep the last noreturn statement value around in case we need to return it
5632		noreturn_return_value = statement_value;
5633		}
5634		// This logic must be kept in sync with
5635		// [STMT_EXPR_TEST_THING] <--- (search this token)
5636		if (statement_node->type == NodeTypeDefer) {
5637		// defer starts a new scope
5638		child_scope = statement_node->data.defer.child_scope;
5639		assert(child_scope);
5640		} else if (statement_value->id == IrInstSrcIdDeclVar) {
5641		// variable declarations start a new scope
5642		IrInstSrcDeclVar decl_var_instruction = (IrInstSrcDeclVar )statement_value;
5643		child_scope = decl_var_instruction->var->child_scope;
5644		} else if (!is_continuation_unreachable) {
5645		// this statement's value must be void
5646		ir_mark_gen(ir_build_check_statement_is_void(irb, child_scope, statement_node, statement_value));
5647		}
5648		}
5649
5650		if (scope_block->name != nullptr && scope_block->name_used == false) {
5651		add_node_error(irb->codegen, block_node, buf_sprintf("unused block label"));
5652		}
5653
5654		if (found_invalid_inst)
5655		return irb->codegen->invalid_inst_src;
5656
5657		if (is_continuation_unreachable) {
5658		assert(noreturn_return_value != nullptr);
5659		if (block_node->data.block.name == nullptr \|\| incoming_blocks.length == 0) {
5660		return noreturn_return_value;
5661		}
5662
5663		if (scope_block->peer_parent != nullptr && scope_block->peer_parent->peers.length != 0) {
5664		scope_block->peer_parent->peers.last()->next_bb = scope_block->end_block;
5665		}
5666		ir_set_cursor_at_end_and_append_block(irb, scope_block->end_block);
5667		IrInstSrc *phi = ir_build_phi(irb, parent_scope, block_node, incoming_blocks.length,
5668		incoming_blocks.items, incoming_values.items, scope_block->peer_parent);
5669		return ir_expr_wrap(irb, parent_scope, phi, result_loc);
5670		} else {
5671		incoming_blocks.append(irb->current_basic_block);
5672		IrInstSrc *else_expr_result = ir_mark_gen(ir_build_const_void(irb, parent_scope, block_node));
5673
5674		if (scope_block->peer_parent != nullptr) {
5675		ResultLocPeer *peer_result = create_peer_result(scope_block->peer_parent);
5676		scope_block->peer_parent->peers.append(peer_result);
5677		ir_build_end_expr(irb, parent_scope, block_node, else_expr_result, &peer_result->base);
5678
5679		if (scope_block->peer_parent->peers.length != 0) {
5680		scope_block->peer_parent->peers.last()->next_bb = scope_block->end_block;
5681		}
5682		}
5683
5684		incoming_values.append(else_expr_result);
5685		}
5686
5687		bool is_return_from_fn = block_node == irb->main_block_node;
5688		if (!is_return_from_fn) {
5689		if (!ir_gen_defers_for_block(irb, child_scope, outer_block_scope, nullptr, nullptr))
5690		return irb->codegen->invalid_inst_src;
5691		}
5692
5693		IrInstSrc *result;
5694		if (block_node->data.block.name != nullptr) {
5695		ir_mark_gen(ir_build_br(irb, parent_scope, block_node, scope_block->end_block, scope_block->is_comptime));
5696		ir_set_cursor_at_end_and_append_block(irb, scope_block->end_block);
5697		IrInstSrc *phi = ir_build_phi(irb, parent_scope, block_node, incoming_blocks.length,
5698		incoming_blocks.items, incoming_values.items, scope_block->peer_parent);
5699		result = ir_expr_wrap(irb, parent_scope, phi, result_loc);
5700		} else {
5701		IrInstSrc *void_inst = ir_mark_gen(ir_build_const_void(irb, child_scope, block_node));
5702		result = ir_lval_wrap(irb, parent_scope, void_inst, lval, result_loc);
5703		}
5704		if (!is_return_from_fn)
5705		return result;
5706
5707		// no need for save_err_ret_addr because this cannot return error
5708		// only generate unconditional defers
5709
5710		ir_mark_gen(ir_build_add_implicit_return_type(irb, child_scope, block_node, result, nullptr));
5711		ResultLocReturn *result_loc_ret = heap::c_allocator.create<ResultLocReturn>();
5712		result_loc_ret->base.id = ResultLocIdReturn;
5713		ir_build_reset_result(irb, parent_scope, block_node, &result_loc_ret->base);
5714		ir_mark_gen(ir_build_end_expr(irb, parent_scope, block_node, result, &result_loc_ret->base));
5715		if (!ir_gen_defers_for_block(irb, child_scope, outer_block_scope, nullptr, nullptr))
5716		return irb->codegen->invalid_inst_src;
5717		return ir_mark_gen(ir_build_return_src(irb, child_scope, result->base.source_node, result));
5718		}
5719
5720		static IrInstSrc ir_gen_bin_op_id(IrBuilderSrc irb, Scope scope, AstNode node, IrBinOp op_id) {
5721		Scope *inner_scope = scope;
5722		if (op_id == IrBinOpArrayCat \|\| op_id == IrBinOpArrayMult) {
5723		inner_scope = create_comptime_scope(irb->codegen, node, scope);
5724		}
5725
5726		IrInstSrc *op1 = ir_gen_node(irb, node->data.bin_op_expr.op1, inner_scope);
5727		IrInstSrc *op2 = ir_gen_node(irb, node->data.bin_op_expr.op2, inner_scope);
5728
5729		if (op1 == irb->codegen->invalid_inst_src \|\| op2 == irb->codegen->invalid_inst_src)
5730		return irb->codegen->invalid_inst_src;
5731
5732		return ir_build_bin_op(irb, scope, node, op_id, op1, op2, true);
5733		}
5734
5735		static IrInstSrc ir_gen_merge_err_sets(IrBuilderSrc irb, Scope scope, AstNode node) {
5736		IrInstSrc *op1 = ir_gen_node(irb, node->data.bin_op_expr.op1, scope);
5737		IrInstSrc *op2 = ir_gen_node(irb, node->data.bin_op_expr.op2, scope);
5738
5739		if (op1 == irb->codegen->invalid_inst_src \|\| op2 == irb->codegen->invalid_inst_src)
5740		return irb->codegen->invalid_inst_src;
5741
5742		// TODO only pass type_name when the \|\| operator is the top level AST node in the var decl expr
5743		Buf bare_name = BUF_INIT;
5744		Buf *type_name = get_anon_type_name(irb->codegen, irb->exec, "error", scope, node, &bare_name);
5745
5746		return ir_build_merge_err_sets(irb, scope, node, op1, op2, type_name);
5747		}
5748
5749		static IrInstSrc ir_gen_assign(IrBuilderSrc irb, Scope scope, AstNode node) {
5750		IrInstSrc *lvalue = ir_gen_node_extra(irb, node->data.bin_op_expr.op1, scope, LValAssign, nullptr);
5751		if (lvalue == irb->codegen->invalid_inst_src)
5752		return irb->codegen->invalid_inst_src;
5753
5754		ResultLocInstruction *result_loc_inst = heap::c_allocator.create<ResultLocInstruction>();
5755		result_loc_inst->base.id = ResultLocIdInstruction;
5756		result_loc_inst->base.source_instruction = lvalue;
5757		ir_ref_instruction(lvalue, irb->current_basic_block);
5758		ir_build_reset_result(irb, scope, node, &result_loc_inst->base);
5759
5760		IrInstSrc *rvalue = ir_gen_node_extra(irb, node->data.bin_op_expr.op2, scope, LValNone,
5761		&result_loc_inst->base);
5762		if (rvalue == irb->codegen->invalid_inst_src)
5763		return irb->codegen->invalid_inst_src;
5764
5765		return ir_build_const_void(irb, scope, node);
5766		}
5767
5768		static IrInstSrc ir_gen_assign_op(IrBuilderSrc irb, Scope scope, AstNode node, IrBinOp op_id) {
5769		IrInstSrc *lvalue = ir_gen_node_extra(irb, node->data.bin_op_expr.op1, scope, LValAssign, nullptr);
5770		if (lvalue == irb->codegen->invalid_inst_src)
5771		return lvalue;
5772		IrInstSrc *op1 = ir_build_load_ptr(irb, scope, node->data.bin_op_expr.op1, lvalue);
5773		IrInstSrc *op2 = ir_gen_node(irb, node->data.bin_op_expr.op2, scope);
5774		if (op2 == irb->codegen->invalid_inst_src)
5775		return op2;
5776		IrInstSrc *result = ir_build_bin_op(irb, scope, node, op_id, op1, op2, true);
5777		ir_build_store_ptr(irb, scope, node, lvalue, result);
5778		return ir_build_const_void(irb, scope, node);
5779		}
5780
5781		static IrInstSrc ir_gen_bool_or(IrBuilderSrc irb, Scope scope, AstNode node) {
5782		assert(node->type == NodeTypeBinOpExpr);
5783
5784		IrInstSrc *val1 = ir_gen_node(irb, node->data.bin_op_expr.op1, scope);
5785		if (val1 == irb->codegen->invalid_inst_src)
5786		return irb->codegen->invalid_inst_src;
5787		IrBasicBlockSrc *post_val1_block = irb->current_basic_block;
5788
5789		IrInstSrc *is_comptime;
5790		if (ir_should_inline(irb->exec, scope)) {
5791		is_comptime = ir_build_const_bool(irb, scope, node, true);
5792		} else {
5793		is_comptime = ir_build_test_comptime(irb, scope, node, val1);
5794		}
5795
5796		// block for when val1 == false
5797		IrBasicBlockSrc *false_block = ir_create_basic_block(irb, scope, "BoolOrFalse");
5798		// block for when val1 == true (don't even evaluate the second part)
5799		IrBasicBlockSrc *true_block = ir_create_basic_block(irb, scope, "BoolOrTrue");
5800
5801		ir_build_cond_br(irb, scope, node, val1, true_block, false_block, is_comptime);
5802
5803		ir_set_cursor_at_end_and_append_block(irb, false_block);
5804		IrInstSrc *val2 = ir_gen_node(irb, node->data.bin_op_expr.op2, scope);
5805		if (val2 == irb->codegen->invalid_inst_src)
5806		return irb->codegen->invalid_inst_src;
5807		IrBasicBlockSrc *post_val2_block = irb->current_basic_block;
5808
5809		ir_build_br(irb, scope, node, true_block, is_comptime);
5810
5811		ir_set_cursor_at_end_and_append_block(irb, true_block);
5812
5813		IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
5814		incoming_values[0] = val1;
5815		incoming_values[1] = val2;
5816		IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
5817		incoming_blocks[0] = post_val1_block;
5818		incoming_blocks[1] = post_val2_block;
5819
5820		return ir_build_phi(irb, scope, node, 2, incoming_blocks, incoming_values, nullptr);
5821		}
5822
5823		static IrInstSrc ir_gen_bool_and(IrBuilderSrc irb, Scope scope, AstNode node) {
5824		assert(node->type == NodeTypeBinOpExpr);
5825
5826		IrInstSrc *val1 = ir_gen_node(irb, node->data.bin_op_expr.op1, scope);
5827		if (val1 == irb->codegen->invalid_inst_src)
5828		return irb->codegen->invalid_inst_src;
5829		IrBasicBlockSrc *post_val1_block = irb->current_basic_block;
5830
5831		IrInstSrc *is_comptime;
5832		if (ir_should_inline(irb->exec, scope)) {
5833		is_comptime = ir_build_const_bool(irb, scope, node, true);
5834		} else {
5835		is_comptime = ir_build_test_comptime(irb, scope, node, val1);
5836		}
5837
5838		// block for when val1 == true
5839		IrBasicBlockSrc *true_block = ir_create_basic_block(irb, scope, "BoolAndTrue");
5840		// block for when val1 == false (don't even evaluate the second part)
5841		IrBasicBlockSrc *false_block = ir_create_basic_block(irb, scope, "BoolAndFalse");
5842
5843		ir_build_cond_br(irb, scope, node, val1, true_block, false_block, is_comptime);
5844
5845		ir_set_cursor_at_end_and_append_block(irb, true_block);
5846		IrInstSrc *val2 = ir_gen_node(irb, node->data.bin_op_expr.op2, scope);
5847		if (val2 == irb->codegen->invalid_inst_src)
5848		return irb->codegen->invalid_inst_src;
5849		IrBasicBlockSrc *post_val2_block = irb->current_basic_block;
5850
5851		ir_build_br(irb, scope, node, false_block, is_comptime);
5852
5853		ir_set_cursor_at_end_and_append_block(irb, false_block);
5854
5855		IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
5856		incoming_values[0] = val1;
5857		incoming_values[1] = val2;
5858		IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
5859		incoming_blocks[0] = post_val1_block;
5860		incoming_blocks[1] = post_val2_block;
5861
5862		return ir_build_phi(irb, scope, node, 2, incoming_blocks, incoming_values, nullptr);
5863		}
5864
5865		static ResultLocPeerParent ir_build_result_peers(IrBuilderSrc irb, IrInstSrc *cond_br_inst,
5866		IrBasicBlockSrc end_block, ResultLoc parent, IrInstSrc *is_comptime)
5867		{
5868		ResultLocPeerParent *peer_parent = heap::c_allocator.create<ResultLocPeerParent>();
5869		peer_parent->base.id = ResultLocIdPeerParent;
5870		peer_parent->base.source_instruction = cond_br_inst;
5871		peer_parent->base.allow_write_through_const = parent->allow_write_through_const;
5872		peer_parent->end_bb = end_block;
5873		peer_parent->is_comptime = is_comptime;
5874		peer_parent->parent = parent;
5875
5876		IrInstSrc *popped_inst = irb->current_basic_block->instruction_list.pop();
5877		ir_assert(popped_inst == cond_br_inst, &cond_br_inst->base);
5878
5879		ir_build_reset_result(irb, cond_br_inst->base.scope, cond_br_inst->base.source_node, &peer_parent->base);
5880		irb->current_basic_block->instruction_list.append(popped_inst);
5881
5882		return peer_parent;
5883		}
5884
5885		static ResultLocPeerParent ir_build_binary_result_peers(IrBuilderSrc irb, IrInstSrc *cond_br_inst,
5886		IrBasicBlockSrc else_block, IrBasicBlockSrc end_block, ResultLoc parent, IrInstSrc is_comptime)
5887		{
5888		ResultLocPeerParent *peer_parent = ir_build_result_peers(irb, cond_br_inst, end_block, parent, is_comptime);
5889
5890		peer_parent->peers.append(create_peer_result(peer_parent));
5891		peer_parent->peers.last()->next_bb = else_block;
5892
5893		peer_parent->peers.append(create_peer_result(peer_parent));
5894		peer_parent->peers.last()->next_bb = end_block;
5895
5896		return peer_parent;
5897		}
5898
5899		static IrInstSrc ir_gen_orelse(IrBuilderSrc irb, Scope parent_scope, AstNode node, LVal lval,
5900		ResultLoc *result_loc)
5901		{
5902		assert(node->type == NodeTypeBinOpExpr);
5903
5904		AstNode *op1_node = node->data.bin_op_expr.op1;
5905		AstNode *op2_node = node->data.bin_op_expr.op2;
5906
5907		IrInstSrc *maybe_ptr = ir_gen_node_extra(irb, op1_node, parent_scope, LValPtr, nullptr);
5908		if (maybe_ptr == irb->codegen->invalid_inst_src)
5909		return irb->codegen->invalid_inst_src;
5910
5911		IrInstSrc *maybe_val = ir_build_load_ptr(irb, parent_scope, node, maybe_ptr);
5912		IrInstSrc *is_non_null = ir_build_test_non_null_src(irb, parent_scope, node, maybe_val);
5913
5914		IrInstSrc *is_comptime;
5915		if (ir_should_inline(irb->exec, parent_scope)) {
5916		is_comptime = ir_build_const_bool(irb, parent_scope, node, true);
5917		} else {
5918		is_comptime = ir_build_test_comptime(irb, parent_scope, node, is_non_null);
5919		}
5920
5921		IrBasicBlockSrc *ok_block = ir_create_basic_block(irb, parent_scope, "OptionalNonNull");
5922		IrBasicBlockSrc *null_block = ir_create_basic_block(irb, parent_scope, "OptionalNull");
5923		IrBasicBlockSrc *end_block = ir_create_basic_block(irb, parent_scope, "OptionalEnd");
5924		IrInstSrc *cond_br_inst = ir_build_cond_br(irb, parent_scope, node, is_non_null, ok_block, null_block, is_comptime);
5925
5926		ResultLocPeerParent *peer_parent = ir_build_binary_result_peers(irb, cond_br_inst, ok_block, end_block,
5927		result_loc, is_comptime);
5928
5929		ir_set_cursor_at_end_and_append_block(irb, null_block);
5930		IrInstSrc *null_result = ir_gen_node_extra(irb, op2_node, parent_scope, LValNone,
5931		&peer_parent->peers.at(0)->base);
5932		if (null_result == irb->codegen->invalid_inst_src)
5933		return irb->codegen->invalid_inst_src;
5934		IrBasicBlockSrc *after_null_block = irb->current_basic_block;
5935		if (!instr_is_unreachable(null_result))
5936		ir_mark_gen(ir_build_br(irb, parent_scope, node, end_block, is_comptime));
5937
5938		ir_set_cursor_at_end_and_append_block(irb, ok_block);
5939		IrInstSrc *unwrapped_ptr = ir_build_optional_unwrap_ptr(irb, parent_scope, node, maybe_ptr, false);
5940		IrInstSrc *unwrapped_payload = ir_build_load_ptr(irb, parent_scope, node, unwrapped_ptr);
5941		ir_build_end_expr(irb, parent_scope, node, unwrapped_payload, &peer_parent->peers.at(1)->base);
5942		IrBasicBlockSrc *after_ok_block = irb->current_basic_block;
5943		ir_build_br(irb, parent_scope, node, end_block, is_comptime);
5944
5945		ir_set_cursor_at_end_and_append_block(irb, end_block);
5946		IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
5947		incoming_values[0] = null_result;
5948		incoming_values[1] = unwrapped_payload;
5949		IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
5950		incoming_blocks[0] = after_null_block;
5951		incoming_blocks[1] = after_ok_block;
5952		IrInstSrc *phi = ir_build_phi(irb, parent_scope, node, 2, incoming_blocks, incoming_values, peer_parent);
5953		return ir_lval_wrap(irb, parent_scope, phi, lval, result_loc);
5954		}
5955
5956		static IrInstSrc ir_gen_error_union(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
5957		assert(node->type == NodeTypeBinOpExpr);
5958
5959		AstNode *op1_node = node->data.bin_op_expr.op1;
5960		AstNode *op2_node = node->data.bin_op_expr.op2;
5961
5962		IrInstSrc *err_set = ir_gen_node(irb, op1_node, parent_scope);
5963		if (err_set == irb->codegen->invalid_inst_src)
5964		return irb->codegen->invalid_inst_src;
5965
5966		IrInstSrc *payload = ir_gen_node(irb, op2_node, parent_scope);
5967		if (payload == irb->codegen->invalid_inst_src)
5968		return irb->codegen->invalid_inst_src;
5969
5970		return ir_build_error_union(irb, parent_scope, node, err_set, payload);
5971		}
5972
5973		static IrInstSrc ir_gen_bin_op(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval, ResultLoc *result_loc) {
5974		assert(node->type == NodeTypeBinOpExpr);
5975
5976		BinOpType bin_op_type = node->data.bin_op_expr.bin_op;
5977		switch (bin_op_type) {
5978		case BinOpTypeInvalid:
5979		zig_unreachable();
5980		case BinOpTypeAssign:
5981		return ir_lval_wrap(irb, scope, ir_gen_assign(irb, scope, node), lval, result_loc);
5982		case BinOpTypeAssignTimes:
5983		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpMult), lval, result_loc);
5984		case BinOpTypeAssignTimesWrap:
5985		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpMultWrap), lval, result_loc);
5986		case BinOpTypeAssignDiv:
5987		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpDivUnspecified), lval, result_loc);
5988		case BinOpTypeAssignMod:
5989		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpRemUnspecified), lval, result_loc);
5990		case BinOpTypeAssignPlus:
5991		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpAdd), lval, result_loc);
5992		case BinOpTypeAssignPlusWrap:
5993		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpAddWrap), lval, result_loc);
5994		case BinOpTypeAssignMinus:
5995		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpSub), lval, result_loc);
5996		case BinOpTypeAssignMinusWrap:
5997		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpSubWrap), lval, result_loc);
5998		case BinOpTypeAssignBitShiftLeft:
5999		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpBitShiftLeftLossy), lval, result_loc);
6000		case BinOpTypeAssignBitShiftRight:
6001		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpBitShiftRightLossy), lval, result_loc);
6002		case BinOpTypeAssignBitAnd:
6003		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpBinAnd), lval, result_loc);
6004		case BinOpTypeAssignBitXor:
6005		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpBinXor), lval, result_loc);
6006		case BinOpTypeAssignBitOr:
6007		return ir_lval_wrap(irb, scope, ir_gen_assign_op(irb, scope, node, IrBinOpBinOr), lval, result_loc);
6008		case BinOpTypeBoolOr:
6009		return ir_lval_wrap(irb, scope, ir_gen_bool_or(irb, scope, node), lval, result_loc);
6010		case BinOpTypeBoolAnd:
6011		return ir_lval_wrap(irb, scope, ir_gen_bool_and(irb, scope, node), lval, result_loc);
6012		case BinOpTypeCmpEq:
6013		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpEq), lval, result_loc);
6014		case BinOpTypeCmpNotEq:
6015		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpNotEq), lval, result_loc);
6016		case BinOpTypeCmpLessThan:
6017		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpLessThan), lval, result_loc);
6018		case BinOpTypeCmpGreaterThan:
6019		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpGreaterThan), lval, result_loc);
6020		case BinOpTypeCmpLessOrEq:
6021		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpLessOrEq), lval, result_loc);
6022		case BinOpTypeCmpGreaterOrEq:
6023		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpCmpGreaterOrEq), lval, result_loc);
6024		case BinOpTypeBinOr:
6025		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpBinOr), lval, result_loc);
6026		case BinOpTypeBinXor:
6027		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpBinXor), lval, result_loc);
6028		case BinOpTypeBinAnd:
6029		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpBinAnd), lval, result_loc);
6030		case BinOpTypeBitShiftLeft:
6031		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpBitShiftLeftLossy), lval, result_loc);
6032		case BinOpTypeBitShiftRight:
6033		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpBitShiftRightLossy), lval, result_loc);
6034		case BinOpTypeAdd:
6035		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpAdd), lval, result_loc);
6036		case BinOpTypeAddWrap:
6037		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpAddWrap), lval, result_loc);
6038		case BinOpTypeSub:
6039		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpSub), lval, result_loc);
6040		case BinOpTypeSubWrap:
6041		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpSubWrap), lval, result_loc);
6042		case BinOpTypeMult:
6043		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpMult), lval, result_loc);
6044		case BinOpTypeMultWrap:
6045		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpMultWrap), lval, result_loc);
6046		case BinOpTypeDiv:
6047		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpDivUnspecified), lval, result_loc);
6048		case BinOpTypeMod:
6049		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpRemUnspecified), lval, result_loc);
6050		case BinOpTypeArrayCat:
6051		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpArrayCat), lval, result_loc);
6052		case BinOpTypeArrayMult:
6053		return ir_lval_wrap(irb, scope, ir_gen_bin_op_id(irb, scope, node, IrBinOpArrayMult), lval, result_loc);
6054		case BinOpTypeMergeErrorSets:
6055		return ir_lval_wrap(irb, scope, ir_gen_merge_err_sets(irb, scope, node), lval, result_loc);
6056		case BinOpTypeUnwrapOptional:
6057		return ir_gen_orelse(irb, scope, node, lval, result_loc);
6058		case BinOpTypeErrorUnion:
6059		return ir_lval_wrap(irb, scope, ir_gen_error_union(irb, scope, node), lval, result_loc);
6060		}
6061		zig_unreachable();
6062		}
6063
6064		static IrInstSrc ir_gen_int_lit(IrBuilderSrc irb, Scope scope, AstNode node) {
6065		assert(node->type == NodeTypeIntLiteral);
6066
6067		return ir_build_const_bigint(irb, scope, node, node->data.int_literal.bigint);
6068		}
6069
6070		static IrInstSrc ir_gen_float_lit(IrBuilderSrc irb, Scope scope, AstNode node) {
6071		assert(node->type == NodeTypeFloatLiteral);
6072
6073		if (node->data.float_literal.overflow) {
6074		add_node_error(irb->codegen, node, buf_sprintf("float literal out of range of any type"));
6075		return irb->codegen->invalid_inst_src;
6076		}
6077
6078		return ir_build_const_bigfloat(irb, scope, node, node->data.float_literal.bigfloat);
6079		}
6080
6081		static IrInstSrc ir_gen_char_lit(IrBuilderSrc irb, Scope scope, AstNode node) {
6082		assert(node->type == NodeTypeCharLiteral);
6083
6084		return ir_build_const_uint(irb, scope, node, node->data.char_literal.value);
6085		}
6086
6087		static IrInstSrc ir_gen_null_literal(IrBuilderSrc irb, Scope scope, AstNode node) {
6088		assert(node->type == NodeTypeNullLiteral);
6089
6090		return ir_build_const_null(irb, scope, node);
6091		}
6092
6093		static void populate_invalid_variable_in_scope(CodeGen g, Scope scope, AstNode node, Buf var_name) {
6094		ScopeDecls *scope_decls = nullptr;
6095		while (scope != nullptr) {
6096		if (scope->id == ScopeIdDecls) {
6097		scope_decls = reinterpret_cast<ScopeDecls *>(scope);
6098		}
6099		scope = scope->parent;
6100		}
6101		TldVar *tld_var = heap::c_allocator.create<TldVar>();
6102		init_tld(&tld_var->base, TldIdVar, var_name, VisibModPub, node, &scope_decls->base);
6103		tld_var->base.resolution = TldResolutionInvalid;
6104		tld_var->var = add_variable(g, node, &scope_decls->base, var_name, false,
6105		g->invalid_inst_gen->value, &tld_var->base, g->builtin_types.entry_invalid);
6106		scope_decls->decl_table.put(var_name, &tld_var->base);
6107		}
6108
6109		static IrInstSrc ir_gen_symbol(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval, ResultLoc *result_loc) {
6110		Error err;
6111		assert(node->type == NodeTypeSymbol);
6112
6113		Buf *variable_name = node->data.symbol_expr.symbol;
6114
6115		if (buf_eql_str(variable_name, "_")) {
6116		if (lval == LValAssign) {
6117		IrInstSrcConst *const_instruction = ir_build_instruction<IrInstSrcConst>(irb, scope, node);
6118		const_instruction->value = irb->codegen->pass1_arena->create<ZigValue>();
6119		const_instruction->value->type = get_pointer_to_type(irb->codegen,
6120		irb->codegen->builtin_types.entry_void, false);
6121		const_instruction->value->special = ConstValSpecialStatic;
6122		const_instruction->value->data.x_ptr.special = ConstPtrSpecialDiscard;
6123		return &const_instruction->base;
6124		} else {
6125		add_node_error(irb->codegen, node, buf_sprintf("`_` may only be used to assign things to"));
6126		return irb->codegen->invalid_inst_src;
6127		}
6128		}
6129
6130		ZigType *primitive_type;
6131		if ((err = get_primitive_type(irb->codegen, variable_name, &primitive_type))) {
6132		if (err == ErrorOverflow) {
6133		add_node_error(irb->codegen, node,
6134		buf_sprintf("primitive integer type '%s' exceeds maximum bit width of 65535",
6135		buf_ptr(variable_name)));
6136		return irb->codegen->invalid_inst_src;
6137		}
6138		assert(err == ErrorPrimitiveTypeNotFound);
6139		} else {
6140		IrInstSrc *value = ir_build_const_type(irb, scope, node, primitive_type);
6141		if (lval == LValPtr \|\| lval == LValAssign) {
6142		return ir_build_ref_src(irb, scope, node, value);
6143		} else {
6144		return ir_expr_wrap(irb, scope, value, result_loc);
6145		}
6146		}
6147
6148		ScopeFnDef *crossed_fndef_scope;
6149		ZigVar *var = find_variable(irb->codegen, scope, variable_name, &crossed_fndef_scope);
6150		if (var) {
6151		IrInstSrc *var_ptr = ir_build_var_ptr_x(irb, scope, node, var, crossed_fndef_scope);
6152		if (lval == LValPtr \|\| lval == LValAssign) {
6153		return var_ptr;
6154		} else {
6155		return ir_expr_wrap(irb, scope, ir_build_load_ptr(irb, scope, node, var_ptr), result_loc);
6156		}
6157		}
6158
6159		Tld *tld = find_decl(irb->codegen, scope, variable_name);
6160		if (tld) {
6161		IrInstSrc *decl_ref = ir_build_decl_ref(irb, scope, node, tld, lval);
6162		if (lval == LValPtr \|\| lval == LValAssign) {
6163		return decl_ref;
6164		} else {
6165		return ir_expr_wrap(irb, scope, decl_ref, result_loc);
6166		}
6167		}
6168
6169		if (get_container_scope(node->owner)->any_imports_failed) {
6170		// skip the error message since we had a failing import in this file
6171		// if an import breaks we don't need redundant undeclared identifier errors
6172		return irb->codegen->invalid_inst_src;
6173		}
6174
6175		return ir_build_undeclared_identifier(irb, scope, node, variable_name);
6176		}
6177
6178		static IrInstSrc ir_gen_array_access(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
6179		ResultLoc *result_loc)
6180		{
6181		assert(node->type == NodeTypeArrayAccessExpr);
6182
6183		AstNode *array_ref_node = node->data.array_access_expr.array_ref_expr;
6184		IrInstSrc *array_ref_instruction = ir_gen_node_extra(irb, array_ref_node, scope, LValPtr, nullptr);
6185		if (array_ref_instruction == irb->codegen->invalid_inst_src)
6186		return array_ref_instruction;
6187
6188		// Create an usize-typed result location to hold the subscript value, this
6189		// makes it possible for the compiler to infer the subscript expression type
6190		// if needed
6191		IrInstSrc *usize_type_inst = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize);
6192		ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, usize_type_inst, no_result_loc());
6193
6194		AstNode *subscript_node = node->data.array_access_expr.subscript;
6195		IrInstSrc *subscript_value = ir_gen_node_extra(irb, subscript_node, scope, LValNone, &result_loc_cast->base);
6196		if (subscript_value == irb->codegen->invalid_inst_src)
6197		return irb->codegen->invalid_inst_src;
6198
6199		IrInstSrc *subscript_instruction = ir_build_implicit_cast(irb, scope, subscript_node, subscript_value, result_loc_cast);
6200
6201		IrInstSrc *ptr_instruction = ir_build_elem_ptr(irb, scope, node, array_ref_instruction,
6202		subscript_instruction, true, PtrLenSingle, nullptr);
6203		if (lval == LValPtr \|\| lval == LValAssign)
6204		return ptr_instruction;
6205
6206		IrInstSrc *load_ptr = ir_build_load_ptr(irb, scope, node, ptr_instruction);
6207		return ir_expr_wrap(irb, scope, load_ptr, result_loc);
6208		}
6209
6210		static IrInstSrc ir_gen_field_access(IrBuilderSrc irb, Scope scope, AstNode node) {
6211		assert(node->type == NodeTypeFieldAccessExpr);
6212
6213		AstNode *container_ref_node = node->data.field_access_expr.struct_expr;
6214		Buf *field_name = node->data.field_access_expr.field_name;
6215
6216		IrInstSrc *container_ref_instruction = ir_gen_node_extra(irb, container_ref_node, scope, LValPtr, nullptr);
6217		if (container_ref_instruction == irb->codegen->invalid_inst_src)
6218		return container_ref_instruction;
6219
6220		return ir_build_field_ptr(irb, scope, node, container_ref_instruction, field_name, false);
6221		}
6222
6223		static IrInstSrc ir_gen_overflow_op(IrBuilderSrc irb, Scope scope, AstNode node, IrOverflowOp op) {
6224		assert(node->type == NodeTypeFnCallExpr);
6225
6226		AstNode *type_node = node->data.fn_call_expr.params.at(0);
6227		AstNode *op1_node = node->data.fn_call_expr.params.at(1);
6228		AstNode *op2_node = node->data.fn_call_expr.params.at(2);
6229		AstNode *result_ptr_node = node->data.fn_call_expr.params.at(3);
6230
6231
6232		IrInstSrc *type_value = ir_gen_node(irb, type_node, scope);
6233		if (type_value == irb->codegen->invalid_inst_src)
6234		return irb->codegen->invalid_inst_src;
6235
6236		IrInstSrc *op1 = ir_gen_node(irb, op1_node, scope);
6237		if (op1 == irb->codegen->invalid_inst_src)
6238		return irb->codegen->invalid_inst_src;
6239
6240		IrInstSrc *op2 = ir_gen_node(irb, op2_node, scope);
6241		if (op2 == irb->codegen->invalid_inst_src)
6242		return irb->codegen->invalid_inst_src;
6243
6244		IrInstSrc *result_ptr = ir_gen_node(irb, result_ptr_node, scope);
6245		if (result_ptr == irb->codegen->invalid_inst_src)
6246		return irb->codegen->invalid_inst_src;
6247
6248		return ir_build_overflow_op_src(irb, scope, node, op, type_value, op1, op2, result_ptr);
6249		}
6250
6251		static IrInstSrc ir_gen_mul_add(IrBuilderSrc irb, Scope scope, AstNode node) {
6252		assert(node->type == NodeTypeFnCallExpr);
6253
6254		AstNode *type_node = node->data.fn_call_expr.params.at(0);
6255		AstNode *op1_node = node->data.fn_call_expr.params.at(1);
6256		AstNode *op2_node = node->data.fn_call_expr.params.at(2);
6257		AstNode *op3_node = node->data.fn_call_expr.params.at(3);
6258
6259		IrInstSrc *type_value = ir_gen_node(irb, type_node, scope);
6260		if (type_value == irb->codegen->invalid_inst_src)
6261		return irb->codegen->invalid_inst_src;
6262
6263		IrInstSrc *op1 = ir_gen_node(irb, op1_node, scope);
6264		if (op1 == irb->codegen->invalid_inst_src)
6265		return irb->codegen->invalid_inst_src;
6266
6267		IrInstSrc *op2 = ir_gen_node(irb, op2_node, scope);
6268		if (op2 == irb->codegen->invalid_inst_src)
6269		return irb->codegen->invalid_inst_src;
6270
6271		IrInstSrc *op3 = ir_gen_node(irb, op3_node, scope);
6272		if (op3 == irb->codegen->invalid_inst_src)
6273		return irb->codegen->invalid_inst_src;
6274
6275		return ir_build_mul_add_src(irb, scope, node, type_value, op1, op2, op3);
6276		}
6277
6278		static IrInstSrc ir_gen_this(IrBuilderSrc irb, Scope orig_scope, AstNode node) {
6279		for (Scope *it_scope = orig_scope; it_scope != nullptr; it_scope = it_scope->parent) {
6280		if (it_scope->id == ScopeIdDecls) {
6281		ScopeDecls decls_scope = (ScopeDecls )it_scope;
6282		ZigType *container_type = decls_scope->container_type;
6283		if (container_type != nullptr) {
6284		return ir_build_const_type(irb, orig_scope, node, container_type);
6285		} else {
6286		return ir_build_const_import(irb, orig_scope, node, decls_scope->import);
6287		}
6288		}
6289		}
6290		zig_unreachable();
6291		}
6292
6293		static IrInstSrc ir_gen_async_call(IrBuilderSrc irb, Scope scope, AstNode await_node, AstNode *call_node,
6294		LVal lval, ResultLoc *result_loc)
6295		{
6296		if (call_node->data.fn_call_expr.params.length != 4) {
6297		add_node_error(irb->codegen, call_node,
6298		buf_sprintf("expected 4 arguments, found %" ZIG_PRI_usize,
6299		call_node->data.fn_call_expr.params.length));
6300		return irb->codegen->invalid_inst_src;
6301		}
6302
6303		AstNode *bytes_node = call_node->data.fn_call_expr.params.at(0);
6304		IrInstSrc *bytes = ir_gen_node(irb, bytes_node, scope);
6305		if (bytes == irb->codegen->invalid_inst_src)
6306		return bytes;
6307
6308		AstNode *ret_ptr_node = call_node->data.fn_call_expr.params.at(1);
6309		IrInstSrc *ret_ptr = ir_gen_node(irb, ret_ptr_node, scope);
6310		if (ret_ptr == irb->codegen->invalid_inst_src)
6311		return ret_ptr;
6312
6313		AstNode *fn_ref_node = call_node->data.fn_call_expr.params.at(2);
6314		IrInstSrc *fn_ref = ir_gen_node(irb, fn_ref_node, scope);
6315		if (fn_ref == irb->codegen->invalid_inst_src)
6316		return fn_ref;
6317
6318		CallModifier modifier = (await_node == nullptr) ? CallModifierAsync : CallModifierNone;
6319		bool is_async_call_builtin = true;
6320		AstNode *args_node = call_node->data.fn_call_expr.params.at(3);
6321		if (args_node->type == NodeTypeContainerInitExpr) {
6322		if (args_node->data.container_init_expr.kind == ContainerInitKindArray \|\|
6323		args_node->data.container_init_expr.entries.length == 0)
6324		{
6325		size_t arg_count = args_node->data.container_init_expr.entries.length;
6326		IrInstSrc *args = heap::c_allocator.allocate<IrInstSrc>(arg_count);
6327		for (size_t i = 0; i < arg_count; i += 1) {
6328		AstNode *arg_node = args_node->data.container_init_expr.entries.at(i);
6329		IrInstSrc *arg = ir_gen_node(irb, arg_node, scope);
6330		if (arg == irb->codegen->invalid_inst_src)
6331		return arg;
6332		args[i] = arg;
6333		}
6334
6335		IrInstSrc *call = ir_build_call_src(irb, scope, call_node, nullptr, fn_ref, arg_count, args,
6336		ret_ptr, modifier, is_async_call_builtin, bytes, result_loc);
6337		return ir_lval_wrap(irb, scope, call, lval, result_loc);
6338		} else {
6339		exec_add_error_node(irb->codegen, irb->exec, args_node,
6340		buf_sprintf("TODO: @asyncCall with anon struct literal"));
6341		return irb->codegen->invalid_inst_src;
6342		}
6343		}
6344		IrInstSrc *args = ir_gen_node(irb, args_node, scope);
6345		if (args == irb->codegen->invalid_inst_src)
6346		return args;
6347
6348		IrInstSrc *call = ir_build_async_call_extra(irb, scope, call_node, modifier, fn_ref, ret_ptr, bytes, args, result_loc);
6349		return ir_lval_wrap(irb, scope, call, lval, result_loc);
6350		}
6351
6352		static IrInstSrc ir_gen_fn_call_with_args(IrBuilderSrc irb, Scope scope, AstNode source_node,
6353		AstNode fn_ref_node, CallModifier modifier, IrInstSrc options,
6354		AstNode *args_ptr, size_t args_len, LVal lval, ResultLoc result_loc)
6355		{
6356		IrInstSrc *fn_ref = ir_gen_node(irb, fn_ref_node, scope);
6357		if (fn_ref == irb->codegen->invalid_inst_src)
6358		return fn_ref;
6359
6360		IrInstSrc *fn_type = ir_build_typeof_1(irb, scope, source_node, fn_ref);
6361
6362		IrInstSrc *args = heap::c_allocator.allocate<IrInstSrc>(args_len);
6363		for (size_t i = 0; i < args_len; i += 1) {
6364		AstNode *arg_node = args_ptr[i];
6365
6366		IrInstSrc *arg_index = ir_build_const_usize(irb, scope, arg_node, i);
6367		IrInstSrc *arg_type = ir_build_arg_type(irb, scope, source_node, fn_type, arg_index, true);
6368		ResultLoc *no_result = no_result_loc();
6369		ir_build_reset_result(irb, scope, source_node, no_result);
6370		ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, arg_type, no_result);
6371
6372		IrInstSrc *arg = ir_gen_node_extra(irb, arg_node, scope, LValNone, &result_loc_cast->base);
6373		if (arg == irb->codegen->invalid_inst_src)
6374		return arg;
6375
6376		args[i] = ir_build_implicit_cast(irb, scope, arg_node, arg, result_loc_cast);
6377		}
6378
6379		IrInstSrc *fn_call;
6380		if (options != nullptr) {
6381		fn_call = ir_build_call_args(irb, scope, source_node, options, fn_ref, args, args_len, result_loc);
6382		} else {
6383		fn_call = ir_build_call_src(irb, scope, source_node, nullptr, fn_ref, args_len, args, nullptr,
6384		modifier, false, nullptr, result_loc);
6385		}
6386		return ir_lval_wrap(irb, scope, fn_call, lval, result_loc);
6387		}
6388
6389		static IrInstSrc ir_gen_builtin_fn_call(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
6390		ResultLoc *result_loc)
6391		{
6392		assert(node->type == NodeTypeFnCallExpr);
6393
6394		AstNode *fn_ref_expr = node->data.fn_call_expr.fn_ref_expr;
6395		Buf *name = fn_ref_expr->data.symbol_expr.symbol;
6396		auto entry = irb->codegen->builtin_fn_table.maybe_get(name);
6397
6398		if (!entry) {
6399		add_node_error(irb->codegen, node,
6400		buf_sprintf("invalid builtin function: '%s'", buf_ptr(name)));
6401		return irb->codegen->invalid_inst_src;
6402		}
6403
6404		BuiltinFnEntry *builtin_fn = entry->value;
6405		size_t actual_param_count = node->data.fn_call_expr.params.length;
6406
6407		if (builtin_fn->param_count != SIZE_MAX && builtin_fn->param_count != actual_param_count) {
6408		add_node_error(irb->codegen, node,
6409		buf_sprintf("expected %" ZIG_PRI_usize " argument(s), found %" ZIG_PRI_usize,
6410		builtin_fn->param_count, actual_param_count));
6411		return irb->codegen->invalid_inst_src;
6412		}
6413
6414		switch (builtin_fn->id) {
6415		case BuiltinFnIdInvalid:
6416		zig_unreachable();
6417		case BuiltinFnIdTypeof:
6418		{
6419		Scope *sub_scope = create_typeof_scope(irb->codegen, node, scope);
6420
6421		size_t arg_count = node->data.fn_call_expr.params.length;
6422
6423		IrInstSrc *type_of;
6424
6425		if (arg_count == 0) {
6426		add_node_error(irb->codegen, node,
6427		buf_sprintf("expected at least 1 argument, found 0"));
6428		return irb->codegen->invalid_inst_src;
6429		} else if (arg_count == 1) {
6430		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6431		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, sub_scope);
6432		if (arg0_value == irb->codegen->invalid_inst_src)
6433		return arg0_value;
6434
6435		type_of = ir_build_typeof_1(irb, scope, node, arg0_value);
6436		} else {
6437		IrInstSrc *args = heap::c_allocator.allocate<IrInstSrc>(arg_count);
6438		for (size_t i = 0; i < arg_count; i += 1) {
6439		AstNode *arg_node = node->data.fn_call_expr.params.at(i);
6440		IrInstSrc *arg = ir_gen_node(irb, arg_node, sub_scope);
6441		if (arg == irb->codegen->invalid_inst_src)
6442		return irb->codegen->invalid_inst_src;
6443		args[i] = arg;
6444		}
6445
6446		type_of = ir_build_typeof_n(irb, scope, node, args, arg_count);
6447		}
6448		return ir_lval_wrap(irb, scope, type_of, lval, result_loc);
6449		}
6450		case BuiltinFnIdSetCold:
6451		{
6452		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6453		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6454		if (arg0_value == irb->codegen->invalid_inst_src)
6455		return arg0_value;
6456
6457		IrInstSrc *set_cold = ir_build_set_cold(irb, scope, node, arg0_value);
6458		return ir_lval_wrap(irb, scope, set_cold, lval, result_loc);
6459		}
6460		case BuiltinFnIdSetRuntimeSafety:
6461		{
6462		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6463		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6464		if (arg0_value == irb->codegen->invalid_inst_src)
6465		return arg0_value;
6466
6467		IrInstSrc *set_safety = ir_build_set_runtime_safety(irb, scope, node, arg0_value);
6468		return ir_lval_wrap(irb, scope, set_safety, lval, result_loc);
6469		}
6470		case BuiltinFnIdSetFloatMode:
6471		{
6472		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6473		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6474		if (arg0_value == irb->codegen->invalid_inst_src)
6475		return arg0_value;
6476
6477		IrInstSrc *set_float_mode = ir_build_set_float_mode(irb, scope, node, arg0_value);
6478		return ir_lval_wrap(irb, scope, set_float_mode, lval, result_loc);
6479		}
6480		case BuiltinFnIdSizeof:
6481		case BuiltinFnIdBitSizeof:
6482		{
6483		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6484		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6485		if (arg0_value == irb->codegen->invalid_inst_src)
6486		return arg0_value;
6487
6488		IrInstSrc *size_of = ir_build_size_of(irb, scope, node, arg0_value, builtin_fn->id == BuiltinFnIdBitSizeof);
6489		return ir_lval_wrap(irb, scope, size_of, lval, result_loc);
6490		}
6491		case BuiltinFnIdImport:
6492		{
6493		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6494		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6495		if (arg0_value == irb->codegen->invalid_inst_src)
6496		return arg0_value;
6497
6498		IrInstSrc *import = ir_build_import(irb, scope, node, arg0_value);
6499		return ir_lval_wrap(irb, scope, import, lval, result_loc);
6500		}
6501		case BuiltinFnIdCImport:
6502		{
6503		IrInstSrc *c_import = ir_build_c_import(irb, scope, node);
6504		return ir_lval_wrap(irb, scope, c_import, lval, result_loc);
6505		}
6506		case BuiltinFnIdCInclude:
6507		{
6508		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6509		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6510		if (arg0_value == irb->codegen->invalid_inst_src)
6511		return arg0_value;
6512
6513		if (!exec_c_import_buf(irb->exec)) {
6514		add_node_error(irb->codegen, node, buf_sprintf("C include valid only inside C import block"));
6515		return irb->codegen->invalid_inst_src;
6516		}
6517
6518		IrInstSrc *c_include = ir_build_c_include(irb, scope, node, arg0_value);
6519		return ir_lval_wrap(irb, scope, c_include, lval, result_loc);
6520		}
6521		case BuiltinFnIdCDefine:
6522		{
6523		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6524		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6525		if (arg0_value == irb->codegen->invalid_inst_src)
6526		return arg0_value;
6527
6528		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6529		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6530		if (arg1_value == irb->codegen->invalid_inst_src)
6531		return arg1_value;
6532
6533		if (!exec_c_import_buf(irb->exec)) {
6534		add_node_error(irb->codegen, node, buf_sprintf("C define valid only inside C import block"));
6535		return irb->codegen->invalid_inst_src;
6536		}
6537
6538		IrInstSrc *c_define = ir_build_c_define(irb, scope, node, arg0_value, arg1_value);
6539		return ir_lval_wrap(irb, scope, c_define, lval, result_loc);
6540		}
6541		case BuiltinFnIdCUndef:
6542		{
6543		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6544		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6545		if (arg0_value == irb->codegen->invalid_inst_src)
6546		return arg0_value;
6547
6548		if (!exec_c_import_buf(irb->exec)) {
6549		add_node_error(irb->codegen, node, buf_sprintf("C undef valid only inside C import block"));
6550		return irb->codegen->invalid_inst_src;
6551		}
6552
6553		IrInstSrc *c_undef = ir_build_c_undef(irb, scope, node, arg0_value);
6554		return ir_lval_wrap(irb, scope, c_undef, lval, result_loc);
6555		}
6556		case BuiltinFnIdCompileErr:
6557		{
6558		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6559		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6560		if (arg0_value == irb->codegen->invalid_inst_src)
6561		return arg0_value;
6562
6563		IrInstSrc *compile_err = ir_build_compile_err(irb, scope, node, arg0_value);
6564		return ir_lval_wrap(irb, scope, compile_err, lval, result_loc);
6565		}
6566		case BuiltinFnIdCompileLog:
6567		{
6568		IrInstSrc *args = heap::c_allocator.allocate<IrInstSrc>(actual_param_count);
6569
6570		for (size_t i = 0; i < actual_param_count; i += 1) {
6571		AstNode *arg_node = node->data.fn_call_expr.params.at(i);
6572		args[i] = ir_gen_node(irb, arg_node, scope);
6573		if (args[i] == irb->codegen->invalid_inst_src)
6574		return irb->codegen->invalid_inst_src;
6575		}
6576
6577		IrInstSrc *compile_log = ir_build_compile_log(irb, scope, node, actual_param_count, args);
6578		return ir_lval_wrap(irb, scope, compile_log, lval, result_loc);
6579		}
6580		case BuiltinFnIdErrName:
6581		{
6582		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6583		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6584		if (arg0_value == irb->codegen->invalid_inst_src)
6585		return arg0_value;
6586
6587		IrInstSrc *err_name = ir_build_err_name(irb, scope, node, arg0_value);
6588		return ir_lval_wrap(irb, scope, err_name, lval, result_loc);
6589		}
6590		case BuiltinFnIdEmbedFile:
6591		{
6592		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6593		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6594		if (arg0_value == irb->codegen->invalid_inst_src)
6595		return arg0_value;
6596
6597		IrInstSrc *embed_file = ir_build_embed_file(irb, scope, node, arg0_value);
6598		return ir_lval_wrap(irb, scope, embed_file, lval, result_loc);
6599		}
6600		case BuiltinFnIdCmpxchgWeak:
6601		case BuiltinFnIdCmpxchgStrong:
6602		{
6603		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6604		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6605		if (arg0_value == irb->codegen->invalid_inst_src)
6606		return arg0_value;
6607
6608		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6609		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6610		if (arg1_value == irb->codegen->invalid_inst_src)
6611		return arg1_value;
6612
6613		AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
6614		IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
6615		if (arg2_value == irb->codegen->invalid_inst_src)
6616		return arg2_value;
6617
6618		AstNode *arg3_node = node->data.fn_call_expr.params.at(3);
6619		IrInstSrc *arg3_value = ir_gen_node(irb, arg3_node, scope);
6620		if (arg3_value == irb->codegen->invalid_inst_src)
6621		return arg3_value;
6622
6623		AstNode *arg4_node = node->data.fn_call_expr.params.at(4);
6624		IrInstSrc *arg4_value = ir_gen_node(irb, arg4_node, scope);
6625		if (arg4_value == irb->codegen->invalid_inst_src)
6626		return arg4_value;
6627
6628		AstNode *arg5_node = node->data.fn_call_expr.params.at(5);
6629		IrInstSrc *arg5_value = ir_gen_node(irb, arg5_node, scope);
6630		if (arg5_value == irb->codegen->invalid_inst_src)
6631		return arg5_value;
6632
6633		IrInstSrc *cmpxchg = ir_build_cmpxchg_src(irb, scope, node, arg0_value, arg1_value,
6634		arg2_value, arg3_value, arg4_value, arg5_value, (builtin_fn->id == BuiltinFnIdCmpxchgWeak),
6635		result_loc);
6636		return ir_lval_wrap(irb, scope, cmpxchg, lval, result_loc);
6637		}
6638		case BuiltinFnIdFence:
6639		{
6640		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6641		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6642		if (arg0_value == irb->codegen->invalid_inst_src)
6643		return arg0_value;
6644
6645		IrInstSrc *fence = ir_build_fence(irb, scope, node, arg0_value);
6646		return ir_lval_wrap(irb, scope, fence, lval, result_loc);
6647		}
6648		case BuiltinFnIdReduce:
6649		{
6650		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6651		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6652		if (arg0_value == irb->codegen->invalid_inst_src)
6653		return arg0_value;
6654
6655		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6656		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6657		if (arg1_value == irb->codegen->invalid_inst_src)
6658		return arg1_value;
6659
6660		IrInstSrc *reduce = ir_build_reduce(irb, scope, node, arg0_value, arg1_value);
6661		return ir_lval_wrap(irb, scope, reduce, lval, result_loc);
6662		}
6663		case BuiltinFnIdDivExact:
6664		{
6665		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6666		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6667		if (arg0_value == irb->codegen->invalid_inst_src)
6668		return arg0_value;
6669
6670		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6671		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6672		if (arg1_value == irb->codegen->invalid_inst_src)
6673		return arg1_value;
6674
6675		IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpDivExact, arg0_value, arg1_value, true);
6676		return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
6677		}
6678		case BuiltinFnIdDivTrunc:
6679		{
6680		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6681		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6682		if (arg0_value == irb->codegen->invalid_inst_src)
6683		return arg0_value;
6684
6685		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6686		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6687		if (arg1_value == irb->codegen->invalid_inst_src)
6688		return arg1_value;
6689
6690		IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpDivTrunc, arg0_value, arg1_value, true);
6691		return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
6692		}
6693		case BuiltinFnIdDivFloor:
6694		{
6695		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6696		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6697		if (arg0_value == irb->codegen->invalid_inst_src)
6698		return arg0_value;
6699
6700		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6701		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6702		if (arg1_value == irb->codegen->invalid_inst_src)
6703		return arg1_value;
6704
6705		IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpDivFloor, arg0_value, arg1_value, true);
6706		return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
6707		}
6708		case BuiltinFnIdRem:
6709		{
6710		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6711		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6712		if (arg0_value == irb->codegen->invalid_inst_src)
6713		return arg0_value;
6714
6715		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6716		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6717		if (arg1_value == irb->codegen->invalid_inst_src)
6718		return arg1_value;
6719
6720		IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpRemRem, arg0_value, arg1_value, true);
6721		return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
6722		}
6723		case BuiltinFnIdMod:
6724		{
6725		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6726		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6727		if (arg0_value == irb->codegen->invalid_inst_src)
6728		return arg0_value;
6729
6730		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6731		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6732		if (arg1_value == irb->codegen->invalid_inst_src)
6733		return arg1_value;
6734
6735		IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpRemMod, arg0_value, arg1_value, true);
6736		return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
6737		}
6738		case BuiltinFnIdSqrt:
6739		case BuiltinFnIdSin:
6740		case BuiltinFnIdCos:
6741		case BuiltinFnIdExp:
6742		case BuiltinFnIdExp2:
6743		case BuiltinFnIdLog:
6744		case BuiltinFnIdLog2:
6745		case BuiltinFnIdLog10:
6746		case BuiltinFnIdFabs:
6747		case BuiltinFnIdFloor:
6748		case BuiltinFnIdCeil:
6749		case BuiltinFnIdTrunc:
6750		case BuiltinFnIdNearbyInt:
6751		case BuiltinFnIdRound:
6752		{
6753		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6754		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6755		if (arg0_value == irb->codegen->invalid_inst_src)
6756		return arg0_value;
6757
6758		IrInstSrc *inst = ir_build_float_op_src(irb, scope, node, arg0_value, builtin_fn->id);
6759		return ir_lval_wrap(irb, scope, inst, lval, result_loc);
6760		}
6761		case BuiltinFnIdTruncate:
6762		{
6763		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6764		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6765		if (arg0_value == irb->codegen->invalid_inst_src)
6766		return arg0_value;
6767
6768		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6769		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6770		if (arg1_value == irb->codegen->invalid_inst_src)
6771		return arg1_value;
6772
6773		IrInstSrc *truncate = ir_build_truncate(irb, scope, node, arg0_value, arg1_value);
6774		return ir_lval_wrap(irb, scope, truncate, lval, result_loc);
6775		}
6776		case BuiltinFnIdIntCast:
6777		{
6778		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6779		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6780		if (arg0_value == irb->codegen->invalid_inst_src)
6781		return arg0_value;
6782
6783		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6784		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6785		if (arg1_value == irb->codegen->invalid_inst_src)
6786		return arg1_value;
6787
6788		IrInstSrc *result = ir_build_int_cast(irb, scope, node, arg0_value, arg1_value);
6789		return ir_lval_wrap(irb, scope, result, lval, result_loc);
6790		}
6791		case BuiltinFnIdFloatCast:
6792		{
6793		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6794		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6795		if (arg0_value == irb->codegen->invalid_inst_src)
6796		return arg0_value;
6797
6798		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6799		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6800		if (arg1_value == irb->codegen->invalid_inst_src)
6801		return arg1_value;
6802
6803		IrInstSrc *result = ir_build_float_cast(irb, scope, node, arg0_value, arg1_value);
6804		return ir_lval_wrap(irb, scope, result, lval, result_loc);
6805		}
6806		case BuiltinFnIdErrSetCast:
6807		{
6808		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6809		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6810		if (arg0_value == irb->codegen->invalid_inst_src)
6811		return arg0_value;
6812
6813		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6814		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6815		if (arg1_value == irb->codegen->invalid_inst_src)
6816		return arg1_value;
6817
6818		IrInstSrc *result = ir_build_err_set_cast(irb, scope, node, arg0_value, arg1_value);
6819		return ir_lval_wrap(irb, scope, result, lval, result_loc);
6820		}
6821		case BuiltinFnIdIntToFloat:
6822		{
6823		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6824		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6825		if (arg0_value == irb->codegen->invalid_inst_src)
6826		return arg0_value;
6827
6828		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6829		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6830		if (arg1_value == irb->codegen->invalid_inst_src)
6831		return arg1_value;
6832
6833		IrInstSrc *result = ir_build_int_to_float(irb, scope, node, arg0_value, arg1_value);
6834		return ir_lval_wrap(irb, scope, result, lval, result_loc);
6835		}
6836		case BuiltinFnIdFloatToInt:
6837		{
6838		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6839		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6840		if (arg0_value == irb->codegen->invalid_inst_src)
6841		return arg0_value;
6842
6843		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6844		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6845		if (arg1_value == irb->codegen->invalid_inst_src)
6846		return arg1_value;
6847
6848		IrInstSrc *result = ir_build_float_to_int(irb, scope, node, arg0_value, arg1_value);
6849		return ir_lval_wrap(irb, scope, result, lval, result_loc);
6850		}
6851		case BuiltinFnIdErrToInt:
6852		{
6853		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6854		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6855		if (arg0_value == irb->codegen->invalid_inst_src)
6856		return arg0_value;
6857
6858		IrInstSrc *result = ir_build_err_to_int_src(irb, scope, node, arg0_value);
6859		return ir_lval_wrap(irb, scope, result, lval, result_loc);
6860		}
6861		case BuiltinFnIdIntToErr:
6862		{
6863		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6864		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6865		if (arg0_value == irb->codegen->invalid_inst_src)
6866		return arg0_value;
6867
6868		IrInstSrc *result = ir_build_int_to_err_src(irb, scope, node, arg0_value);
6869		return ir_lval_wrap(irb, scope, result, lval, result_loc);
6870		}
6871		case BuiltinFnIdBoolToInt:
6872		{
6873		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6874		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6875		if (arg0_value == irb->codegen->invalid_inst_src)
6876		return arg0_value;
6877
6878		IrInstSrc *result = ir_build_bool_to_int(irb, scope, node, arg0_value);
6879		return ir_lval_wrap(irb, scope, result, lval, result_loc);
6880		}
6881		case BuiltinFnIdVectorType:
6882		{
6883		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6884		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6885		if (arg0_value == irb->codegen->invalid_inst_src)
6886		return arg0_value;
6887
6888		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6889		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6890		if (arg1_value == irb->codegen->invalid_inst_src)
6891		return arg1_value;
6892
6893		IrInstSrc *vector_type = ir_build_vector_type(irb, scope, node, arg0_value, arg1_value);
6894		return ir_lval_wrap(irb, scope, vector_type, lval, result_loc);
6895		}
6896		case BuiltinFnIdShuffle:
6897		{
6898		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6899		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6900		if (arg0_value == irb->codegen->invalid_inst_src)
6901		return arg0_value;
6902
6903		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6904		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6905		if (arg1_value == irb->codegen->invalid_inst_src)
6906		return arg1_value;
6907
6908		AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
6909		IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
6910		if (arg2_value == irb->codegen->invalid_inst_src)
6911		return arg2_value;
6912
6913		AstNode *arg3_node = node->data.fn_call_expr.params.at(3);
6914		IrInstSrc *arg3_value = ir_gen_node(irb, arg3_node, scope);
6915		if (arg3_value == irb->codegen->invalid_inst_src)
6916		return arg3_value;
6917
6918		IrInstSrc *shuffle_vector = ir_build_shuffle_vector(irb, scope, node,
6919		arg0_value, arg1_value, arg2_value, arg3_value);
6920		return ir_lval_wrap(irb, scope, shuffle_vector, lval, result_loc);
6921		}
6922		case BuiltinFnIdSplat:
6923		{
6924		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6925		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6926		if (arg0_value == irb->codegen->invalid_inst_src)
6927		return arg0_value;
6928
6929		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6930		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6931		if (arg1_value == irb->codegen->invalid_inst_src)
6932		return arg1_value;
6933
6934		IrInstSrc *splat = ir_build_splat_src(irb, scope, node,
6935		arg0_value, arg1_value);
6936		return ir_lval_wrap(irb, scope, splat, lval, result_loc);
6937		}
6938		case BuiltinFnIdMemcpy:
6939		{
6940		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6941		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6942		if (arg0_value == irb->codegen->invalid_inst_src)
6943		return arg0_value;
6944
6945		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6946		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6947		if (arg1_value == irb->codegen->invalid_inst_src)
6948		return arg1_value;
6949
6950		AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
6951		IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
6952		if (arg2_value == irb->codegen->invalid_inst_src)
6953		return arg2_value;
6954
6955		IrInstSrc *ir_memcpy = ir_build_memcpy_src(irb, scope, node, arg0_value, arg1_value, arg2_value);
6956		return ir_lval_wrap(irb, scope, ir_memcpy, lval, result_loc);
6957		}
6958		case BuiltinFnIdMemset:
6959		{
6960		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6961		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6962		if (arg0_value == irb->codegen->invalid_inst_src)
6963		return arg0_value;
6964
6965		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6966		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6967		if (arg1_value == irb->codegen->invalid_inst_src)
6968		return arg1_value;
6969
6970		AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
6971		IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
6972		if (arg2_value == irb->codegen->invalid_inst_src)
6973		return arg2_value;
6974
6975		IrInstSrc *ir_memset = ir_build_memset_src(irb, scope, node, arg0_value, arg1_value, arg2_value);
6976		return ir_lval_wrap(irb, scope, ir_memset, lval, result_loc);
6977		}
6978		case BuiltinFnIdWasmMemorySize:
6979		{
6980		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6981		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6982		if (arg0_value == irb->codegen->invalid_inst_src)
6983		return arg0_value;
6984
6985		IrInstSrc *ir_wasm_memory_size = ir_build_wasm_memory_size_src(irb, scope, node, arg0_value);
6986		return ir_lval_wrap(irb, scope, ir_wasm_memory_size, lval, result_loc);
6987		}
6988		case BuiltinFnIdWasmMemoryGrow:
6989		{
6990		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6991		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6992		if (arg0_value == irb->codegen->invalid_inst_src)
6993		return arg0_value;
6994
6995		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6996		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6997		if (arg1_value == irb->codegen->invalid_inst_src)
6998		return arg1_value;
6999
7000		IrInstSrc *ir_wasm_memory_grow = ir_build_wasm_memory_grow_src(irb, scope, node, arg0_value, arg1_value);
7001		return ir_lval_wrap(irb, scope, ir_wasm_memory_grow, lval, result_loc);
7002		}
7003		case BuiltinFnIdField:
7004		{
7005		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7006		IrInstSrc *arg0_value = ir_gen_node_extra(irb, arg0_node, scope, LValPtr, nullptr);
7007		if (arg0_value == irb->codegen->invalid_inst_src)
7008		return arg0_value;
7009
7010		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7011		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7012		if (arg1_value == irb->codegen->invalid_inst_src)
7013		return arg1_value;
7014
7015		IrInstSrc *ptr_instruction = ir_build_field_ptr_instruction(irb, scope, node,
7016		arg0_value, arg1_value, false);
7017
7018		if (lval == LValPtr \|\| lval == LValAssign)
7019		return ptr_instruction;
7020
7021		IrInstSrc *load_ptr = ir_build_load_ptr(irb, scope, node, ptr_instruction);
7022		return ir_expr_wrap(irb, scope, load_ptr, result_loc);
7023		}
7024		case BuiltinFnIdHasField:
7025		{
7026		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7027		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7028		if (arg0_value == irb->codegen->invalid_inst_src)
7029		return arg0_value;
7030
7031		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7032		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7033		if (arg1_value == irb->codegen->invalid_inst_src)
7034		return arg1_value;
7035
7036		IrInstSrc *type_info = ir_build_has_field(irb, scope, node, arg0_value, arg1_value);
7037		return ir_lval_wrap(irb, scope, type_info, lval, result_loc);
7038		}
7039		case BuiltinFnIdTypeInfo:
7040		{
7041		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7042		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7043		if (arg0_value == irb->codegen->invalid_inst_src)
7044		return arg0_value;
7045
7046		IrInstSrc *type_info = ir_build_type_info(irb, scope, node, arg0_value);
7047		return ir_lval_wrap(irb, scope, type_info, lval, result_loc);
7048		}
7049		case BuiltinFnIdType:
7050		{
7051		AstNode *arg_node = node->data.fn_call_expr.params.at(0);
7052		IrInstSrc *arg = ir_gen_node(irb, arg_node, scope);
7053		if (arg == irb->codegen->invalid_inst_src)
7054		return arg;
7055
7056		IrInstSrc *type = ir_build_type(irb, scope, node, arg);
7057		return ir_lval_wrap(irb, scope, type, lval, result_loc);
7058		}
7059		case BuiltinFnIdBreakpoint:
7060		return ir_lval_wrap(irb, scope, ir_build_breakpoint(irb, scope, node), lval, result_loc);
7061		case BuiltinFnIdReturnAddress:
7062		return ir_lval_wrap(irb, scope, ir_build_return_address_src(irb, scope, node), lval, result_loc);
7063		case BuiltinFnIdFrameAddress:
7064		return ir_lval_wrap(irb, scope, ir_build_frame_address_src(irb, scope, node), lval, result_loc);
7065		case BuiltinFnIdFrameHandle:
7066		if (!irb->exec->fn_entry) {
7067		add_node_error(irb->codegen, node, buf_sprintf("@frame() called outside of function definition"));
7068		return irb->codegen->invalid_inst_src;
7069		}
7070		return ir_lval_wrap(irb, scope, ir_build_handle_src(irb, scope, node), lval, result_loc);
7071		case BuiltinFnIdFrameType: {
7072		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7073		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7074		if (arg0_value == irb->codegen->invalid_inst_src)
7075		return arg0_value;
7076
7077		IrInstSrc *frame_type = ir_build_frame_type(irb, scope, node, arg0_value);
7078		return ir_lval_wrap(irb, scope, frame_type, lval, result_loc);
7079		}
7080		case BuiltinFnIdFrameSize: {
7081		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7082		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7083		if (arg0_value == irb->codegen->invalid_inst_src)
7084		return arg0_value;
7085
7086		IrInstSrc *frame_size = ir_build_frame_size_src(irb, scope, node, arg0_value);
7087		return ir_lval_wrap(irb, scope, frame_size, lval, result_loc);
7088		}
7089		case BuiltinFnIdAlignOf:
7090		{
7091		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7092		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7093		if (arg0_value == irb->codegen->invalid_inst_src)
7094		return arg0_value;
7095
7096		IrInstSrc *align_of = ir_build_align_of(irb, scope, node, arg0_value);
7097		return ir_lval_wrap(irb, scope, align_of, lval, result_loc);
7098		}
7099		case BuiltinFnIdAddWithOverflow:
7100		return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpAdd), lval, result_loc);
7101		case BuiltinFnIdSubWithOverflow:
7102		return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpSub), lval, result_loc);
7103		case BuiltinFnIdMulWithOverflow:
7104		return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpMul), lval, result_loc);
7105		case BuiltinFnIdShlWithOverflow:
7106		return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpShl), lval, result_loc);
7107		case BuiltinFnIdMulAdd:
7108		return ir_lval_wrap(irb, scope, ir_gen_mul_add(irb, scope, node), lval, result_loc);
7109		case BuiltinFnIdTypeName:
7110		{
7111		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7112		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7113		if (arg0_value == irb->codegen->invalid_inst_src)
7114		return arg0_value;
7115
7116		IrInstSrc *type_name = ir_build_type_name(irb, scope, node, arg0_value);
7117		return ir_lval_wrap(irb, scope, type_name, lval, result_loc);
7118		}
7119		case BuiltinFnIdPanic:
7120		{
7121		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7122		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7123		if (arg0_value == irb->codegen->invalid_inst_src)
7124		return arg0_value;
7125
7126		IrInstSrc *panic = ir_build_panic_src(irb, scope, node, arg0_value);
7127		return ir_lval_wrap(irb, scope, panic, lval, result_loc);
7128		}
7129		case BuiltinFnIdPtrCast:
7130		{
7131		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7132		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7133		if (arg0_value == irb->codegen->invalid_inst_src)
7134		return arg0_value;
7135
7136		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7137		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7138		if (arg1_value == irb->codegen->invalid_inst_src)
7139		return arg1_value;
7140
7141		IrInstSrc *ptr_cast = ir_build_ptr_cast_src(irb, scope, node, arg0_value, arg1_value, true);
7142		return ir_lval_wrap(irb, scope, ptr_cast, lval, result_loc);
7143		}
7144		case BuiltinFnIdBitCast:
7145		{
7146		AstNode *dest_type_node = node->data.fn_call_expr.params.at(0);
7147		IrInstSrc *dest_type = ir_gen_node(irb, dest_type_node, scope);
7148		if (dest_type == irb->codegen->invalid_inst_src)
7149		return dest_type;
7150
7151		ResultLocBitCast *result_loc_bit_cast = heap::c_allocator.create<ResultLocBitCast>();
7152		result_loc_bit_cast->base.id = ResultLocIdBitCast;
7153		result_loc_bit_cast->base.source_instruction = dest_type;
7154		result_loc_bit_cast->base.allow_write_through_const = result_loc->allow_write_through_const;
7155		ir_ref_instruction(dest_type, irb->current_basic_block);
7156		result_loc_bit_cast->parent = result_loc;
7157
7158		ir_build_reset_result(irb, scope, node, &result_loc_bit_cast->base);
7159
7160		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7161		IrInstSrc *arg1_value = ir_gen_node_extra(irb, arg1_node, scope, LValNone,
7162		&result_loc_bit_cast->base);
7163		if (arg1_value == irb->codegen->invalid_inst_src)
7164		return arg1_value;
7165
7166		IrInstSrc *bitcast = ir_build_bit_cast_src(irb, scope, arg1_node, arg1_value, result_loc_bit_cast);
7167		return ir_lval_wrap(irb, scope, bitcast, lval, result_loc);
7168		}
7169		case BuiltinFnIdAs:
7170		{
7171		AstNode *dest_type_node = node->data.fn_call_expr.params.at(0);
7172		IrInstSrc *dest_type = ir_gen_node(irb, dest_type_node, scope);
7173		if (dest_type == irb->codegen->invalid_inst_src)
7174		return dest_type;
7175
7176		ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, dest_type, result_loc);
7177
7178		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7179		IrInstSrc *arg1_value = ir_gen_node_extra(irb, arg1_node, scope, LValNone,
7180		&result_loc_cast->base);
7181		if (arg1_value == irb->codegen->invalid_inst_src)
7182		return arg1_value;
7183
7184		IrInstSrc *result = ir_build_implicit_cast(irb, scope, node, arg1_value, result_loc_cast);
7185		return ir_lval_wrap(irb, scope, result, lval, result_loc);
7186		}
7187		case BuiltinFnIdIntToPtr:
7188		{
7189		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7190		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7191		if (arg0_value == irb->codegen->invalid_inst_src)
7192		return arg0_value;
7193
7194		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7195		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7196		if (arg1_value == irb->codegen->invalid_inst_src)
7197		return arg1_value;
7198
7199		IrInstSrc *int_to_ptr = ir_build_int_to_ptr_src(irb, scope, node, arg0_value, arg1_value);
7200		return ir_lval_wrap(irb, scope, int_to_ptr, lval, result_loc);
7201		}
7202		case BuiltinFnIdPtrToInt:
7203		{
7204		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7205		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7206		if (arg0_value == irb->codegen->invalid_inst_src)
7207		return arg0_value;
7208
7209		IrInstSrc *ptr_to_int = ir_build_ptr_to_int_src(irb, scope, node, arg0_value);
7210		return ir_lval_wrap(irb, scope, ptr_to_int, lval, result_loc);
7211		}
7212		case BuiltinFnIdTagName:
7213		{
7214		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7215		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7216		if (arg0_value == irb->codegen->invalid_inst_src)
7217		return arg0_value;
7218
7219		IrInstSrc *tag_name = ir_build_tag_name_src(irb, scope, node, arg0_value);
7220		return ir_lval_wrap(irb, scope, tag_name, lval, result_loc);
7221		}
7222		case BuiltinFnIdFieldParentPtr:
7223		{
7224		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7225		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7226		if (arg0_value == irb->codegen->invalid_inst_src)
7227		return arg0_value;
7228
7229		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7230		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7231		if (arg1_value == irb->codegen->invalid_inst_src)
7232		return arg1_value;
7233
7234		AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
7235		IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
7236		if (arg2_value == irb->codegen->invalid_inst_src)
7237		return arg2_value;
7238
7239		IrInstSrc *field_parent_ptr = ir_build_field_parent_ptr_src(irb, scope, node,
7240		arg0_value, arg1_value, arg2_value);
7241		return ir_lval_wrap(irb, scope, field_parent_ptr, lval, result_loc);
7242		}
7243		case BuiltinFnIdByteOffsetOf:
7244		{
7245		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7246		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7247		if (arg0_value == irb->codegen->invalid_inst_src)
7248		return arg0_value;
7249
7250		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7251		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7252		if (arg1_value == irb->codegen->invalid_inst_src)
7253		return arg1_value;
7254
7255		IrInstSrc *offset_of = ir_build_byte_offset_of(irb, scope, node, arg0_value, arg1_value);
7256		return ir_lval_wrap(irb, scope, offset_of, lval, result_loc);
7257		}
7258		case BuiltinFnIdBitOffsetOf:
7259		{
7260		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7261		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7262		if (arg0_value == irb->codegen->invalid_inst_src)
7263		return arg0_value;
7264
7265		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7266		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7267		if (arg1_value == irb->codegen->invalid_inst_src)
7268		return arg1_value;
7269
7270		IrInstSrc *offset_of = ir_build_bit_offset_of(irb, scope, node, arg0_value, arg1_value);
7271		return ir_lval_wrap(irb, scope, offset_of, lval, result_loc);
7272		}
7273		case BuiltinFnIdCall: {
7274		// Cast the options parameter to the options type
7275		ZigType *options_type = get_builtin_type(irb->codegen, "CallOptions");
7276		IrInstSrc *options_type_inst = ir_build_const_type(irb, scope, node, options_type);
7277		ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, options_type_inst, no_result_loc());
7278
7279		AstNode *options_node = node->data.fn_call_expr.params.at(0);
7280		IrInstSrc *options_inner = ir_gen_node_extra(irb, options_node, scope,
7281		LValNone, &result_loc_cast->base);
7282		if (options_inner == irb->codegen->invalid_inst_src)
7283		return options_inner;
7284		IrInstSrc *options = ir_build_implicit_cast(irb, scope, options_node, options_inner, result_loc_cast);
7285
7286		AstNode *fn_ref_node = node->data.fn_call_expr.params.at(1);
7287		AstNode *args_node = node->data.fn_call_expr.params.at(2);
7288		if (args_node->type == NodeTypeContainerInitExpr) {
7289		if (args_node->data.container_init_expr.kind == ContainerInitKindArray \|\|
7290		args_node->data.container_init_expr.entries.length == 0)
7291		{
7292		return ir_gen_fn_call_with_args(irb, scope, node,
7293		fn_ref_node, CallModifierNone, options,
7294		args_node->data.container_init_expr.entries.items,
7295		args_node->data.container_init_expr.entries.length,
7296		lval, result_loc);
7297		} else {
7298		exec_add_error_node(irb->codegen, irb->exec, args_node,
7299		buf_sprintf("TODO: @call with anon struct literal"));
7300		return irb->codegen->invalid_inst_src;
7301		}
7302		} else {
7303		IrInstSrc *fn_ref = ir_gen_node(irb, fn_ref_node, scope);
7304		if (fn_ref == irb->codegen->invalid_inst_src)
7305		return fn_ref;
7306
7307		IrInstSrc *args = ir_gen_node(irb, args_node, scope);
7308		if (args == irb->codegen->invalid_inst_src)
7309		return args;
7310
7311		IrInstSrc *call = ir_build_call_extra(irb, scope, node, options, fn_ref, args, result_loc);
7312		return ir_lval_wrap(irb, scope, call, lval, result_loc);
7313		}
7314		}
7315		case BuiltinFnIdAsyncCall:
7316		return ir_gen_async_call(irb, scope, nullptr, node, lval, result_loc);
7317		case BuiltinFnIdShlExact:
7318		{
7319		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7320		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7321		if (arg0_value == irb->codegen->invalid_inst_src)
7322		return arg0_value;
7323
7324		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7325		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7326		if (arg1_value == irb->codegen->invalid_inst_src)
7327		return arg1_value;
7328
7329		IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpBitShiftLeftExact, arg0_value, arg1_value, true);
7330		return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
7331		}
7332		case BuiltinFnIdShrExact:
7333		{
7334		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7335		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7336		if (arg0_value == irb->codegen->invalid_inst_src)
7337		return arg0_value;
7338
7339		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7340		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7341		if (arg1_value == irb->codegen->invalid_inst_src)
7342		return arg1_value;
7343
7344		IrInstSrc *bin_op = ir_build_bin_op(irb, scope, node, IrBinOpBitShiftRightExact, arg0_value, arg1_value, true);
7345		return ir_lval_wrap(irb, scope, bin_op, lval, result_loc);
7346		}
7347		case BuiltinFnIdSetEvalBranchQuota:
7348		{
7349		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7350		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7351		if (arg0_value == irb->codegen->invalid_inst_src)
7352		return arg0_value;
7353
7354		IrInstSrc *set_eval_branch_quota = ir_build_set_eval_branch_quota(irb, scope, node, arg0_value);
7355		return ir_lval_wrap(irb, scope, set_eval_branch_quota, lval, result_loc);
7356		}
7357		case BuiltinFnIdAlignCast:
7358		{
7359		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7360		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7361		if (arg0_value == irb->codegen->invalid_inst_src)
7362		return arg0_value;
7363
7364		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7365		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7366		if (arg1_value == irb->codegen->invalid_inst_src)
7367		return arg1_value;
7368
7369		IrInstSrc *align_cast = ir_build_align_cast_src(irb, scope, node, arg0_value, arg1_value);
7370		return ir_lval_wrap(irb, scope, align_cast, lval, result_loc);
7371		}
7372		case BuiltinFnIdThis:
7373		{
7374		IrInstSrc *this_inst = ir_gen_this(irb, scope, node);
7375		return ir_lval_wrap(irb, scope, this_inst, lval, result_loc);
7376		}
7377		case BuiltinFnIdSetAlignStack:
7378		{
7379		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7380		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7381		if (arg0_value == irb->codegen->invalid_inst_src)
7382		return arg0_value;
7383
7384		IrInstSrc *set_align_stack = ir_build_set_align_stack(irb, scope, node, arg0_value);
7385		return ir_lval_wrap(irb, scope, set_align_stack, lval, result_loc);
7386		}
7387		case BuiltinFnIdExport:
7388		{
7389		// Cast the options parameter to the options type
7390		ZigType *options_type = get_builtin_type(irb->codegen, "ExportOptions");
7391		IrInstSrc *options_type_inst = ir_build_const_type(irb, scope, node, options_type);
7392		ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, options_type_inst, no_result_loc());
7393
7394		AstNode *target_node = node->data.fn_call_expr.params.at(0);
7395		IrInstSrc *target_value = ir_gen_node(irb, target_node, scope);
7396		if (target_value == irb->codegen->invalid_inst_src)
7397		return target_value;
7398
7399		AstNode *options_node = node->data.fn_call_expr.params.at(1);
7400		IrInstSrc *options_value = ir_gen_node_extra(irb, options_node,
7401		scope, LValNone, &result_loc_cast->base);
7402		if (options_value == irb->codegen->invalid_inst_src)
7403		return options_value;
7404
7405		IrInstSrc *casted_options_value = ir_build_implicit_cast(
7406		irb, scope, options_node, options_value, result_loc_cast);
7407
7408		IrInstSrc *ir_export = ir_build_export(irb, scope, node, target_value, casted_options_value);
7409		return ir_lval_wrap(irb, scope, ir_export, lval, result_loc);
7410		}
7411		case BuiltinFnIdExtern:
7412		{
7413		// Cast the options parameter to the options type
7414		ZigType *options_type = get_builtin_type(irb->codegen, "ExternOptions");
7415		IrInstSrc *options_type_inst = ir_build_const_type(irb, scope, node, options_type);
7416		ResultLocCast *result_loc_cast = ir_build_cast_result_loc(irb, options_type_inst, no_result_loc());
7417
7418		AstNode *type_node = node->data.fn_call_expr.params.at(0);
7419		IrInstSrc *type_value = ir_gen_node(irb, type_node, scope);
7420		if (type_value == irb->codegen->invalid_inst_src)
7421		return type_value;
7422
7423		AstNode *options_node = node->data.fn_call_expr.params.at(1);
7424		IrInstSrc *options_value = ir_gen_node_extra(irb, options_node,
7425		scope, LValNone, &result_loc_cast->base);
7426		if (options_value == irb->codegen->invalid_inst_src)
7427		return options_value;
7428
7429		IrInstSrc *casted_options_value = ir_build_implicit_cast(
7430		irb, scope, options_node, options_value, result_loc_cast);
7431
7432		IrInstSrc *ir_extern = ir_build_extern(irb, scope, node, type_value, casted_options_value);
7433		return ir_lval_wrap(irb, scope, ir_extern, lval, result_loc);
7434		}
7435		case BuiltinFnIdErrorReturnTrace:
7436		{
7437		IrInstSrc *error_return_trace = ir_build_error_return_trace_src(irb, scope, node,
7438		IrInstErrorReturnTraceNull);
7439		return ir_lval_wrap(irb, scope, error_return_trace, lval, result_loc);
7440		}
7441		case BuiltinFnIdAtomicRmw:
7442		{
7443		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7444		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7445		if (arg0_value == irb->codegen->invalid_inst_src)
7446		return arg0_value;
7447
7448		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7449		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7450		if (arg1_value == irb->codegen->invalid_inst_src)
7451		return arg1_value;
7452
7453		AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
7454		IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
7455		if (arg2_value == irb->codegen->invalid_inst_src)
7456		return arg2_value;
7457
7458		AstNode *arg3_node = node->data.fn_call_expr.params.at(3);
7459		IrInstSrc *arg3_value = ir_gen_node(irb, arg3_node, scope);
7460		if (arg3_value == irb->codegen->invalid_inst_src)
7461		return arg3_value;
7462
7463		AstNode *arg4_node = node->data.fn_call_expr.params.at(4);
7464		IrInstSrc *arg4_value = ir_gen_node(irb, arg4_node, scope);
7465		if (arg4_value == irb->codegen->invalid_inst_src)
7466		return arg4_value;
7467
7468		IrInstSrc *inst = ir_build_atomic_rmw_src(irb, scope, node,
7469		arg0_value, arg1_value, arg2_value, arg3_value, arg4_value);
7470		return ir_lval_wrap(irb, scope, inst, lval, result_loc);
7471		}
7472		case BuiltinFnIdAtomicLoad:
7473		{
7474		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7475		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7476		if (arg0_value == irb->codegen->invalid_inst_src)
7477		return arg0_value;
7478
7479		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7480		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7481		if (arg1_value == irb->codegen->invalid_inst_src)
7482		return arg1_value;
7483
7484		AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
7485		IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
7486		if (arg2_value == irb->codegen->invalid_inst_src)
7487		return arg2_value;
7488
7489		IrInstSrc *inst = ir_build_atomic_load_src(irb, scope, node, arg0_value, arg1_value, arg2_value);
7490		return ir_lval_wrap(irb, scope, inst, lval, result_loc);
7491		}
7492		case BuiltinFnIdAtomicStore:
7493		{
7494		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7495		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7496		if (arg0_value == irb->codegen->invalid_inst_src)
7497		return arg0_value;
7498
7499		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7500		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7501		if (arg1_value == irb->codegen->invalid_inst_src)
7502		return arg1_value;
7503
7504		AstNode *arg2_node = node->data.fn_call_expr.params.at(2);
7505		IrInstSrc *arg2_value = ir_gen_node(irb, arg2_node, scope);
7506		if (arg2_value == irb->codegen->invalid_inst_src)
7507		return arg2_value;
7508
7509		AstNode *arg3_node = node->data.fn_call_expr.params.at(3);
7510		IrInstSrc *arg3_value = ir_gen_node(irb, arg3_node, scope);
7511		if (arg3_value == irb->codegen->invalid_inst_src)
7512		return arg3_value;
7513
7514		IrInstSrc *inst = ir_build_atomic_store_src(irb, scope, node, arg0_value, arg1_value,
7515		arg2_value, arg3_value);
7516		return ir_lval_wrap(irb, scope, inst, lval, result_loc);
7517		}
7518		case BuiltinFnIdIntToEnum:
7519		{
7520		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7521		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7522		if (arg0_value == irb->codegen->invalid_inst_src)
7523		return arg0_value;
7524
7525		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7526		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7527		if (arg1_value == irb->codegen->invalid_inst_src)
7528		return arg1_value;
7529
7530		IrInstSrc *result = ir_build_int_to_enum_src(irb, scope, node, arg0_value, arg1_value);
7531		return ir_lval_wrap(irb, scope, result, lval, result_loc);
7532		}
7533		case BuiltinFnIdEnumToInt:
7534		{
7535		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7536		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7537		if (arg0_value == irb->codegen->invalid_inst_src)
7538		return arg0_value;
7539
7540		IrInstSrc *result = ir_build_enum_to_int(irb, scope, node, arg0_value);
7541		return ir_lval_wrap(irb, scope, result, lval, result_loc);
7542		}
7543		case BuiltinFnIdCtz:
7544		case BuiltinFnIdPopCount:
7545		case BuiltinFnIdClz:
7546		case BuiltinFnIdBswap:
7547		case BuiltinFnIdBitReverse:
7548		{
7549		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7550		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7551		if (arg0_value == irb->codegen->invalid_inst_src)
7552		return arg0_value;
7553
7554		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7555		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7556		if (arg1_value == irb->codegen->invalid_inst_src)
7557		return arg1_value;
7558
7559		IrInstSrc *result;
7560		switch (builtin_fn->id) {
7561		case BuiltinFnIdCtz:
7562		result = ir_build_ctz(irb, scope, node, arg0_value, arg1_value);
7563		break;
7564		case BuiltinFnIdPopCount:
7565		result = ir_build_pop_count(irb, scope, node, arg0_value, arg1_value);
7566		break;
7567		case BuiltinFnIdClz:
7568		result = ir_build_clz(irb, scope, node, arg0_value, arg1_value);
7569		break;
7570		case BuiltinFnIdBswap:
7571		result = ir_build_bswap(irb, scope, node, arg0_value, arg1_value);
7572		break;
7573		case BuiltinFnIdBitReverse:
7574		result = ir_build_bit_reverse(irb, scope, node, arg0_value, arg1_value);
7575		break;
7576		default:
7577		zig_unreachable();
7578		}
7579		return ir_lval_wrap(irb, scope, result, lval, result_loc);
7580		}
7581		case BuiltinFnIdHasDecl:
7582		{
7583		AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
7584		IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
7585		if (arg0_value == irb->codegen->invalid_inst_src)
7586		return arg0_value;
7587
7588		AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
7589		IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
7590		if (arg1_value == irb->codegen->invalid_inst_src)
7591		return arg1_value;
7592
7593		IrInstSrc *has_decl = ir_build_has_decl(irb, scope, node, arg0_value, arg1_value);
7594		return ir_lval_wrap(irb, scope, has_decl, lval, result_loc);
7595		}
7596		case BuiltinFnIdUnionInit:
7597		{
7598		AstNode *union_type_node = node->data.fn_call_expr.params.at(0);
7599		IrInstSrc *union_type_inst = ir_gen_node(irb, union_type_node, scope);
7600		if (union_type_inst == irb->codegen->invalid_inst_src)
7601		return union_type_inst;
7602
7603		AstNode *name_node = node->data.fn_call_expr.params.at(1);
7604		IrInstSrc *name_inst = ir_gen_node(irb, name_node, scope);
7605		if (name_inst == irb->codegen->invalid_inst_src)
7606		return name_inst;
7607
7608		AstNode *init_node = node->data.fn_call_expr.params.at(2);
7609
7610		return ir_gen_union_init_expr(irb, scope, node, union_type_inst, name_inst, init_node,
7611		lval, result_loc);
7612		}
7613		case BuiltinFnIdSrc:
7614		{
7615		IrInstSrc *src_inst = ir_build_src(irb, scope, node);
7616		return ir_lval_wrap(irb, scope, src_inst, lval, result_loc);
7617		}
7618		}
7619		zig_unreachable();
7620		}
7621
7622		static ScopeNoSuspend get_scope_nosuspend(Scope scope) {
7623		while (scope) {
7624		if (scope->id == ScopeIdNoSuspend)
7625		return (ScopeNoSuspend *)scope;
7626		if (scope->id == ScopeIdFnDef)
7627		return nullptr;
7628
7629		scope = scope->parent;
7630		}
7631		return nullptr;
7632		}
7633
7634		static IrInstSrc ir_gen_fn_call(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
7635		ResultLoc *result_loc)
7636		{
7637		assert(node->type == NodeTypeFnCallExpr);
7638
7639		if (node->data.fn_call_expr.modifier == CallModifierBuiltin)
7640		return ir_gen_builtin_fn_call(irb, scope, node, lval, result_loc);
7641
7642		bool is_nosuspend = get_scope_nosuspend(scope) != nullptr;
7643		CallModifier modifier = node->data.fn_call_expr.modifier;
7644		if (is_nosuspend && modifier != CallModifierAsync) {
7645		modifier = CallModifierNoSuspend;
7646		}
7647
7648		AstNode *fn_ref_node = node->data.fn_call_expr.fn_ref_expr;
7649		return ir_gen_fn_call_with_args(irb, scope, node, fn_ref_node, modifier,
7650		nullptr, node->data.fn_call_expr.params.items, node->data.fn_call_expr.params.length, lval, result_loc);
7651		}
7652
7653		static IrInstSrc ir_gen_if_bool_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
7654		ResultLoc *result_loc)
7655		{
7656		assert(node->type == NodeTypeIfBoolExpr);
7657
7658		IrInstSrc *condition = ir_gen_node(irb, node->data.if_bool_expr.condition, scope);
7659		if (condition == irb->codegen->invalid_inst_src)
7660		return irb->codegen->invalid_inst_src;
7661
7662		IrInstSrc *is_comptime;
7663		if (ir_should_inline(irb->exec, scope)) {
7664		is_comptime = ir_build_const_bool(irb, scope, node, true);
7665		} else {
7666		is_comptime = ir_build_test_comptime(irb, scope, node, condition);
7667		}
7668
7669		AstNode *then_node = node->data.if_bool_expr.then_block;
7670		AstNode *else_node = node->data.if_bool_expr.else_node;
7671
7672		IrBasicBlockSrc *then_block = ir_create_basic_block(irb, scope, "Then");
7673		IrBasicBlockSrc *else_block = ir_create_basic_block(irb, scope, "Else");
7674		IrBasicBlockSrc *endif_block = ir_create_basic_block(irb, scope, "EndIf");
7675
7676		IrInstSrc *cond_br_inst = ir_build_cond_br(irb, scope, node, condition,
7677		then_block, else_block, is_comptime);
7678		ResultLocPeerParent *peer_parent = ir_build_binary_result_peers(irb, cond_br_inst, else_block, endif_block,
7679		result_loc, is_comptime);
7680
7681		ir_set_cursor_at_end_and_append_block(irb, then_block);
7682
7683		Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
7684		IrInstSrc *then_expr_result = ir_gen_node_extra(irb, then_node, subexpr_scope, lval,
7685		&peer_parent->peers.at(0)->base);
7686		if (then_expr_result == irb->codegen->invalid_inst_src)
7687		return irb->codegen->invalid_inst_src;
7688		IrBasicBlockSrc *after_then_block = irb->current_basic_block;
7689		if (!instr_is_unreachable(then_expr_result))
7690		ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
7691
7692		ir_set_cursor_at_end_and_append_block(irb, else_block);
7693		IrInstSrc *else_expr_result;
7694		if (else_node) {
7695		else_expr_result = ir_gen_node_extra(irb, else_node, subexpr_scope, lval, &peer_parent->peers.at(1)->base);
7696		if (else_expr_result == irb->codegen->invalid_inst_src)
7697		return irb->codegen->invalid_inst_src;
7698		} else {
7699		else_expr_result = ir_build_const_void(irb, scope, node);
7700		ir_build_end_expr(irb, scope, node, else_expr_result, &peer_parent->peers.at(1)->base);
7701		}
7702		IrBasicBlockSrc *after_else_block = irb->current_basic_block;
7703		if (!instr_is_unreachable(else_expr_result))
7704		ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
7705
7706		ir_set_cursor_at_end_and_append_block(irb, endif_block);
7707		IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
7708		incoming_values[0] = then_expr_result;
7709		incoming_values[1] = else_expr_result;
7710		IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
7711		incoming_blocks[0] = after_then_block;
7712		incoming_blocks[1] = after_else_block;
7713
7714		IrInstSrc *phi = ir_build_phi(irb, scope, node, 2, incoming_blocks, incoming_values, peer_parent);
7715		return ir_expr_wrap(irb, scope, phi, result_loc);
7716		}
7717
7718		static IrInstSrc ir_gen_prefix_op_id_lval(IrBuilderSrc irb, Scope scope, AstNode node, IrUnOp op_id, LVal lval) {
7719		assert(node->type == NodeTypePrefixOpExpr);
7720		AstNode *expr_node = node->data.prefix_op_expr.primary_expr;
7721
7722		IrInstSrc *value = ir_gen_node_extra(irb, expr_node, scope, lval, nullptr);
7723		if (value == irb->codegen->invalid_inst_src)
7724		return value;
7725
7726		return ir_build_un_op(irb, scope, node, op_id, value);
7727		}
7728
7729		static IrInstSrc ir_gen_prefix_op_id(IrBuilderSrc irb, Scope scope, AstNode node, IrUnOp op_id) {
7730		return ir_gen_prefix_op_id_lval(irb, scope, node, op_id, LValNone);
7731		}
7732
7733		static IrInstSrc ir_expr_wrap(IrBuilderSrc irb, Scope scope, IrInstSrc inst, ResultLoc *result_loc) {
7734		if (inst == irb->codegen->invalid_inst_src) return inst;
7735		ir_build_end_expr(irb, scope, inst->base.source_node, inst, result_loc);
7736		return inst;
7737		}
7738
7739		static IrInstSrc ir_lval_wrap(IrBuilderSrc irb, Scope scope, IrInstSrc value, LVal lval,
7740		ResultLoc *result_loc)
7741		{
7742		// This logic must be kept in sync with
7743		// [STMT_EXPR_TEST_THING] <--- (search this token)
7744		if (value == irb->codegen->invalid_inst_src \|\|
7745		instr_is_unreachable(value) \|\|
7746		value->base.source_node->type == NodeTypeDefer \|\|
7747		value->id == IrInstSrcIdDeclVar)
7748		{
7749		return value;
7750		}
7751
7752		assert(lval != LValAssign);
7753		if (lval == LValPtr) {
7754		// We needed a pointer to a value, but we got a value. So we create
7755		// an instruction which just makes a pointer of it.
7756		return ir_build_ref_src(irb, scope, value->base.source_node, value);
7757		} else if (result_loc != nullptr) {
7758		return ir_expr_wrap(irb, scope, value, result_loc);
7759		} else {
7760		return value;
7761		}
7762
7763		}
7764
7765		static PtrLen star_token_to_ptr_len(TokenId token_id) {
7766		switch (token_id) {
7767		case TokenIdStar:
7768		case TokenIdStarStar:
7769		return PtrLenSingle;
7770		case TokenIdLBracket:
7771		return PtrLenUnknown;
7772		case TokenIdSymbol:
7773		return PtrLenC;
7774		default:
7775		zig_unreachable();
7776		}
7777		}
7778
7779		static IrInstSrc ir_gen_pointer_type(IrBuilderSrc irb, Scope scope, AstNode node) {
7780		assert(node->type == NodeTypePointerType);
7781
7782		PtrLen ptr_len = star_token_to_ptr_len(node->data.pointer_type.star_token->id);
7783
7784		bool is_const = node->data.pointer_type.is_const;
7785		bool is_volatile = node->data.pointer_type.is_volatile;
7786		bool is_allow_zero = node->data.pointer_type.allow_zero_token != nullptr;
7787		AstNode *sentinel_expr = node->data.pointer_type.sentinel;
7788		AstNode *expr_node = node->data.pointer_type.op_expr;
7789		AstNode *align_expr = node->data.pointer_type.align_expr;
7790
7791		IrInstSrc *sentinel;
7792		if (sentinel_expr != nullptr) {
7793		sentinel = ir_gen_node(irb, sentinel_expr, scope);
7794		if (sentinel == irb->codegen->invalid_inst_src)
7795		return sentinel;
7796		} else {
7797		sentinel = nullptr;
7798		}
7799
7800		IrInstSrc *align_value;
7801		if (align_expr != nullptr) {
7802		align_value = ir_gen_node(irb, align_expr, scope);
7803		if (align_value == irb->codegen->invalid_inst_src)
7804		return align_value;
7805		} else {
7806		align_value = nullptr;
7807		}
7808
7809		IrInstSrc *child_type = ir_gen_node(irb, expr_node, scope);
7810		if (child_type == irb->codegen->invalid_inst_src)
7811		return child_type;
7812
7813		uint32_t bit_offset_start = 0;
7814		if (node->data.pointer_type.bit_offset_start != nullptr) {
7815		if (!bigint_fits_in_bits(node->data.pointer_type.bit_offset_start, 32, false)) {
7816		Buf *val_buf = buf_alloc();
7817		bigint_append_buf(val_buf, node->data.pointer_type.bit_offset_start, 10);
7818		exec_add_error_node(irb->codegen, irb->exec, node,
7819		buf_sprintf("value %s too large for u32 bit offset", buf_ptr(val_buf)));
7820		return irb->codegen->invalid_inst_src;
7821		}
7822		bit_offset_start = bigint_as_u32(node->data.pointer_type.bit_offset_start);
7823		}
7824
7825		uint32_t host_int_bytes = 0;
7826		if (node->data.pointer_type.host_int_bytes != nullptr) {
7827		if (!bigint_fits_in_bits(node->data.pointer_type.host_int_bytes, 32, false)) {
7828		Buf *val_buf = buf_alloc();
7829		bigint_append_buf(val_buf, node->data.pointer_type.host_int_bytes, 10);
7830		exec_add_error_node(irb->codegen, irb->exec, node,
7831		buf_sprintf("value %s too large for u32 byte count", buf_ptr(val_buf)));
7832		return irb->codegen->invalid_inst_src;
7833		}
7834		host_int_bytes = bigint_as_u32(node->data.pointer_type.host_int_bytes);
7835		}
7836
7837		if (host_int_bytes != 0 && bit_offset_start >= host_int_bytes * 8) {
7838		exec_add_error_node(irb->codegen, irb->exec, node,
7839		buf_sprintf("bit offset starts after end of host integer"));
7840		return irb->codegen->invalid_inst_src;
7841		}
7842
7843		return ir_build_ptr_type(irb, scope, node, child_type, is_const, is_volatile,
7844		ptr_len, sentinel, align_value, bit_offset_start, host_int_bytes, is_allow_zero);
7845		}
7846
7847		static IrInstSrc ir_gen_catch_unreachable(IrBuilderSrc irb, Scope scope, AstNode source_node,
7848		AstNode expr_node, LVal lval, ResultLoc result_loc)
7849		{
7850		IrInstSrc *err_union_ptr = ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr);
7851		if (err_union_ptr == irb->codegen->invalid_inst_src)
7852		return irb->codegen->invalid_inst_src;
7853
7854		IrInstSrc *payload_ptr = ir_build_unwrap_err_payload_src(irb, scope, source_node, err_union_ptr, true, false);
7855		if (payload_ptr == irb->codegen->invalid_inst_src)
7856		return irb->codegen->invalid_inst_src;
7857
7858		if (lval == LValPtr)
7859		return payload_ptr;
7860
7861		IrInstSrc *load_ptr = ir_build_load_ptr(irb, scope, source_node, payload_ptr);
7862		return ir_expr_wrap(irb, scope, load_ptr, result_loc);
7863		}
7864
7865		static IrInstSrc ir_gen_bool_not(IrBuilderSrc irb, Scope scope, AstNode node) {
7866		assert(node->type == NodeTypePrefixOpExpr);
7867		AstNode *expr_node = node->data.prefix_op_expr.primary_expr;
7868
7869		IrInstSrc *value = ir_gen_node(irb, expr_node, scope);
7870		if (value == irb->codegen->invalid_inst_src)
7871		return irb->codegen->invalid_inst_src;
7872
7873		return ir_build_bool_not(irb, scope, node, value);
7874		}
7875
7876		static IrInstSrc ir_gen_prefix_op_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
7877		ResultLoc *result_loc)
7878		{
7879		assert(node->type == NodeTypePrefixOpExpr);
7880
7881		PrefixOp prefix_op = node->data.prefix_op_expr.prefix_op;
7882
7883		switch (prefix_op) {
7884		case PrefixOpInvalid:
7885		zig_unreachable();
7886		case PrefixOpBoolNot:
7887		return ir_lval_wrap(irb, scope, ir_gen_bool_not(irb, scope, node), lval, result_loc);
7888		case PrefixOpBinNot:
7889		return ir_lval_wrap(irb, scope, ir_gen_prefix_op_id(irb, scope, node, IrUnOpBinNot), lval, result_loc);
7890		case PrefixOpNegation:
7891		return ir_lval_wrap(irb, scope, ir_gen_prefix_op_id(irb, scope, node, IrUnOpNegation), lval, result_loc);
7892		case PrefixOpNegationWrap:
7893		return ir_lval_wrap(irb, scope, ir_gen_prefix_op_id(irb, scope, node, IrUnOpNegationWrap), lval, result_loc);
7894		case PrefixOpOptional:
7895		return ir_lval_wrap(irb, scope, ir_gen_prefix_op_id(irb, scope, node, IrUnOpOptional), lval, result_loc);
7896		case PrefixOpAddrOf: {
7897		AstNode *expr_node = node->data.prefix_op_expr.primary_expr;
7898		return ir_lval_wrap(irb, scope, ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr), lval, result_loc);
7899		}
7900		}
7901		zig_unreachable();
7902		}
7903
7904		static IrInstSrc ir_gen_union_init_expr(IrBuilderSrc irb, Scope scope, AstNode source_node,
7905		IrInstSrc union_type, IrInstSrc field_name, AstNode *expr_node,
7906		LVal lval, ResultLoc *parent_result_loc)
7907		{
7908		IrInstSrc *container_ptr = ir_build_resolve_result(irb, scope, source_node, parent_result_loc, union_type);
7909		IrInstSrc *field_ptr = ir_build_field_ptr_instruction(irb, scope, source_node, container_ptr,
7910		field_name, true);
7911
7912		ResultLocInstruction *result_loc_inst = heap::c_allocator.create<ResultLocInstruction>();
7913		result_loc_inst->base.id = ResultLocIdInstruction;
7914		result_loc_inst->base.source_instruction = field_ptr;
7915		ir_ref_instruction(field_ptr, irb->current_basic_block);
7916		ir_build_reset_result(irb, scope, expr_node, &result_loc_inst->base);
7917
7918		IrInstSrc *expr_value = ir_gen_node_extra(irb, expr_node, scope, LValNone,
7919		&result_loc_inst->base);
7920		if (expr_value == irb->codegen->invalid_inst_src)
7921		return expr_value;
7922
7923		IrInstSrc *init_union = ir_build_union_init_named_field(irb, scope, source_node, union_type,
7924		field_name, field_ptr, container_ptr);
7925
7926		return ir_lval_wrap(irb, scope, init_union, lval, parent_result_loc);
7927		}
7928
7929		static IrInstSrc ir_gen_container_init_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
7930		ResultLoc *parent_result_loc)
7931		{
7932		assert(node->type == NodeTypeContainerInitExpr);
7933
7934		AstNodeContainerInitExpr *container_init_expr = &node->data.container_init_expr;
7935		ContainerInitKind kind = container_init_expr->kind;
7936
7937		ResultLocCast *result_loc_cast = nullptr;
7938		ResultLoc *child_result_loc;
7939		AstNode *init_array_type_source_node;
7940		if (container_init_expr->type != nullptr) {
7941		IrInstSrc *container_type;
7942		if (container_init_expr->type->type == NodeTypeInferredArrayType) {
7943		if (kind == ContainerInitKindStruct) {
7944		add_node_error(irb->codegen, container_init_expr->type,
7945		buf_sprintf("initializing array with struct syntax"));
7946		return irb->codegen->invalid_inst_src;
7947		}
7948		IrInstSrc *sentinel;
7949		if (container_init_expr->type->data.inferred_array_type.sentinel != nullptr) {
7950		sentinel = ir_gen_node(irb, container_init_expr->type->data.inferred_array_type.sentinel, scope);
7951		if (sentinel == irb->codegen->invalid_inst_src)
7952		return sentinel;
7953		} else {
7954		sentinel = nullptr;
7955		}
7956
7957		IrInstSrc *elem_type = ir_gen_node(irb,
7958		container_init_expr->type->data.inferred_array_type.child_type, scope);
7959		if (elem_type == irb->codegen->invalid_inst_src)
7960		return elem_type;
7961		size_t item_count = container_init_expr->entries.length;
7962		IrInstSrc *item_count_inst = ir_build_const_usize(irb, scope, node, item_count);
7963		container_type = ir_build_array_type(irb, scope, node, item_count_inst, sentinel, elem_type);
7964		} else {
7965		container_type = ir_gen_node(irb, container_init_expr->type, scope);
7966		if (container_type == irb->codegen->invalid_inst_src)
7967		return container_type;
7968		}
7969
7970		result_loc_cast = ir_build_cast_result_loc(irb, container_type, parent_result_loc);
7971		child_result_loc = &result_loc_cast->base;
7972		init_array_type_source_node = container_type->base.source_node;
7973		} else {
7974		child_result_loc = parent_result_loc;
7975		if (parent_result_loc->source_instruction != nullptr) {
7976		init_array_type_source_node = parent_result_loc->source_instruction->base.source_node;
7977		} else {
7978		init_array_type_source_node = node;
7979		}
7980		}
7981
7982		switch (kind) {
7983		case ContainerInitKindStruct: {
7984		IrInstSrc *container_ptr = ir_build_resolve_result(irb, scope, node, child_result_loc,
7985		nullptr);
7986
7987		size_t field_count = container_init_expr->entries.length;
7988		IrInstSrcContainerInitFieldsField *fields = heap::c_allocator.allocate<IrInstSrcContainerInitFieldsField>(field_count);
7989		for (size_t i = 0; i < field_count; i += 1) {
7990		AstNode *entry_node = container_init_expr->entries.at(i);
7991		assert(entry_node->type == NodeTypeStructValueField);
7992
7993		Buf *name = entry_node->data.struct_val_field.name;
7994		AstNode *expr_node = entry_node->data.struct_val_field.expr;
7995
7996		IrInstSrc *field_ptr = ir_build_field_ptr(irb, scope, entry_node, container_ptr, name, true);
7997		ResultLocInstruction *result_loc_inst = heap::c_allocator.create<ResultLocInstruction>();
7998		result_loc_inst->base.id = ResultLocIdInstruction;
7999		result_loc_inst->base.source_instruction = field_ptr;
8000		result_loc_inst->base.allow_write_through_const = true;
8001		ir_ref_instruction(field_ptr, irb->current_basic_block);
8002		ir_build_reset_result(irb, scope, expr_node, &result_loc_inst->base);
8003
8004		IrInstSrc *expr_value = ir_gen_node_extra(irb, expr_node, scope, LValNone,
8005		&result_loc_inst->base);
8006		if (expr_value == irb->codegen->invalid_inst_src)
8007		return expr_value;
8008
8009		fields[i].name = name;
8010		fields[i].source_node = entry_node;
8011		fields[i].result_loc = field_ptr;
8012		}
8013		IrInstSrc *result = ir_build_container_init_fields(irb, scope, node, field_count,
8014		fields, container_ptr);
8015
8016		if (result_loc_cast != nullptr) {
8017		result = ir_build_implicit_cast(irb, scope, node, result, result_loc_cast);
8018		}
8019		return ir_lval_wrap(irb, scope, result, lval, parent_result_loc);
8020		}
8021		case ContainerInitKindArray: {
8022		size_t item_count = container_init_expr->entries.length;
8023
8024		IrInstSrc *container_ptr = ir_build_resolve_result(irb, scope, node, child_result_loc,
8025		nullptr);
8026
8027		IrInstSrc *result_locs = heap::c_allocator.allocate<IrInstSrc >(item_count);
8028		for (size_t i = 0; i < item_count; i += 1) {
8029		AstNode *expr_node = container_init_expr->entries.at(i);
8030
8031		IrInstSrc *elem_index = ir_build_const_usize(irb, scope, expr_node, i);
8032		IrInstSrc *elem_ptr = ir_build_elem_ptr(irb, scope, expr_node, container_ptr,
8033		elem_index, false, PtrLenSingle, init_array_type_source_node);
8034		ResultLocInstruction *result_loc_inst = heap::c_allocator.create<ResultLocInstruction>();
8035		result_loc_inst->base.id = ResultLocIdInstruction;
8036		result_loc_inst->base.source_instruction = elem_ptr;
8037		result_loc_inst->base.allow_write_through_const = true;
8038		ir_ref_instruction(elem_ptr, irb->current_basic_block);
8039		ir_build_reset_result(irb, scope, expr_node, &result_loc_inst->base);
8040
8041		IrInstSrc *expr_value = ir_gen_node_extra(irb, expr_node, scope, LValNone,
8042		&result_loc_inst->base);
8043		if (expr_value == irb->codegen->invalid_inst_src)
8044		return expr_value;
8045
8046		result_locs[i] = elem_ptr;
8047		}
8048		IrInstSrc *result = ir_build_container_init_list(irb, scope, node, item_count,
8049		result_locs, container_ptr, init_array_type_source_node);
8050		if (result_loc_cast != nullptr) {
8051		result = ir_build_implicit_cast(irb, scope, node, result, result_loc_cast);
8052		}
8053		return ir_lval_wrap(irb, scope, result, lval, parent_result_loc);
8054		}
8055		}
8056		zig_unreachable();
8057		}
8058
8059		static ResultLocVar ir_build_var_result_loc(IrBuilderSrc irb, IrInstSrc alloca, ZigVar var) {
8060		ResultLocVar *result_loc_var = heap::c_allocator.create<ResultLocVar>();
8061		result_loc_var->base.id = ResultLocIdVar;
8062		result_loc_var->base.source_instruction = alloca;
8063		result_loc_var->base.allow_write_through_const = true;
8064		result_loc_var->var = var;
8065
8066		ir_build_reset_result(irb, alloca->base.scope, alloca->base.source_node, &result_loc_var->base);
8067
8068		return result_loc_var;
8069		}
8070
8071		static ResultLocCast ir_build_cast_result_loc(IrBuilderSrc irb, IrInstSrc *dest_type,
8072		ResultLoc *parent_result_loc)
8073		{
8074		ResultLocCast *result_loc_cast = heap::c_allocator.create<ResultLocCast>();
8075		result_loc_cast->base.id = ResultLocIdCast;
8076		result_loc_cast->base.source_instruction = dest_type;
8077		result_loc_cast->base.allow_write_through_const = parent_result_loc->allow_write_through_const;
8078		ir_ref_instruction(dest_type, irb->current_basic_block);
8079		result_loc_cast->parent = parent_result_loc;
8080
8081		ir_build_reset_result(irb, dest_type->base.scope, dest_type->base.source_node, &result_loc_cast->base);
8082
8083		return result_loc_cast;
8084		}
8085
8086		static void build_decl_var_and_init(IrBuilderSrc irb, Scope scope, AstNode source_node, ZigVar var,
8087		IrInstSrc init, const char name_hint, IrInstSrc *is_comptime)
8088		{
8089		IrInstSrc *alloca = ir_build_alloca_src(irb, scope, source_node, nullptr, name_hint, is_comptime);
8090		ResultLocVar *var_result_loc = ir_build_var_result_loc(irb, alloca, var);
8091		ir_build_end_expr(irb, scope, source_node, init, &var_result_loc->base);
8092		ir_build_var_decl_src(irb, scope, source_node, var, nullptr, alloca);
8093		}
8094
8095		static IrInstSrc ir_gen_var_decl(IrBuilderSrc irb, Scope scope, AstNode node) {
8096		assert(node->type == NodeTypeVariableDeclaration);
8097
8098		AstNodeVariableDeclaration *variable_declaration = &node->data.variable_declaration;
8099
8100		if (buf_eql_str(variable_declaration->symbol, "_")) {
8101		add_node_error(irb->codegen, node, buf_sprintf("`_` is not a declarable symbol"));
8102		return irb->codegen->invalid_inst_src;
8103		}
8104
8105		// Used for the type expr and the align expr
8106		Scope *comptime_scope = create_comptime_scope(irb->codegen, node, scope);
8107
8108		IrInstSrc *type_instruction;
8109		if (variable_declaration->type != nullptr) {
8110		type_instruction = ir_gen_node(irb, variable_declaration->type, comptime_scope);
8111		if (type_instruction == irb->codegen->invalid_inst_src)
8112		return type_instruction;
8113		} else {
8114		type_instruction = nullptr;
8115		}
8116
8117		bool is_shadowable = false;
8118		bool is_const = variable_declaration->is_const;
8119		bool is_extern = variable_declaration->is_extern;
8120
8121		bool is_comptime_scalar = ir_should_inline(irb->exec, scope) \|\| variable_declaration->is_comptime;
8122		IrInstSrc *is_comptime = ir_build_const_bool(irb, scope, node, is_comptime_scalar);
8123		ZigVar *var = ir_create_var(irb, node, scope, variable_declaration->symbol,
8124		is_const, is_const, is_shadowable, is_comptime);
8125		// we detect IrInstSrcDeclVar in gen_block to make sure the next node
8126		// is inside var->child_scope
8127
8128		if (!is_extern && !variable_declaration->expr) {
8129		var->var_type = irb->codegen->builtin_types.entry_invalid;
8130		add_node_error(irb->codegen, node, buf_sprintf("variables must be initialized"));
8131		return irb->codegen->invalid_inst_src;
8132		}
8133
8134		IrInstSrc *align_value = nullptr;
8135		if (variable_declaration->align_expr != nullptr) {
8136		align_value = ir_gen_node(irb, variable_declaration->align_expr, comptime_scope);
8137		if (align_value == irb->codegen->invalid_inst_src)
8138		return align_value;
8139		}
8140
8141		if (variable_declaration->section_expr != nullptr) {
8142		add_node_error(irb->codegen, variable_declaration->section_expr,
8143		buf_sprintf("cannot set section of local variable '%s'", buf_ptr(variable_declaration->symbol)));
8144		}
8145
8146		// Parser should ensure that this never happens
8147		assert(variable_declaration->threadlocal_tok == nullptr);
8148
8149		IrInstSrc *alloca = ir_build_alloca_src(irb, scope, node, align_value,
8150		buf_ptr(variable_declaration->symbol), is_comptime);
8151
8152		// Create a result location for the initialization expression.
8153		ResultLocVar *result_loc_var = ir_build_var_result_loc(irb, alloca, var);
8154		ResultLoc *init_result_loc;
8155		ResultLocCast *result_loc_cast;
8156		if (type_instruction != nullptr) {
8157		result_loc_cast = ir_build_cast_result_loc(irb, type_instruction, &result_loc_var->base);
8158		init_result_loc = &result_loc_cast->base;
8159		} else {
8160		result_loc_cast = nullptr;
8161		init_result_loc = &result_loc_var->base;
8162		}
8163
8164		Scope *init_scope = is_comptime_scalar ?
8165		create_comptime_scope(irb->codegen, variable_declaration->expr, scope) : scope;
8166
8167		// Temporarily set the name of the IrExecutableSrc to the VariableDeclaration
8168		// so that the struct or enum from the init expression inherits the name.
8169		Buf *old_exec_name = irb->exec->name;
8170		irb->exec->name = variable_declaration->symbol;
8171		IrInstSrc *init_value = ir_gen_node_extra(irb, variable_declaration->expr, init_scope,
8172		LValNone, init_result_loc);
8173		irb->exec->name = old_exec_name;
8174
8175		if (init_value == irb->codegen->invalid_inst_src)
8176		return irb->codegen->invalid_inst_src;
8177
8178		if (result_loc_cast != nullptr) {
8179		IrInstSrc *implicit_cast = ir_build_implicit_cast(irb, scope, init_value->base.source_node,
8180		init_value, result_loc_cast);
8181		ir_build_end_expr(irb, scope, node, implicit_cast, &result_loc_var->base);
8182		}
8183
8184		return ir_build_var_decl_src(irb, scope, node, var, align_value, alloca);
8185		}
8186
8187		static IrInstSrc ir_gen_while_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
8188		ResultLoc *result_loc)
8189		{
8190		assert(node->type == NodeTypeWhileExpr);
8191
8192		AstNode *continue_expr_node = node->data.while_expr.continue_expr;
8193		AstNode *else_node = node->data.while_expr.else_node;
8194
8195		IrBasicBlockSrc *cond_block = ir_create_basic_block(irb, scope, "WhileCond");
8196		IrBasicBlockSrc *body_block = ir_create_basic_block(irb, scope, "WhileBody");
8197		IrBasicBlockSrc *continue_block = continue_expr_node ?
8198		ir_create_basic_block(irb, scope, "WhileContinue") : cond_block;
8199		IrBasicBlockSrc *end_block = ir_create_basic_block(irb, scope, "WhileEnd");
8200		IrBasicBlockSrc *else_block = else_node ?
8201		ir_create_basic_block(irb, scope, "WhileElse") : end_block;
8202
8203		IrInstSrc *is_comptime = ir_build_const_bool(irb, scope, node,
8204		ir_should_inline(irb->exec, scope) \|\| node->data.while_expr.is_inline);
8205		ir_build_br(irb, scope, node, cond_block, is_comptime);
8206
8207		Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
8208		Buf *var_symbol = node->data.while_expr.var_symbol;
8209		Buf *err_symbol = node->data.while_expr.err_symbol;
8210		if (err_symbol != nullptr) {
8211		ir_set_cursor_at_end_and_append_block(irb, cond_block);
8212
8213		Scope *payload_scope;
8214		AstNode *symbol_node = node; // TODO make more accurate
8215		ZigVar *payload_var;
8216		if (var_symbol) {
8217		// TODO make it an error to write to payload variable
8218		payload_var = ir_create_var(irb, symbol_node, subexpr_scope, var_symbol,
8219		true, false, false, is_comptime);
8220		payload_scope = payload_var->child_scope;
8221		} else {
8222		payload_scope = subexpr_scope;
8223		}
8224		ScopeExpr *spill_scope = create_expr_scope(irb->codegen, node, payload_scope);
8225		IrInstSrc *err_val_ptr = ir_gen_node_extra(irb, node->data.while_expr.condition, subexpr_scope,
8226		LValPtr, nullptr);
8227		if (err_val_ptr == irb->codegen->invalid_inst_src)
8228		return err_val_ptr;
8229		IrInstSrc *is_err = ir_build_test_err_src(irb, scope, node->data.while_expr.condition, err_val_ptr,
8230		true, false);
8231		IrBasicBlockSrc *after_cond_block = irb->current_basic_block;
8232		IrInstSrc *void_else_result = else_node ? nullptr : ir_mark_gen(ir_build_const_void(irb, scope, node));
8233		IrInstSrc *cond_br_inst;
8234		if (!instr_is_unreachable(is_err)) {
8235		cond_br_inst = ir_build_cond_br(irb, scope, node->data.while_expr.condition, is_err,
8236		else_block, body_block, is_comptime);
8237		cond_br_inst->is_gen = true;
8238		} else {
8239		// for the purposes of the source instruction to ir_build_result_peers
8240		cond_br_inst = irb->current_basic_block->instruction_list.last();
8241		}
8242
8243		ResultLocPeerParent *peer_parent = ir_build_result_peers(irb, cond_br_inst, end_block, result_loc,
8244		is_comptime);
8245
8246		ir_set_cursor_at_end_and_append_block(irb, body_block);
8247		if (var_symbol) {
8248		IrInstSrc *payload_ptr = ir_build_unwrap_err_payload_src(irb, &spill_scope->base, symbol_node,
8249		err_val_ptr, false, false);
8250		IrInstSrc *var_value = node->data.while_expr.var_is_ptr ?
8251		payload_ptr : ir_build_load_ptr(irb, &spill_scope->base, symbol_node, payload_ptr);
8252		build_decl_var_and_init(irb, payload_scope, symbol_node, payload_var, var_value, buf_ptr(var_symbol), is_comptime);
8253		}
8254
8255		ZigList<IrInstSrc *> incoming_values = {0};
8256		ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
8257
8258		if (is_duplicate_label(irb->codegen, payload_scope, node, node->data.while_expr.name))
8259		return irb->codegen->invalid_inst_src;
8260
8261		ScopeLoop *loop_scope = create_loop_scope(irb->codegen, node, payload_scope);
8262		loop_scope->break_block = end_block;
8263		loop_scope->continue_block = continue_block;
8264		loop_scope->is_comptime = is_comptime;
8265		loop_scope->incoming_blocks = &incoming_blocks;
8266		loop_scope->incoming_values = &incoming_values;
8267		loop_scope->lval = lval;
8268		loop_scope->peer_parent = peer_parent;
8269		loop_scope->spill_scope = spill_scope;
8270
8271		// Note the body block of the loop is not the place that lval and result_loc are used -
8272		// it's actually in break statements, handled similarly to return statements.
8273		// That is why we set those values in loop_scope above and not in this ir_gen_node call.
8274		IrInstSrc *body_result = ir_gen_node(irb, node->data.while_expr.body, &loop_scope->base);
8275		if (body_result == irb->codegen->invalid_inst_src)
8276		return body_result;
8277
8278		if (loop_scope->name != nullptr && loop_scope->name_used == false) {
8279		add_node_error(irb->codegen, node, buf_sprintf("unused while label"));
8280		}
8281
8282		if (!instr_is_unreachable(body_result)) {
8283		ir_mark_gen(ir_build_check_statement_is_void(irb, payload_scope, node->data.while_expr.body, body_result));
8284		ir_mark_gen(ir_build_br(irb, payload_scope, node, continue_block, is_comptime));
8285		}
8286
8287		if (continue_expr_node) {
8288		ir_set_cursor_at_end_and_append_block(irb, continue_block);
8289		IrInstSrc *expr_result = ir_gen_node(irb, continue_expr_node, payload_scope);
8290		if (expr_result == irb->codegen->invalid_inst_src)
8291		return expr_result;
8292		if (!instr_is_unreachable(expr_result)) {
8293		ir_mark_gen(ir_build_check_statement_is_void(irb, payload_scope, continue_expr_node, expr_result));
8294		ir_mark_gen(ir_build_br(irb, payload_scope, node, cond_block, is_comptime));
8295		}
8296		}
8297
8298		ir_set_cursor_at_end_and_append_block(irb, else_block);
8299		assert(else_node != nullptr);
8300
8301		// TODO make it an error to write to error variable
8302		AstNode *err_symbol_node = else_node; // TODO make more accurate
8303		ZigVar *err_var = ir_create_var(irb, err_symbol_node, scope, err_symbol,
8304		true, false, false, is_comptime);
8305		Scope *err_scope = err_var->child_scope;
8306		IrInstSrc *err_ptr = ir_build_unwrap_err_code_src(irb, err_scope, err_symbol_node, err_val_ptr);
8307		IrInstSrc *err_value = ir_build_load_ptr(irb, err_scope, err_symbol_node, err_ptr);
8308		build_decl_var_and_init(irb, err_scope, err_symbol_node, err_var, err_value, buf_ptr(err_symbol), is_comptime);
8309
8310		if (peer_parent->peers.length != 0) {
8311		peer_parent->peers.last()->next_bb = else_block;
8312		}
8313		ResultLocPeer *peer_result = create_peer_result(peer_parent);
8314		peer_parent->peers.append(peer_result);
8315		IrInstSrc *else_result = ir_gen_node_extra(irb, else_node, err_scope, lval, &peer_result->base);
8316		if (else_result == irb->codegen->invalid_inst_src)
8317		return else_result;
8318		if (!instr_is_unreachable(else_result))
8319		ir_mark_gen(ir_build_br(irb, scope, node, end_block, is_comptime));
8320		IrBasicBlockSrc *after_else_block = irb->current_basic_block;
8321		ir_set_cursor_at_end_and_append_block(irb, end_block);
8322		if (else_result) {
8323		incoming_blocks.append(after_else_block);
8324		incoming_values.append(else_result);
8325		} else {
8326		incoming_blocks.append(after_cond_block);
8327		incoming_values.append(void_else_result);
8328		}
8329		if (peer_parent->peers.length != 0) {
8330		peer_parent->peers.last()->next_bb = end_block;
8331		}
8332
8333		IrInstSrc *phi = ir_build_phi(irb, scope, node, incoming_blocks.length,
8334		incoming_blocks.items, incoming_values.items, peer_parent);
8335		return ir_expr_wrap(irb, scope, phi, result_loc);
8336		} else if (var_symbol != nullptr) {
8337		ir_set_cursor_at_end_and_append_block(irb, cond_block);
8338		Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
8339		// TODO make it an error to write to payload variable
8340		AstNode *symbol_node = node; // TODO make more accurate
8341
8342		ZigVar *payload_var = ir_create_var(irb, symbol_node, subexpr_scope, var_symbol,
8343		true, false, false, is_comptime);
8344		Scope *child_scope = payload_var->child_scope;
8345		ScopeExpr *spill_scope = create_expr_scope(irb->codegen, node, child_scope);
8346		IrInstSrc *maybe_val_ptr = ir_gen_node_extra(irb, node->data.while_expr.condition, subexpr_scope,
8347		LValPtr, nullptr);
8348		if (maybe_val_ptr == irb->codegen->invalid_inst_src)
8349		return maybe_val_ptr;
8350		IrInstSrc *maybe_val = ir_build_load_ptr(irb, scope, node->data.while_expr.condition, maybe_val_ptr);
8351		IrInstSrc *is_non_null = ir_build_test_non_null_src(irb, scope, node->data.while_expr.condition, maybe_val);
8352		IrBasicBlockSrc *after_cond_block = irb->current_basic_block;
8353		IrInstSrc *void_else_result = else_node ? nullptr : ir_mark_gen(ir_build_const_void(irb, scope, node));
8354		IrInstSrc *cond_br_inst;
8355		if (!instr_is_unreachable(is_non_null)) {
8356		cond_br_inst = ir_build_cond_br(irb, scope, node->data.while_expr.condition, is_non_null,
8357		body_block, else_block, is_comptime);
8358		cond_br_inst->is_gen = true;
8359		} else {
8360		// for the purposes of the source instruction to ir_build_result_peers
8361		cond_br_inst = irb->current_basic_block->instruction_list.last();
8362		}
8363
8364		ResultLocPeerParent *peer_parent = ir_build_result_peers(irb, cond_br_inst, end_block, result_loc,
8365		is_comptime);
8366
8367		ir_set_cursor_at_end_and_append_block(irb, body_block);
8368		IrInstSrc *payload_ptr = ir_build_optional_unwrap_ptr(irb, &spill_scope->base, symbol_node, maybe_val_ptr, false);
8369		IrInstSrc *var_value = node->data.while_expr.var_is_ptr ?
8370		payload_ptr : ir_build_load_ptr(irb, &spill_scope->base, symbol_node, payload_ptr);
8371		build_decl_var_and_init(irb, child_scope, symbol_node, payload_var, var_value, buf_ptr(var_symbol), is_comptime);
8372
8373		ZigList<IrInstSrc *> incoming_values = {0};
8374		ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
8375
8376		if (is_duplicate_label(irb->codegen, child_scope, node, node->data.while_expr.name))
8377		return irb->codegen->invalid_inst_src;
8378
8379		ScopeLoop *loop_scope = create_loop_scope(irb->codegen, node, child_scope);
8380		loop_scope->break_block = end_block;
8381		loop_scope->continue_block = continue_block;
8382		loop_scope->is_comptime = is_comptime;
8383		loop_scope->incoming_blocks = &incoming_blocks;
8384		loop_scope->incoming_values = &incoming_values;
8385		loop_scope->lval = lval;
8386		loop_scope->peer_parent = peer_parent;
8387		loop_scope->spill_scope = spill_scope;
8388
8389		// Note the body block of the loop is not the place that lval and result_loc are used -
8390		// it's actually in break statements, handled similarly to return statements.
8391		// That is why we set those values in loop_scope above and not in this ir_gen_node call.
8392		IrInstSrc *body_result = ir_gen_node(irb, node->data.while_expr.body, &loop_scope->base);
8393		if (body_result == irb->codegen->invalid_inst_src)
8394		return body_result;
8395
8396		if (loop_scope->name != nullptr && loop_scope->name_used == false) {
8397		add_node_error(irb->codegen, node, buf_sprintf("unused while label"));
8398		}
8399
8400		if (!instr_is_unreachable(body_result)) {
8401		ir_mark_gen(ir_build_check_statement_is_void(irb, child_scope, node->data.while_expr.body, body_result));
8402		ir_mark_gen(ir_build_br(irb, child_scope, node, continue_block, is_comptime));
8403		}
8404
8405		if (continue_expr_node) {
8406		ir_set_cursor_at_end_and_append_block(irb, continue_block);
8407		IrInstSrc *expr_result = ir_gen_node(irb, continue_expr_node, child_scope);
8408		if (expr_result == irb->codegen->invalid_inst_src)
8409		return expr_result;
8410		if (!instr_is_unreachable(expr_result)) {
8411		ir_mark_gen(ir_build_check_statement_is_void(irb, child_scope, continue_expr_node, expr_result));
8412		ir_mark_gen(ir_build_br(irb, child_scope, node, cond_block, is_comptime));
8413		}
8414		}
8415
8416		IrInstSrc *else_result = nullptr;
8417		if (else_node) {
8418		ir_set_cursor_at_end_and_append_block(irb, else_block);
8419
8420		if (peer_parent->peers.length != 0) {
8421		peer_parent->peers.last()->next_bb = else_block;
8422		}
8423		ResultLocPeer *peer_result = create_peer_result(peer_parent);
8424		peer_parent->peers.append(peer_result);
8425		else_result = ir_gen_node_extra(irb, else_node, scope, lval, &peer_result->base);
8426		if (else_result == irb->codegen->invalid_inst_src)
8427		return else_result;
8428		if (!instr_is_unreachable(else_result))
8429		ir_mark_gen(ir_build_br(irb, scope, node, end_block, is_comptime));
8430		}
8431		IrBasicBlockSrc *after_else_block = irb->current_basic_block;
8432		ir_set_cursor_at_end_and_append_block(irb, end_block);
8433		if (else_result) {
8434		incoming_blocks.append(after_else_block);
8435		incoming_values.append(else_result);
8436		} else {
8437		incoming_blocks.append(after_cond_block);
8438		incoming_values.append(void_else_result);
8439		}
8440		if (peer_parent->peers.length != 0) {
8441		peer_parent->peers.last()->next_bb = end_block;
8442		}
8443
8444		IrInstSrc *phi = ir_build_phi(irb, scope, node, incoming_blocks.length,
8445		incoming_blocks.items, incoming_values.items, peer_parent);
8446		return ir_expr_wrap(irb, scope, phi, result_loc);
8447		} else {
8448		ir_set_cursor_at_end_and_append_block(irb, cond_block);
8449		IrInstSrc *cond_val = ir_gen_node(irb, node->data.while_expr.condition, scope);
8450		if (cond_val == irb->codegen->invalid_inst_src)
8451		return cond_val;
8452		IrBasicBlockSrc *after_cond_block = irb->current_basic_block;
8453		IrInstSrc *void_else_result = else_node ? nullptr : ir_mark_gen(ir_build_const_void(irb, scope, node));
8454		IrInstSrc *cond_br_inst;
8455		if (!instr_is_unreachable(cond_val)) {
8456		cond_br_inst = ir_build_cond_br(irb, scope, node->data.while_expr.condition, cond_val,
8457		body_block, else_block, is_comptime);
8458		cond_br_inst->is_gen = true;
8459		} else {
8460		// for the purposes of the source instruction to ir_build_result_peers
8461		cond_br_inst = irb->current_basic_block->instruction_list.last();
8462		}
8463
8464		ResultLocPeerParent *peer_parent = ir_build_result_peers(irb, cond_br_inst, end_block, result_loc,
8465		is_comptime);
8466		ir_set_cursor_at_end_and_append_block(irb, body_block);
8467
8468		ZigList<IrInstSrc *> incoming_values = {0};
8469		ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
8470
8471		Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
8472
8473		if (is_duplicate_label(irb->codegen, subexpr_scope, node, node->data.while_expr.name))
8474		return irb->codegen->invalid_inst_src;
8475
8476		ScopeLoop *loop_scope = create_loop_scope(irb->codegen, node, subexpr_scope);
8477		loop_scope->break_block = end_block;
8478		loop_scope->continue_block = continue_block;
8479		loop_scope->is_comptime = is_comptime;
8480		loop_scope->incoming_blocks = &incoming_blocks;
8481		loop_scope->incoming_values = &incoming_values;
8482		loop_scope->lval = lval;
8483		loop_scope->peer_parent = peer_parent;
8484
8485		// Note the body block of the loop is not the place that lval and result_loc are used -
8486		// it's actually in break statements, handled similarly to return statements.
8487		// That is why we set those values in loop_scope above and not in this ir_gen_node call.
8488		IrInstSrc *body_result = ir_gen_node(irb, node->data.while_expr.body, &loop_scope->base);
8489		if (body_result == irb->codegen->invalid_inst_src)
8490		return body_result;
8491
8492		if (loop_scope->name != nullptr && loop_scope->name_used == false) {
8493		add_node_error(irb->codegen, node, buf_sprintf("unused while label"));
8494		}
8495
8496		if (!instr_is_unreachable(body_result)) {
8497		ir_mark_gen(ir_build_check_statement_is_void(irb, scope, node->data.while_expr.body, body_result));
8498		ir_mark_gen(ir_build_br(irb, scope, node, continue_block, is_comptime));
8499		}
8500
8501		if (continue_expr_node) {
8502		ir_set_cursor_at_end_and_append_block(irb, continue_block);
8503		IrInstSrc *expr_result = ir_gen_node(irb, continue_expr_node, subexpr_scope);
8504		if (expr_result == irb->codegen->invalid_inst_src)
8505		return expr_result;
8506		if (!instr_is_unreachable(expr_result)) {
8507		ir_mark_gen(ir_build_check_statement_is_void(irb, scope, continue_expr_node, expr_result));
8508		ir_mark_gen(ir_build_br(irb, scope, node, cond_block, is_comptime));
8509		}
8510		}
8511
8512		IrInstSrc *else_result = nullptr;
8513		if (else_node) {
8514		ir_set_cursor_at_end_and_append_block(irb, else_block);
8515
8516		if (peer_parent->peers.length != 0) {
8517		peer_parent->peers.last()->next_bb = else_block;
8518		}
8519		ResultLocPeer *peer_result = create_peer_result(peer_parent);
8520		peer_parent->peers.append(peer_result);
8521
8522		else_result = ir_gen_node_extra(irb, else_node, subexpr_scope, lval, &peer_result->base);
8523		if (else_result == irb->codegen->invalid_inst_src)
8524		return else_result;
8525		if (!instr_is_unreachable(else_result))
8526		ir_mark_gen(ir_build_br(irb, scope, node, end_block, is_comptime));
8527		}
8528		IrBasicBlockSrc *after_else_block = irb->current_basic_block;
8529		ir_set_cursor_at_end_and_append_block(irb, end_block);
8530		if (else_result) {
8531		incoming_blocks.append(after_else_block);
8532		incoming_values.append(else_result);
8533		} else {
8534		incoming_blocks.append(after_cond_block);
8535		incoming_values.append(void_else_result);
8536		}
8537		if (peer_parent->peers.length != 0) {
8538		peer_parent->peers.last()->next_bb = end_block;
8539		}
8540
8541		IrInstSrc *phi = ir_build_phi(irb, scope, node, incoming_blocks.length,
8542		incoming_blocks.items, incoming_values.items, peer_parent);
8543		return ir_expr_wrap(irb, scope, phi, result_loc);
8544		}
8545		}
8546
8547		static IrInstSrc ir_gen_for_expr(IrBuilderSrc irb, Scope parent_scope, AstNode node, LVal lval,
8548		ResultLoc *result_loc)
8549		{
8550		assert(node->type == NodeTypeForExpr);
8551
8552		AstNode *array_node = node->data.for_expr.array_expr;
8553		AstNode *elem_node = node->data.for_expr.elem_node;
8554		AstNode *index_node = node->data.for_expr.index_node;
8555		AstNode *body_node = node->data.for_expr.body;
8556		AstNode *else_node = node->data.for_expr.else_node;
8557
8558		if (!elem_node) {
8559		add_node_error(irb->codegen, node, buf_sprintf("for loop expression missing element parameter"));
8560		return irb->codegen->invalid_inst_src;
8561		}
8562		assert(elem_node->type == NodeTypeSymbol);
8563
8564		ScopeExpr *spill_scope = create_expr_scope(irb->codegen, node, parent_scope);
8565
8566		IrInstSrc *array_val_ptr = ir_gen_node_extra(irb, array_node, &spill_scope->base, LValPtr, nullptr);
8567		if (array_val_ptr == irb->codegen->invalid_inst_src)
8568		return array_val_ptr;
8569
8570		IrInstSrc *is_comptime = ir_build_const_bool(irb, parent_scope, node,
8571		ir_should_inline(irb->exec, parent_scope) \|\| node->data.for_expr.is_inline);
8572
8573		AstNode *index_var_source_node;
8574		ZigVar *index_var;
8575		const char *index_var_name;
8576		if (index_node) {
8577		index_var_source_node = index_node;
8578		Buf *index_var_name_buf = index_node->data.symbol_expr.symbol;
8579		index_var = ir_create_var(irb, index_node, parent_scope, index_var_name_buf, true, false, false, is_comptime);
8580		index_var_name = buf_ptr(index_var_name_buf);
8581		} else {
8582		index_var_source_node = node;
8583		index_var = ir_create_var(irb, node, parent_scope, nullptr, true, false, true, is_comptime);
8584		index_var_name = "i";
8585		}
8586
8587		IrInstSrc *zero = ir_build_const_usize(irb, parent_scope, node, 0);
8588		build_decl_var_and_init(irb, parent_scope, index_var_source_node, index_var, zero, index_var_name, is_comptime);
8589		parent_scope = index_var->child_scope;
8590
8591		IrInstSrc *one = ir_build_const_usize(irb, parent_scope, node, 1);
8592		IrInstSrc *index_ptr = ir_build_var_ptr(irb, parent_scope, node, index_var);
8593
8594
8595		IrBasicBlockSrc *cond_block = ir_create_basic_block(irb, parent_scope, "ForCond");
8596		IrBasicBlockSrc *body_block = ir_create_basic_block(irb, parent_scope, "ForBody");
8597		IrBasicBlockSrc *end_block = ir_create_basic_block(irb, parent_scope, "ForEnd");
8598		IrBasicBlockSrc *else_block = else_node ? ir_create_basic_block(irb, parent_scope, "ForElse") : end_block;
8599		IrBasicBlockSrc *continue_block = ir_create_basic_block(irb, parent_scope, "ForContinue");
8600
8601		Buf *len_field_name = buf_create_from_str("len");
8602		IrInstSrc *len_ref = ir_build_field_ptr(irb, parent_scope, node, array_val_ptr, len_field_name, false);
8603		IrInstSrc *len_val = ir_build_load_ptr(irb, &spill_scope->base, node, len_ref);
8604		ir_build_br(irb, parent_scope, node, cond_block, is_comptime);
8605
8606		ir_set_cursor_at_end_and_append_block(irb, cond_block);
8607		IrInstSrc *index_val = ir_build_load_ptr(irb, &spill_scope->base, node, index_ptr);
8608		IrInstSrc *cond = ir_build_bin_op(irb, parent_scope, node, IrBinOpCmpLessThan, index_val, len_val, false);
8609		IrBasicBlockSrc *after_cond_block = irb->current_basic_block;
8610		IrInstSrc *void_else_value = else_node ? nullptr : ir_mark_gen(ir_build_const_void(irb, parent_scope, node));
8611		IrInstSrc *cond_br_inst = ir_mark_gen(ir_build_cond_br(irb, parent_scope, node, cond,
8612		body_block, else_block, is_comptime));
8613
8614		ResultLocPeerParent *peer_parent = ir_build_result_peers(irb, cond_br_inst, end_block, result_loc, is_comptime);
8615
8616		ir_set_cursor_at_end_and_append_block(irb, body_block);
8617		IrInstSrc *elem_ptr = ir_build_elem_ptr(irb, &spill_scope->base, node, array_val_ptr, index_val,
8618		false, PtrLenSingle, nullptr);
8619		// TODO make it an error to write to element variable or i variable.
8620		Buf *elem_var_name = elem_node->data.symbol_expr.symbol;
8621		ZigVar *elem_var = ir_create_var(irb, elem_node, parent_scope, elem_var_name, true, false, false, is_comptime);
8622		Scope *child_scope = elem_var->child_scope;
8623
8624		IrInstSrc *elem_value = node->data.for_expr.elem_is_ptr ?
8625		elem_ptr : ir_build_load_ptr(irb, &spill_scope->base, elem_node, elem_ptr);
8626		build_decl_var_and_init(irb, parent_scope, elem_node, elem_var, elem_value, buf_ptr(elem_var_name), is_comptime);
8627
8628		if (is_duplicate_label(irb->codegen, child_scope, node, node->data.for_expr.name))
8629		return irb->codegen->invalid_inst_src;
8630
8631		ZigList<IrInstSrc *> incoming_values = {0};
8632		ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
8633		ScopeLoop *loop_scope = create_loop_scope(irb->codegen, node, child_scope);
8634		loop_scope->break_block = end_block;
8635		loop_scope->continue_block = continue_block;
8636		loop_scope->is_comptime = is_comptime;
8637		loop_scope->incoming_blocks = &incoming_blocks;
8638		loop_scope->incoming_values = &incoming_values;
8639		loop_scope->lval = LValNone;
8640		loop_scope->peer_parent = peer_parent;
8641		loop_scope->spill_scope = spill_scope;
8642
8643		// Note the body block of the loop is not the place that lval and result_loc are used -
8644		// it's actually in break statements, handled similarly to return statements.
8645		// That is why we set those values in loop_scope above and not in this ir_gen_node call.
8646		IrInstSrc *body_result = ir_gen_node(irb, body_node, &loop_scope->base);
8647		if (body_result == irb->codegen->invalid_inst_src)
8648		return irb->codegen->invalid_inst_src;
8649
8650		if (loop_scope->name != nullptr && loop_scope->name_used == false) {
8651		add_node_error(irb->codegen, node, buf_sprintf("unused for label"));
8652		}
8653
8654		if (!instr_is_unreachable(body_result)) {
8655		ir_mark_gen(ir_build_check_statement_is_void(irb, child_scope, node->data.for_expr.body, body_result));
8656		ir_mark_gen(ir_build_br(irb, child_scope, node, continue_block, is_comptime));
8657		}
8658
8659		ir_set_cursor_at_end_and_append_block(irb, continue_block);
8660		IrInstSrc *new_index_val = ir_build_bin_op(irb, child_scope, node, IrBinOpAdd, index_val, one, false);
8661		ir_build_store_ptr(irb, child_scope, node, index_ptr, new_index_val)->allow_write_through_const = true;
8662		ir_build_br(irb, child_scope, node, cond_block, is_comptime);
8663
8664		IrInstSrc *else_result = nullptr;
8665		if (else_node) {
8666		ir_set_cursor_at_end_and_append_block(irb, else_block);
8667
8668		if (peer_parent->peers.length != 0) {
8669		peer_parent->peers.last()->next_bb = else_block;
8670		}
8671		ResultLocPeer *peer_result = create_peer_result(peer_parent);
8672		peer_parent->peers.append(peer_result);
8673		else_result = ir_gen_node_extra(irb, else_node, parent_scope, LValNone, &peer_result->base);
8674		if (else_result == irb->codegen->invalid_inst_src)
8675		return else_result;
8676		if (!instr_is_unreachable(else_result))
8677		ir_mark_gen(ir_build_br(irb, parent_scope, node, end_block, is_comptime));
8678		}
8679		IrBasicBlockSrc *after_else_block = irb->current_basic_block;
8680		ir_set_cursor_at_end_and_append_block(irb, end_block);
8681
8682		if (else_result) {
8683		incoming_blocks.append(after_else_block);
8684		incoming_values.append(else_result);
8685		} else {
8686		incoming_blocks.append(after_cond_block);
8687		incoming_values.append(void_else_value);
8688		}
8689		if (peer_parent->peers.length != 0) {
8690		peer_parent->peers.last()->next_bb = end_block;
8691		}
8692
8693		IrInstSrc *phi = ir_build_phi(irb, parent_scope, node, incoming_blocks.length,
8694		incoming_blocks.items, incoming_values.items, peer_parent);
8695		return ir_lval_wrap(irb, parent_scope, phi, lval, result_loc);
8696		}
8697
8698		static IrInstSrc ir_gen_bool_literal(IrBuilderSrc irb, Scope scope, AstNode node) {
8699		assert(node->type == NodeTypeBoolLiteral);
8700		return ir_build_const_bool(irb, scope, node, node->data.bool_literal.value);
8701		}
8702
8703		static IrInstSrc ir_gen_enum_literal(IrBuilderSrc irb, Scope scope, AstNode node) {
8704		assert(node->type == NodeTypeEnumLiteral);
8705		Buf *name = &node->data.enum_literal.identifier->data.str_lit.str;
8706		return ir_build_const_enum_literal(irb, scope, node, name);
8707		}
8708
8709		static IrInstSrc ir_gen_string_literal(IrBuilderSrc irb, Scope scope, AstNode node) {
8710		assert(node->type == NodeTypeStringLiteral);
8711
8712		return ir_build_const_str_lit(irb, scope, node, node->data.string_literal.buf);
8713		}
8714
8715		static IrInstSrc ir_gen_array_type(IrBuilderSrc irb, Scope scope, AstNode node) {
8716		assert(node->type == NodeTypeArrayType);
8717
8718		AstNode *size_node = node->data.array_type.size;
8719		AstNode *child_type_node = node->data.array_type.child_type;
8720		bool is_const = node->data.array_type.is_const;
8721		bool is_volatile = node->data.array_type.is_volatile;
8722		bool is_allow_zero = node->data.array_type.allow_zero_token != nullptr;
8723		AstNode *sentinel_expr = node->data.array_type.sentinel;
8724		AstNode *align_expr = node->data.array_type.align_expr;
8725
8726		Scope *comptime_scope = create_comptime_scope(irb->codegen, node, scope);
8727
8728		IrInstSrc *sentinel;
8729		if (sentinel_expr != nullptr) {
8730		sentinel = ir_gen_node(irb, sentinel_expr, comptime_scope);
8731		if (sentinel == irb->codegen->invalid_inst_src)
8732		return sentinel;
8733		} else {
8734		sentinel = nullptr;
8735		}
8736
8737		if (size_node) {
8738		if (is_const) {
8739		add_node_error(irb->codegen, node, buf_create_from_str("const qualifier invalid on array type"));
8740		return irb->codegen->invalid_inst_src;
8741		}
8742		if (is_volatile) {
8743		add_node_error(irb->codegen, node, buf_create_from_str("volatile qualifier invalid on array type"));
8744		return irb->codegen->invalid_inst_src;
8745		}
8746		if (is_allow_zero) {
8747		add_node_error(irb->codegen, node, buf_create_from_str("allowzero qualifier invalid on array type"));
8748		return irb->codegen->invalid_inst_src;
8749		}
8750		if (align_expr != nullptr) {
8751		add_node_error(irb->codegen, node, buf_create_from_str("align qualifier invalid on array type"));
8752		return irb->codegen->invalid_inst_src;
8753		}
8754
8755		IrInstSrc *size_value = ir_gen_node(irb, size_node, comptime_scope);
8756		if (size_value == irb->codegen->invalid_inst_src)
8757		return size_value;
8758
8759		IrInstSrc *child_type = ir_gen_node(irb, child_type_node, comptime_scope);
8760		if (child_type == irb->codegen->invalid_inst_src)
8761		return child_type;
8762
8763		return ir_build_array_type(irb, scope, node, size_value, sentinel, child_type);
8764		} else {
8765		IrInstSrc *align_value;
8766		if (align_expr != nullptr) {
8767		align_value = ir_gen_node(irb, align_expr, comptime_scope);
8768		if (align_value == irb->codegen->invalid_inst_src)
8769		return align_value;
8770		} else {
8771		align_value = nullptr;
8772		}
8773
8774		IrInstSrc *child_type = ir_gen_node(irb, child_type_node, comptime_scope);
8775		if (child_type == irb->codegen->invalid_inst_src)
8776		return child_type;
8777
8778		return ir_build_slice_type(irb, scope, node, child_type, is_const, is_volatile, sentinel,
8779		align_value, is_allow_zero);
8780		}
8781		}
8782
8783		static IrInstSrc ir_gen_anyframe_type(IrBuilderSrc irb, Scope scope, AstNode node) {
8784		assert(node->type == NodeTypeAnyFrameType);
8785
8786		AstNode *payload_type_node = node->data.anyframe_type.payload_type;
8787		IrInstSrc *payload_type_value = nullptr;
8788
8789		if (payload_type_node != nullptr) {
8790		payload_type_value = ir_gen_node(irb, payload_type_node, scope);
8791		if (payload_type_value == irb->codegen->invalid_inst_src)
8792		return payload_type_value;
8793
8794		}
8795
8796		return ir_build_anyframe_type(irb, scope, node, payload_type_value);
8797		}
8798
8799		static IrInstSrc ir_gen_undefined_literal(IrBuilderSrc irb, Scope scope, AstNode node) {
8800		assert(node->type == NodeTypeUndefinedLiteral);
8801		return ir_build_const_undefined(irb, scope, node);
8802		}
8803
8804		static Error parse_asm_template(IrAnalyze ira, AstNode source_node, Buf *asm_template,
8805		ZigList<AsmToken> *tok_list)
8806		{
8807		// TODO Connect the errors in this function back up to the actual source location
8808		// rather than just the token. https://github.com/ziglang/zig/issues/2080
8809		enum State {
8810		StateStart,
8811		StatePercent,
8812		StateTemplate,
8813		StateVar,
8814		};
8815
8816		assert(tok_list->length == 0);
8817
8818		AsmToken *cur_tok = nullptr;
8819
8820		enum State state = StateStart;
8821
8822		for (size_t i = 0; i < buf_len(asm_template); i += 1) {
8823		uint8_t c = ((uint8_t)buf_ptr(asm_template) + i);
8824		switch (state) {
8825		case StateStart:
8826		if (c == '%') {
8827		tok_list->add_one();
8828		cur_tok = &tok_list->last();
8829		cur_tok->id = AsmTokenIdPercent;
8830		cur_tok->start = i;
8831		state = StatePercent;
8832		} else {
8833		tok_list->add_one();
8834		cur_tok = &tok_list->last();
8835		cur_tok->id = AsmTokenIdTemplate;
8836		cur_tok->start = i;
8837		state = StateTemplate;
8838		}
8839		break;
8840		case StatePercent:
8841		if (c == '%') {
8842		cur_tok->end = i;
8843		state = StateStart;
8844		} else if (c == '[') {
8845		cur_tok->id = AsmTokenIdVar;
8846		state = StateVar;
8847		} else if (c == '=') {
8848		cur_tok->id = AsmTokenIdUniqueId;
8849		cur_tok->end = i;
8850		state = StateStart;
8851		} else {
8852		add_node_error(ira->codegen, source_node,
8853		buf_create_from_str("expected a '%' or '['"));
8854		return ErrorSemanticAnalyzeFail;
8855		}
8856		break;
8857		case StateTemplate:
8858		if (c == '%') {
8859		cur_tok->end = i;
8860		i -= 1;
8861		cur_tok = nullptr;
8862		state = StateStart;
8863		}
8864		break;
8865		case StateVar:
8866		if (c == ']') {
8867		cur_tok->end = i;
8868		state = StateStart;
8869		} else if ((c >= 'a' && c <= 'z') \|\|
8870		(c >= '0' && c <= '9') \|\|
8871		(c == '_'))
8872		{
8873		// do nothing
8874		} else {
8875		add_node_error(ira->codegen, source_node,
8876		buf_sprintf("invalid substitution character: '%c'", c));
8877		return ErrorSemanticAnalyzeFail;
8878		}
8879		break;
8880		}
8881		}
8882
8883		switch (state) {
8884		case StateStart:
8885		break;
8886		case StatePercent:
8887		case StateVar:
8888		add_node_error(ira->codegen, source_node, buf_sprintf("unexpected end of assembly template"));
8889		return ErrorSemanticAnalyzeFail;
8890		case StateTemplate:
8891		cur_tok->end = buf_len(asm_template);
8892		break;
8893		}
8894		return ErrorNone;
8895		}
8896
8897		static size_t find_asm_index(CodeGen g, AstNode node, AsmToken tok, Buf src_template) {
8898		const char *ptr = buf_ptr(src_template) + tok->start + 2;
8899		size_t len = tok->end - tok->start - 2;
8900		size_t result = 0;
8901		for (size_t i = 0; i < node->data.asm_expr.output_list.length; i += 1, result += 1) {
8902		AsmOutput *asm_output = node->data.asm_expr.output_list.at(i);
8903		if (buf_eql_mem(asm_output->asm_symbolic_name, ptr, len)) {
8904		return result;
8905		}
8906		}
8907		for (size_t i = 0; i < node->data.asm_expr.input_list.length; i += 1, result += 1) {
8908		AsmInput *asm_input = node->data.asm_expr.input_list.at(i);
8909		if (buf_eql_mem(asm_input->asm_symbolic_name, ptr, len)) {
8910		return result;
8911		}
8912		}
8913		return SIZE_MAX;
8914		}
8915
8916		static IrInstSrc ir_gen_asm_expr(IrBuilderSrc irb, Scope scope, AstNode node) {
8917		assert(node->type == NodeTypeAsmExpr);
8918		AstNodeAsmExpr *asm_expr = &node->data.asm_expr;
8919
8920		IrInstSrc *asm_template = ir_gen_node(irb, asm_expr->asm_template, scope);
8921		if (asm_template == irb->codegen->invalid_inst_src)
8922		return irb->codegen->invalid_inst_src;
8923
8924		bool is_volatile = asm_expr->volatile_token != nullptr;
8925		bool in_fn_scope = (scope_fn_entry(scope) != nullptr);
8926
8927		if (!in_fn_scope) {
8928		if (is_volatile) {
8929		add_token_error(irb->codegen, node->owner, asm_expr->volatile_token,
8930		buf_sprintf("volatile is meaningless on global assembly"));
8931		return irb->codegen->invalid_inst_src;
8932		}
8933
8934		if (asm_expr->output_list.length != 0 \|\| asm_expr->input_list.length != 0 \|\|
8935		asm_expr->clobber_list.length != 0)
8936		{
8937		add_node_error(irb->codegen, node,
8938		buf_sprintf("global assembly cannot have inputs, outputs, or clobbers"));
8939		return irb->codegen->invalid_inst_src;
8940		}
8941
8942		return ir_build_asm_src(irb, scope, node, asm_template, nullptr, nullptr,
8943		nullptr, 0, is_volatile, true);
8944		}
8945
8946		IrInstSrc *input_list = heap::c_allocator.allocate<IrInstSrc >(asm_expr->input_list.length);
8947		IrInstSrc *output_types = heap::c_allocator.allocate<IrInstSrc >(asm_expr->output_list.length);
8948		ZigVar *output_vars = heap::c_allocator.allocate<ZigVar >(asm_expr->output_list.length);
8949		size_t return_count = 0;
8950		if (!is_volatile && asm_expr->output_list.length == 0) {
8951		add_node_error(irb->codegen, node,
8952		buf_sprintf("assembly expression with no output must be marked volatile"));
8953		return irb->codegen->invalid_inst_src;
8954		}
8955		for (size_t i = 0; i < asm_expr->output_list.length; i += 1) {
8956		AsmOutput *asm_output = asm_expr->output_list.at(i);
8957		if (asm_output->return_type) {
8958		return_count += 1;
8959
8960		IrInstSrc *return_type = ir_gen_node(irb, asm_output->return_type, scope);
8961		if (return_type == irb->codegen->invalid_inst_src)
8962		return irb->codegen->invalid_inst_src;
8963		if (return_count > 1) {
8964		add_node_error(irb->codegen, node,
8965		buf_sprintf("inline assembly allows up to one output value"));
8966		return irb->codegen->invalid_inst_src;
8967		}
8968		output_types[i] = return_type;
8969		} else {
8970		Buf *variable_name = asm_output->variable_name;
8971		// TODO there is some duplication here with ir_gen_symbol. I need to do a full audit of how
8972		// inline assembly works. https://github.com/ziglang/zig/issues/215
8973		ZigVar *var = find_variable(irb->codegen, scope, variable_name, nullptr);
8974		if (var) {
8975		output_vars[i] = var;
8976		} else {
8977		add_node_error(irb->codegen, node,
8978		buf_sprintf("use of undeclared identifier '%s'", buf_ptr(variable_name)));
8979		return irb->codegen->invalid_inst_src;
8980		}
8981		}
8982
8983		const char modifier = *buf_ptr(asm_output->constraint);
8984		if (modifier != '=') {
8985		add_node_error(irb->codegen, node,
8986		buf_sprintf("invalid modifier starting output constraint for '%s': '%c', only '=' is supported."
8987		" Compiler TODO: see https://github.com/ziglang/zig/issues/215",
8988		buf_ptr(asm_output->asm_symbolic_name), modifier));
8989		return irb->codegen->invalid_inst_src;
8990		}
8991		}
8992		for (size_t i = 0; i < asm_expr->input_list.length; i += 1) {
8993		AsmInput *asm_input = asm_expr->input_list.at(i);
8994		IrInstSrc *input_value = ir_gen_node(irb, asm_input->expr, scope);
8995		if (input_value == irb->codegen->invalid_inst_src)
8996		return irb->codegen->invalid_inst_src;
8997
8998		input_list[i] = input_value;
8999		}
9000
9001		return ir_build_asm_src(irb, scope, node, asm_template, input_list, output_types,
9002		output_vars, return_count, is_volatile, false);
9003		}
9004
9005		static IrInstSrc ir_gen_if_optional_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
9006		ResultLoc *result_loc)
9007		{
9008		assert(node->type == NodeTypeIfOptional);
9009
9010		Buf *var_symbol = node->data.test_expr.var_symbol;
9011		AstNode *expr_node = node->data.test_expr.target_node;
9012		AstNode *then_node = node->data.test_expr.then_node;
9013		AstNode *else_node = node->data.test_expr.else_node;
9014		bool var_is_ptr = node->data.test_expr.var_is_ptr;
9015
9016		ScopeExpr *spill_scope = create_expr_scope(irb->codegen, expr_node, scope);
9017		spill_scope->spill_harder = true;
9018
9019		IrInstSrc *maybe_val_ptr = ir_gen_node_extra(irb, expr_node, &spill_scope->base, LValPtr, nullptr);
9020		if (maybe_val_ptr == irb->codegen->invalid_inst_src)
9021		return maybe_val_ptr;
9022
9023		IrInstSrc *maybe_val = ir_build_load_ptr(irb, scope, node, maybe_val_ptr);
9024		IrInstSrc *is_non_null = ir_build_test_non_null_src(irb, scope, node, maybe_val);
9025
9026		IrBasicBlockSrc *then_block = ir_create_basic_block(irb, scope, "OptionalThen");
9027		IrBasicBlockSrc *else_block = ir_create_basic_block(irb, scope, "OptionalElse");
9028		IrBasicBlockSrc *endif_block = ir_create_basic_block(irb, scope, "OptionalEndIf");
9029
9030		IrInstSrc *is_comptime;
9031		if (ir_should_inline(irb->exec, scope)) {
9032		is_comptime = ir_build_const_bool(irb, scope, node, true);
9033		} else {
9034		is_comptime = ir_build_test_comptime(irb, scope, node, is_non_null);
9035		}
9036		IrInstSrc *cond_br_inst = ir_build_cond_br(irb, scope, node, is_non_null,
9037		then_block, else_block, is_comptime);
9038
9039		ResultLocPeerParent *peer_parent = ir_build_binary_result_peers(irb, cond_br_inst, else_block, endif_block,
9040		result_loc, is_comptime);
9041
9042		ir_set_cursor_at_end_and_append_block(irb, then_block);
9043
9044		Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, &spill_scope->base, is_comptime);
9045		Scope *var_scope;
9046		if (var_symbol) {
9047		bool is_shadowable = false;
9048		bool is_const = true;
9049		ZigVar *var = ir_create_var(irb, node, subexpr_scope,
9050		var_symbol, is_const, is_const, is_shadowable, is_comptime);
9051
9052		IrInstSrc *payload_ptr = ir_build_optional_unwrap_ptr(irb, subexpr_scope, node, maybe_val_ptr, false);
9053		IrInstSrc *var_value = var_is_ptr ?
9054		payload_ptr : ir_build_load_ptr(irb, &spill_scope->base, node, payload_ptr);
9055		build_decl_var_and_init(irb, subexpr_scope, node, var, var_value, buf_ptr(var_symbol), is_comptime);
9056		var_scope = var->child_scope;
9057		} else {
9058		var_scope = subexpr_scope;
9059		}
9060		IrInstSrc *then_expr_result = ir_gen_node_extra(irb, then_node, var_scope, lval,
9061		&peer_parent->peers.at(0)->base);
9062		if (then_expr_result == irb->codegen->invalid_inst_src)
9063		return then_expr_result;
9064		IrBasicBlockSrc *after_then_block = irb->current_basic_block;
9065		if (!instr_is_unreachable(then_expr_result))
9066		ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
9067
9068		ir_set_cursor_at_end_and_append_block(irb, else_block);
9069		IrInstSrc *else_expr_result;
9070		if (else_node) {
9071		else_expr_result = ir_gen_node_extra(irb, else_node, subexpr_scope, lval, &peer_parent->peers.at(1)->base);
9072		if (else_expr_result == irb->codegen->invalid_inst_src)
9073		return else_expr_result;
9074		} else {
9075		else_expr_result = ir_build_const_void(irb, scope, node);
9076		ir_build_end_expr(irb, scope, node, else_expr_result, &peer_parent->peers.at(1)->base);
9077		}
9078		IrBasicBlockSrc *after_else_block = irb->current_basic_block;
9079		if (!instr_is_unreachable(else_expr_result))
9080		ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
9081
9082		ir_set_cursor_at_end_and_append_block(irb, endif_block);
9083		IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
9084		incoming_values[0] = then_expr_result;
9085		incoming_values[1] = else_expr_result;
9086		IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
9087		incoming_blocks[0] = after_then_block;
9088		incoming_blocks[1] = after_else_block;
9089
9090		IrInstSrc *phi = ir_build_phi(irb, scope, node, 2, incoming_blocks, incoming_values, peer_parent);
9091		return ir_expr_wrap(irb, scope, phi, result_loc);
9092		}
9093
9094		static IrInstSrc ir_gen_if_err_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
9095		ResultLoc *result_loc)
9096		{
9097		assert(node->type == NodeTypeIfErrorExpr);
9098
9099		AstNode *target_node = node->data.if_err_expr.target_node;
9100		AstNode *then_node = node->data.if_err_expr.then_node;
9101		AstNode *else_node = node->data.if_err_expr.else_node;
9102		bool var_is_ptr = node->data.if_err_expr.var_is_ptr;
9103		bool var_is_const = true;
9104		Buf *var_symbol = node->data.if_err_expr.var_symbol;
9105		Buf *err_symbol = node->data.if_err_expr.err_symbol;
9106
9107		IrInstSrc *err_val_ptr = ir_gen_node_extra(irb, target_node, scope, LValPtr, nullptr);
9108		if (err_val_ptr == irb->codegen->invalid_inst_src)
9109		return err_val_ptr;
9110
9111		IrInstSrc *err_val = ir_build_load_ptr(irb, scope, node, err_val_ptr);
9112		IrInstSrc *is_err = ir_build_test_err_src(irb, scope, node, err_val_ptr, true, false);
9113
9114		IrBasicBlockSrc *ok_block = ir_create_basic_block(irb, scope, "TryOk");
9115		IrBasicBlockSrc *else_block = ir_create_basic_block(irb, scope, "TryElse");
9116		IrBasicBlockSrc *endif_block = ir_create_basic_block(irb, scope, "TryEnd");
9117
9118		bool force_comptime = ir_should_inline(irb->exec, scope);
9119		IrInstSrc *is_comptime = force_comptime ? ir_build_const_bool(irb, scope, node, true) : ir_build_test_comptime(irb, scope, node, is_err);
9120		IrInstSrc *cond_br_inst = ir_build_cond_br(irb, scope, node, is_err, else_block, ok_block, is_comptime);
9121
9122		ResultLocPeerParent *peer_parent = ir_build_binary_result_peers(irb, cond_br_inst, else_block, endif_block,
9123		result_loc, is_comptime);
9124
9125		ir_set_cursor_at_end_and_append_block(irb, ok_block);
9126
9127		Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
9128		Scope *var_scope;
9129		if (var_symbol) {
9130		bool is_shadowable = false;
9131		IrInstSrc *var_is_comptime = force_comptime ? ir_build_const_bool(irb, subexpr_scope, node, true) : ir_build_test_comptime(irb, subexpr_scope, node, err_val);
9132		ZigVar *var = ir_create_var(irb, node, subexpr_scope,
9133		var_symbol, var_is_const, var_is_const, is_shadowable, var_is_comptime);
9134
9135		IrInstSrc *payload_ptr = ir_build_unwrap_err_payload_src(irb, subexpr_scope, node, err_val_ptr, false, false);
9136		IrInstSrc *var_value = var_is_ptr ?
9137		payload_ptr : ir_build_load_ptr(irb, subexpr_scope, node, payload_ptr);
9138		build_decl_var_and_init(irb, subexpr_scope, node, var, var_value, buf_ptr(var_symbol), var_is_comptime);
9139		var_scope = var->child_scope;
9140		} else {
9141		var_scope = subexpr_scope;
9142		}
9143		IrInstSrc *then_expr_result = ir_gen_node_extra(irb, then_node, var_scope, lval,
9144		&peer_parent->peers.at(0)->base);
9145		if (then_expr_result == irb->codegen->invalid_inst_src)
9146		return then_expr_result;
9147		IrBasicBlockSrc *after_then_block = irb->current_basic_block;
9148		if (!instr_is_unreachable(then_expr_result))
9149		ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
9150
9151		ir_set_cursor_at_end_and_append_block(irb, else_block);
9152
9153		IrInstSrc *else_expr_result;
9154		if (else_node) {
9155		Scope *err_var_scope;
9156		if (err_symbol) {
9157		bool is_shadowable = false;
9158		bool is_const = true;
9159		ZigVar *var = ir_create_var(irb, node, subexpr_scope,
9160		err_symbol, is_const, is_const, is_shadowable, is_comptime);
9161
9162		IrInstSrc *err_ptr = ir_build_unwrap_err_code_src(irb, subexpr_scope, node, err_val_ptr);
9163		IrInstSrc *err_value = ir_build_load_ptr(irb, subexpr_scope, node, err_ptr);
9164		build_decl_var_and_init(irb, subexpr_scope, node, var, err_value, buf_ptr(err_symbol), is_comptime);
9165		err_var_scope = var->child_scope;
9166		} else {
9167		err_var_scope = subexpr_scope;
9168		}
9169		else_expr_result = ir_gen_node_extra(irb, else_node, err_var_scope, lval, &peer_parent->peers.at(1)->base);
9170		if (else_expr_result == irb->codegen->invalid_inst_src)
9171		return else_expr_result;
9172		} else {
9173		else_expr_result = ir_build_const_void(irb, scope, node);
9174		ir_build_end_expr(irb, scope, node, else_expr_result, &peer_parent->peers.at(1)->base);
9175		}
9176		IrBasicBlockSrc *after_else_block = irb->current_basic_block;
9177		if (!instr_is_unreachable(else_expr_result))
9178		ir_mark_gen(ir_build_br(irb, scope, node, endif_block, is_comptime));
9179
9180		ir_set_cursor_at_end_and_append_block(irb, endif_block);
9181		IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
9182		incoming_values[0] = then_expr_result;
9183		incoming_values[1] = else_expr_result;
9184		IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
9185		incoming_blocks[0] = after_then_block;
9186		incoming_blocks[1] = after_else_block;
9187
9188		IrInstSrc *phi = ir_build_phi(irb, scope, node, 2, incoming_blocks, incoming_values, peer_parent);
9189		return ir_expr_wrap(irb, scope, phi, result_loc);
9190		}
9191
9192		static bool ir_gen_switch_prong_expr(IrBuilderSrc irb, Scope scope, AstNode switch_node, AstNode prong_node,
9193		IrBasicBlockSrc end_block, IrInstSrc is_comptime, IrInstSrc *var_is_comptime,
9194		IrInstSrc target_value_ptr, IrInstSrc *prong_values, size_t prong_values_len,
9195		ZigList<IrBasicBlockSrc > incoming_blocks, ZigList<IrInstSrc > incoming_values,
9196		IrInstSrcSwitchElseVar *out_switch_else_var, LVal lval, ResultLoc result_loc)
9197		{
9198		assert(switch_node->type == NodeTypeSwitchExpr);
9199		assert(prong_node->type == NodeTypeSwitchProng);
9200
9201		AstNode *expr_node = prong_node->data.switch_prong.expr;
9202		AstNode *var_symbol_node = prong_node->data.switch_prong.var_symbol;
9203		Scope *child_scope;
9204		if (var_symbol_node) {
9205		assert(var_symbol_node->type == NodeTypeSymbol);
9206		Buf *var_name = var_symbol_node->data.symbol_expr.symbol;
9207		bool var_is_ptr = prong_node->data.switch_prong.var_is_ptr;
9208
9209		bool is_shadowable = false;
9210		bool is_const = true;
9211		ZigVar *var = ir_create_var(irb, var_symbol_node, scope,
9212		var_name, is_const, is_const, is_shadowable, var_is_comptime);
9213		child_scope = var->child_scope;
9214		IrInstSrc *var_value;
9215		if (out_switch_else_var != nullptr) {
9216		IrInstSrcSwitchElseVar *switch_else_var = ir_build_switch_else_var(irb, scope, var_symbol_node,
9217		target_value_ptr);
9218		*out_switch_else_var = switch_else_var;
9219		IrInstSrc *payload_ptr = &switch_else_var->base;
9220		var_value = var_is_ptr ?
9221		payload_ptr : ir_build_load_ptr(irb, scope, var_symbol_node, payload_ptr);
9222		} else if (prong_values != nullptr) {
9223		IrInstSrc *payload_ptr = ir_build_switch_var(irb, scope, var_symbol_node, target_value_ptr,
9224		prong_values, prong_values_len);
9225		var_value = var_is_ptr ?
9226		payload_ptr : ir_build_load_ptr(irb, scope, var_symbol_node, payload_ptr);
9227		} else {
9228		var_value = var_is_ptr ?
9229		target_value_ptr : ir_build_load_ptr(irb, scope, var_symbol_node, target_value_ptr);
9230		}
9231		build_decl_var_and_init(irb, scope, var_symbol_node, var, var_value, buf_ptr(var_name), var_is_comptime);
9232		} else {
9233		child_scope = scope;
9234		}
9235
9236		IrInstSrc *expr_result = ir_gen_node_extra(irb, expr_node, child_scope, lval, result_loc);
9237		if (expr_result == irb->codegen->invalid_inst_src)
9238		return false;
9239		if (!instr_is_unreachable(expr_result))
9240		ir_mark_gen(ir_build_br(irb, scope, switch_node, end_block, is_comptime));
9241		incoming_blocks->append(irb->current_basic_block);
9242		incoming_values->append(expr_result);
9243		return true;
9244		}
9245
9246		static IrInstSrc ir_gen_switch_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
9247		ResultLoc *result_loc)
9248		{
9249		assert(node->type == NodeTypeSwitchExpr);
9250
9251		AstNode *target_node = node->data.switch_expr.expr;
9252		IrInstSrc *target_value_ptr = ir_gen_node_extra(irb, target_node, scope, LValPtr, nullptr);
9253		if (target_value_ptr == irb->codegen->invalid_inst_src)
9254		return target_value_ptr;
9255		IrInstSrc *target_value = ir_build_switch_target(irb, scope, node, target_value_ptr);
9256
9257		IrBasicBlockSrc *else_block = ir_create_basic_block(irb, scope, "SwitchElse");
9258		IrBasicBlockSrc *end_block = ir_create_basic_block(irb, scope, "SwitchEnd");
9259
9260		size_t prong_count = node->data.switch_expr.prongs.length;
9261		ZigList<IrInstSrcSwitchBrCase> cases = {0};
9262
9263		IrInstSrc *is_comptime;
9264		IrInstSrc *var_is_comptime;
9265		if (ir_should_inline(irb->exec, scope)) {
9266		is_comptime = ir_build_const_bool(irb, scope, node, true);
9267		var_is_comptime = is_comptime;
9268		} else {
9269		is_comptime = ir_build_test_comptime(irb, scope, node, target_value);
9270		var_is_comptime = ir_build_test_comptime(irb, scope, node, target_value_ptr);
9271		}
9272
9273		ZigList<IrInstSrc *> incoming_values = {0};
9274		ZigList<IrBasicBlockSrc *> incoming_blocks = {0};
9275		ZigList<IrInstSrcCheckSwitchProngsRange> check_ranges = {0};
9276
9277		IrInstSrcSwitchElseVar *switch_else_var = nullptr;
9278
9279		ResultLocPeerParent *peer_parent = heap::c_allocator.create<ResultLocPeerParent>();
9280		peer_parent->base.id = ResultLocIdPeerParent;
9281		peer_parent->base.allow_write_through_const = result_loc->allow_write_through_const;
9282		peer_parent->end_bb = end_block;
9283		peer_parent->is_comptime = is_comptime;
9284		peer_parent->parent = result_loc;
9285
9286		ir_build_reset_result(irb, scope, node, &peer_parent->base);
9287
9288		// First do the else and the ranges
9289		Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, scope, is_comptime);
9290		Scope *comptime_scope = create_comptime_scope(irb->codegen, node, scope);
9291		AstNode *else_prong = nullptr;
9292		AstNode *underscore_prong = nullptr;
9293		for (size_t prong_i = 0; prong_i < prong_count; prong_i += 1) {
9294		AstNode *prong_node = node->data.switch_expr.prongs.at(prong_i);
9295		size_t prong_item_count = prong_node->data.switch_prong.items.length;
9296		if (prong_node->data.switch_prong.any_items_are_range) {
9297		ResultLocPeer *this_peer_result_loc = create_peer_result(peer_parent);
9298
9299		IrInstSrc *ok_bit = nullptr;
9300		AstNode *last_item_node = nullptr;
9301		for (size_t item_i = 0; item_i < prong_item_count; item_i += 1) {
9302		AstNode *item_node = prong_node->data.switch_prong.items.at(item_i);
9303		last_item_node = item_node;
9304		if (item_node->type == NodeTypeSwitchRange) {
9305		AstNode *start_node = item_node->data.switch_range.start;
9306		AstNode *end_node = item_node->data.switch_range.end;
9307
9308		IrInstSrc *start_value = ir_gen_node(irb, start_node, comptime_scope);
9309		if (start_value == irb->codegen->invalid_inst_src)
9310		return irb->codegen->invalid_inst_src;
9311
9312		IrInstSrc *end_value = ir_gen_node(irb, end_node, comptime_scope);
9313		if (end_value == irb->codegen->invalid_inst_src)
9314		return irb->codegen->invalid_inst_src;
9315
9316		IrInstSrcCheckSwitchProngsRange *check_range = check_ranges.add_one();
9317		check_range->start = start_value;
9318		check_range->end = end_value;
9319
9320		IrInstSrc *lower_range_ok = ir_build_bin_op(irb, scope, item_node, IrBinOpCmpGreaterOrEq,
9321		target_value, start_value, false);
9322		IrInstSrc *upper_range_ok = ir_build_bin_op(irb, scope, item_node, IrBinOpCmpLessOrEq,
9323		target_value, end_value, false);
9324		IrInstSrc *both_ok = ir_build_bin_op(irb, scope, item_node, IrBinOpBoolAnd,
9325		lower_range_ok, upper_range_ok, false);
9326		if (ok_bit) {
9327		ok_bit = ir_build_bin_op(irb, scope, item_node, IrBinOpBoolOr, both_ok, ok_bit, false);
9328		} else {
9329		ok_bit = both_ok;
9330		}
9331		} else {
9332		IrInstSrc *item_value = ir_gen_node(irb, item_node, comptime_scope);
9333		if (item_value == irb->codegen->invalid_inst_src)
9334		return irb->codegen->invalid_inst_src;
9335
9336		IrInstSrcCheckSwitchProngsRange *check_range = check_ranges.add_one();
9337		check_range->start = item_value;
9338		check_range->end = item_value;
9339
9340		IrInstSrc *cmp_ok = ir_build_bin_op(irb, scope, item_node, IrBinOpCmpEq,
9341		item_value, target_value, false);
9342		if (ok_bit) {
9343		ok_bit = ir_build_bin_op(irb, scope, item_node, IrBinOpBoolOr, cmp_ok, ok_bit, false);
9344		} else {
9345		ok_bit = cmp_ok;
9346		}
9347		}
9348		}
9349
9350		IrBasicBlockSrc *range_block_yes = ir_create_basic_block(irb, scope, "SwitchRangeYes");
9351		IrBasicBlockSrc *range_block_no = ir_create_basic_block(irb, scope, "SwitchRangeNo");
9352
9353		assert(ok_bit);
9354		assert(last_item_node);
9355		IrInstSrc *br_inst = ir_mark_gen(ir_build_cond_br(irb, scope, last_item_node, ok_bit,
9356		range_block_yes, range_block_no, is_comptime));
9357		if (peer_parent->base.source_instruction == nullptr) {
9358		peer_parent->base.source_instruction = br_inst;
9359		}
9360
9361		if (peer_parent->peers.length > 0) {
9362		peer_parent->peers.last()->next_bb = range_block_yes;
9363		}
9364		peer_parent->peers.append(this_peer_result_loc);
9365		ir_set_cursor_at_end_and_append_block(irb, range_block_yes);
9366		if (!ir_gen_switch_prong_expr(irb, subexpr_scope, node, prong_node, end_block,
9367		is_comptime, var_is_comptime, target_value_ptr, nullptr, 0,
9368		&incoming_blocks, &incoming_values, nullptr, LValNone, &this_peer_result_loc->base))
9369		{
9370		return irb->codegen->invalid_inst_src;
9371		}
9372
9373		ir_set_cursor_at_end_and_append_block(irb, range_block_no);
9374		} else {
9375		if (prong_item_count == 0) {
9376		if (else_prong) {
9377		ErrorMsg *msg = add_node_error(irb->codegen, prong_node,
9378		buf_sprintf("multiple else prongs in switch expression"));
9379		add_error_note(irb->codegen, msg, else_prong,
9380		buf_sprintf("previous else prong is here"));
9381		return irb->codegen->invalid_inst_src;
9382		}
9383		else_prong = prong_node;
9384		} else if (prong_item_count == 1 &&
9385		prong_node->data.switch_prong.items.at(0)->type == NodeTypeSymbol &&
9386		buf_eql_str(prong_node->data.switch_prong.items.at(0)->data.symbol_expr.symbol, "_")) {
9387		if (underscore_prong) {
9388		ErrorMsg *msg = add_node_error(irb->codegen, prong_node,
9389		buf_sprintf("multiple '_' prongs in switch expression"));
9390		add_error_note(irb->codegen, msg, underscore_prong,
9391		buf_sprintf("previous '_' prong is here"));
9392		return irb->codegen->invalid_inst_src;
9393		}
9394		underscore_prong = prong_node;
9395		} else {
9396		continue;
9397		}
9398		if (underscore_prong && else_prong) {
9399		ErrorMsg *msg = add_node_error(irb->codegen, prong_node,
9400		buf_sprintf("else and '_' prong in switch expression"));
9401		if (underscore_prong == prong_node)
9402		add_error_note(irb->codegen, msg, else_prong,
9403		buf_sprintf("else prong is here"));
9404		else
9405		add_error_note(irb->codegen, msg, underscore_prong,
9406		buf_sprintf("'_' prong is here"));
9407		return irb->codegen->invalid_inst_src;
9408		}
9409		ResultLocPeer *this_peer_result_loc = create_peer_result(peer_parent);
9410
9411		IrBasicBlockSrc *prev_block = irb->current_basic_block;
9412		if (peer_parent->peers.length > 0) {
9413		peer_parent->peers.last()->next_bb = else_block;
9414		}
9415		peer_parent->peers.append(this_peer_result_loc);
9416		ir_set_cursor_at_end_and_append_block(irb, else_block);
9417		if (!ir_gen_switch_prong_expr(irb, subexpr_scope, node, prong_node, end_block,
9418		is_comptime, var_is_comptime, target_value_ptr, nullptr, 0, &incoming_blocks, &incoming_values,
9419		&switch_else_var, LValNone, &this_peer_result_loc->base))
9420		{
9421		return irb->codegen->invalid_inst_src;
9422		}
9423		ir_set_cursor_at_end(irb, prev_block);
9424		}
9425		}
	2130	instruction->base.value->type = operand_type;
	2131	instruction->ptr = ptr;
	2132	instruction->ordering = ordering;
9426	2133
9427		// next do the non-else non-ranges
9428		for (size_t prong_i = 0; prong_i < prong_count; prong_i += 1) {
9429		AstNode *prong_node = node->data.switch_expr.prongs.at(prong_i);
9430		size_t prong_item_count = prong_node->data.switch_prong.items.length;
9431		if (prong_item_count == 0)
9432		continue;
9433		if (prong_node->data.switch_prong.any_items_are_range)
9434		continue;
9435		if (underscore_prong == prong_node)
9436		continue;
	2134	ir_ref_inst_gen(ptr);
9437	2135
9438		ResultLocPeer *this_peer_result_loc = create_peer_result(peer_parent);
	2136	return &instruction->base;
	2137	}
9439	2138
9440		IrBasicBlockSrc *prong_block = ir_create_basic_block(irb, scope, "SwitchProng");
9441		IrInstSrc *items = heap::c_allocator.allocate<IrInstSrc >(prong_item_count);
	2139	static IrInstGen ir_build_atomic_store_gen(IrAnalyze ira, IrInst *source_instr,
	2140	IrInstGen ptr, IrInstGen value, AtomicOrder ordering)
	2141	{
	2142	IrInstGenAtomicStore *instruction = ir_build_inst_void<IrInstGenAtomicStore>(&ira->new_irb,
	2143	source_instr->scope, source_instr->source_node);
	2144	instruction->ptr = ptr;
	2145	instruction->value = value;
	2146	instruction->ordering = ordering;
9442	2147
9443		for (size_t item_i = 0; item_i < prong_item_count; item_i += 1) {
9444		AstNode *item_node = prong_node->data.switch_prong.items.at(item_i);
9445		assert(item_node->type != NodeTypeSwitchRange);
	2148	ir_ref_inst_gen(ptr);
	2149	ir_ref_inst_gen(value);
9446	2150
9447		IrInstSrc *item_value = ir_gen_node(irb, item_node, comptime_scope);
9448		if (item_value == irb->codegen->invalid_inst_src)
9449		return irb->codegen->invalid_inst_src;
	2151	return &instruction->base;
	2152	}
9450	2153
9451		IrInstSrcCheckSwitchProngsRange *check_range = check_ranges.add_one();
9452		check_range->start = item_value;
9453		check_range->end = item_value;
9454	2154
9455		IrInstSrcSwitchBrCase *this_case = cases.add_one();
9456		this_case->value = item_value;
9457		this_case->block = prong_block;
	2155	static IrInstGen ir_build_vector_to_array(IrAnalyze ira, IrInst *source_instruction,
	2156	ZigType result_type, IrInstGen vector, IrInstGen *result_loc)
	2157	{
	2158	IrInstGenVectorToArray *instruction = ir_build_inst_gen<IrInstGenVectorToArray>(&ira->new_irb,
	2159	source_instruction->scope, source_instruction->source_node);
	2160	instruction->base.value->type = result_type;
	2161	instruction->vector = vector;
	2162	instruction->result_loc = result_loc;
9458	2163
9459		items[item_i] = item_value;
9460		}
	2164	ir_ref_inst_gen(vector);
	2165	ir_ref_inst_gen(result_loc);
9461	2166
9462		IrBasicBlockSrc *prev_block = irb->current_basic_block;
9463		if (peer_parent->peers.length > 0) {
9464		peer_parent->peers.last()->next_bb = prong_block;
9465		}
9466		peer_parent->peers.append(this_peer_result_loc);
9467		ir_set_cursor_at_end_and_append_block(irb, prong_block);
9468		if (!ir_gen_switch_prong_expr(irb, subexpr_scope, node, prong_node, end_block,
9469		is_comptime, var_is_comptime, target_value_ptr, items, prong_item_count,
9470		&incoming_blocks, &incoming_values, nullptr, LValNone, &this_peer_result_loc->base))
9471		{
9472		return irb->codegen->invalid_inst_src;
9473		}
	2167	return &instruction->base;
	2168	}
9474	2169
9475		ir_set_cursor_at_end(irb, prev_block);
	2170	static IrInstGen ir_build_ptr_of_array_to_slice(IrAnalyze ira, IrInst *source_instruction,
	2171	ZigType result_type, IrInstGen operand, IrInstGen *result_loc)
	2172	{
	2173	IrInstGenPtrOfArrayToSlice *instruction = ir_build_inst_gen<IrInstGenPtrOfArrayToSlice>(&ira->new_irb,
	2174	source_instruction->scope, source_instruction->source_node);
	2175	instruction->base.value->type = result_type;
	2176	instruction->operand = operand;
	2177	instruction->result_loc = result_loc;
9476	2178
9477		}
	2179	ir_ref_inst_gen(operand);
	2180	ir_ref_inst_gen(result_loc);
9478	2181
9479		IrInstSrc *switch_prongs_void = ir_build_check_switch_prongs(irb, scope, node, target_value,
9480		check_ranges.items, check_ranges.length, else_prong, underscore_prong != nullptr);
	2182	return &instruction->base;
	2183	}
9481	2184
9482		IrInstSrc *br_instruction;
9483		if (cases.length == 0) {
9484		br_instruction = ir_build_br(irb, scope, node, else_block, is_comptime);
9485		} else {
9486		IrInstSrcSwitchBr *switch_br = ir_build_switch_br_src(irb, scope, node, target_value, else_block,
9487		cases.length, cases.items, is_comptime, switch_prongs_void);
9488		if (switch_else_var != nullptr) {
9489		switch_else_var->switch_br = switch_br;
9490		}
9491		br_instruction = &switch_br->base;
9492		}
9493		if (peer_parent->base.source_instruction == nullptr) {
9494		peer_parent->base.source_instruction = br_instruction;
9495		}
9496		for (size_t i = 0; i < peer_parent->peers.length; i += 1) {
9497		peer_parent->peers.at(i)->base.source_instruction = peer_parent->base.source_instruction;
9498		}
	2185	static IrInstGen ir_build_array_to_vector(IrAnalyze ira, IrInst *source_instruction,
	2186	IrInstGen array, ZigType result_type)
	2187	{
	2188	IrInstGenArrayToVector *instruction = ir_build_inst_gen<IrInstGenArrayToVector>(&ira->new_irb,
	2189	source_instruction->scope, source_instruction->source_node);
	2190	instruction->base.value->type = result_type;
	2191	instruction->array = array;
9499	2192
9500		if (!else_prong && !underscore_prong) {
9501		if (peer_parent->peers.length != 0) {
9502		peer_parent->peers.last()->next_bb = else_block;
9503		}
9504		ir_set_cursor_at_end_and_append_block(irb, else_block);
9505		ir_build_unreachable(irb, scope, node);
9506		} else {
9507		if (peer_parent->peers.length != 0) {
9508		peer_parent->peers.last()->next_bb = end_block;
9509		}
9510		}
	2193	ir_ref_inst_gen(array);
9511	2194
9512		ir_set_cursor_at_end_and_append_block(irb, end_block);
9513		assert(incoming_blocks.length == incoming_values.length);
9514		IrInstSrc *result_instruction;
9515		if (incoming_blocks.length == 0) {
9516		result_instruction = ir_build_const_void(irb, scope, node);
9517		} else {
9518		result_instruction = ir_build_phi(irb, scope, node, incoming_blocks.length,
9519		incoming_blocks.items, incoming_values.items, peer_parent);
9520		}
9521		return ir_lval_wrap(irb, scope, result_instruction, lval, result_loc);
	2195	return &instruction->base;
9522	2196	}
9523	2197
9524		static IrInstSrc ir_gen_comptime(IrBuilderSrc irb, Scope parent_scope, AstNode node, LVal lval) {
9525		assert(node->type == NodeTypeCompTime);
9526
9527		Scope *child_scope = create_comptime_scope(irb->codegen, node, parent_scope);
9528		// purposefully pass null for result_loc and let EndExpr handle it
9529		return ir_gen_node_extra(irb, node->data.comptime_expr.expr, child_scope, lval, nullptr);
9530		}
	2198	static IrInstGen ir_build_assert_zero(IrAnalyze ira, IrInst *source_instruction,
	2199	IrInstGen *target)
	2200	{
	2201	IrInstGenAssertZero *instruction = ir_build_inst_gen<IrInstGenAssertZero>(&ira->new_irb,
	2202	source_instruction->scope, source_instruction->source_node);
	2203	instruction->base.value->type = ira->codegen->builtin_types.entry_void;
	2204	instruction->target = target;
9531	2205
9532		static IrInstSrc ir_gen_nosuspend(IrBuilderSrc irb, Scope parent_scope, AstNode node, LVal lval) {
9533		assert(node->type == NodeTypeNoSuspend);
	2206	ir_ref_inst_gen(target);
9534	2207
9535		Scope *child_scope = create_nosuspend_scope(irb->codegen, node, parent_scope);
9536		// purposefully pass null for result_loc and let EndExpr handle it
9537		return ir_gen_node_extra(irb, node->data.nosuspend_expr.expr, child_scope, lval, nullptr);
	2208	return &instruction->base;
9538	2209	}
9539	2210
9540		static IrInstSrc ir_gen_return_from_block(IrBuilderSrc irb, Scope break_scope, AstNode node, ScopeBlock *block_scope) {
9541		IrInstSrc *is_comptime;
9542		if (ir_should_inline(irb->exec, break_scope)) {
9543		is_comptime = ir_build_const_bool(irb, break_scope, node, true);
9544		} else {
9545		is_comptime = block_scope->is_comptime;
9546		}
9547
9548		IrInstSrc *result_value;
9549		if (node->data.break_expr.expr) {
9550		ResultLocPeer *peer_result = create_peer_result(block_scope->peer_parent);
9551		block_scope->peer_parent->peers.append(peer_result);
9552
9553		result_value = ir_gen_node_extra(irb, node->data.break_expr.expr, break_scope, block_scope->lval,
9554		&peer_result->base);
9555		if (result_value == irb->codegen->invalid_inst_src)
9556		return irb->codegen->invalid_inst_src;
9557		} else {
9558		result_value = ir_build_const_void(irb, break_scope, node);
9559		}
	2211	static IrInstGen ir_build_assert_non_null(IrAnalyze ira, IrInst *source_instruction,
	2212	IrInstGen *target)
	2213	{
	2214	IrInstGenAssertNonNull *instruction = ir_build_inst_gen<IrInstGenAssertNonNull>(&ira->new_irb,
	2215	source_instruction->scope, source_instruction->source_node);
	2216	instruction->base.value->type = ira->codegen->builtin_types.entry_void;
	2217	instruction->target = target;
9560	2218
9561		IrBasicBlockSrc *dest_block = block_scope->end_block;
9562		if (!ir_gen_defers_for_block(irb, break_scope, dest_block->scope, nullptr, nullptr))
9563		return irb->codegen->invalid_inst_src;
	2219	ir_ref_inst_gen(target);
9564	2220
9565		block_scope->incoming_blocks->append(irb->current_basic_block);
9566		block_scope->incoming_values->append(result_value);
9567		return ir_build_br(irb, break_scope, node, dest_block, is_comptime);
	2221	return &instruction->base;
9568	2222	}
9569	2223
9570		static IrInstSrc ir_gen_break(IrBuilderSrc irb, Scope break_scope, AstNode node) {
9571		assert(node->type == NodeTypeBreak);
	2224	static IrInstGenAlloca ir_build_alloca_gen(IrAnalyze ira, IrInst *source_instruction,
	2225	uint32_t align, const char *name_hint)
	2226	{
	2227	IrInstGenAlloca *instruction = ir_create_inst_gen<IrInstGenAlloca>(&ira->new_irb,
	2228	source_instruction->scope, source_instruction->source_node);
	2229	instruction->align = align;
	2230	instruction->name_hint = name_hint;
9572	2231
9573		// Search up the scope. We'll find one of these things first:
9574		// * function definition scope or global scope => error, break outside loop
9575		// * defer expression scope => error, cannot break out of defer expression
9576		// * loop scope => OK
9577		// * (if it's a labeled break) labeled block => OK
	2232	return instruction;
	2233	}
9578	2234
9579		Scope *search_scope = break_scope;
9580		ScopeLoop *loop_scope;
9581		for (;;) {
9582		if (search_scope == nullptr \|\| search_scope->id == ScopeIdFnDef) {
9583		if (node->data.break_expr.name != nullptr) {
9584		add_node_error(irb->codegen, node, buf_sprintf("label not found: '%s'", buf_ptr(node->data.break_expr.name)));
9585		return irb->codegen->invalid_inst_src;
9586		} else {
9587		add_node_error(irb->codegen, node, buf_sprintf("break expression outside loop"));
9588		return irb->codegen->invalid_inst_src;
9589		}
9590		} else if (search_scope->id == ScopeIdDeferExpr) {
9591		add_node_error(irb->codegen, node, buf_sprintf("cannot break out of defer expression"));
9592		return irb->codegen->invalid_inst_src;
9593		} else if (search_scope->id == ScopeIdLoop) {
9594		ScopeLoop this_loop_scope = (ScopeLoop )search_scope;
9595		if (node->data.break_expr.name == nullptr \|\|
9596		(this_loop_scope->name != nullptr && buf_eql_buf(node->data.break_expr.name, this_loop_scope->name)))
9597		{
9598		this_loop_scope->name_used = true;
9599		loop_scope = this_loop_scope;
9600		break;
9601		}
9602		} else if (search_scope->id == ScopeIdBlock) {
9603		ScopeBlock this_block_scope = (ScopeBlock )search_scope;
9604		if (node->data.break_expr.name != nullptr &&
9605		(this_block_scope->name != nullptr && buf_eql_buf(node->data.break_expr.name, this_block_scope->name)))
9606		{
9607		assert(this_block_scope->end_block != nullptr);
9608		this_block_scope->name_used = true;
9609		return ir_gen_return_from_block(irb, break_scope, node, this_block_scope);
9610		}
9611		} else if (search_scope->id == ScopeIdSuspend) {
9612		add_node_error(irb->codegen, node, buf_sprintf("cannot break out of suspend block"));
9613		return irb->codegen->invalid_inst_src;
9614		}
9615		search_scope = search_scope->parent;
9616		}
	2235	static IrInstGen ir_build_suspend_finish_gen(IrAnalyze ira, IrInst source_instr, IrInstGenSuspendBegin begin) {
	2236	IrInstGenSuspendFinish *inst = ir_build_inst_void<IrInstGenSuspendFinish>(&ira->new_irb,
	2237	source_instr->scope, source_instr->source_node);
	2238	inst->begin = begin;
9617	2239
9618		IrInstSrc *is_comptime;
9619		if (ir_should_inline(irb->exec, break_scope)) {
9620		is_comptime = ir_build_const_bool(irb, break_scope, node, true);
9621		} else {
9622		is_comptime = loop_scope->is_comptime;
9623		}
	2240	ir_ref_inst_gen(&begin->base);
9624	2241
9625		IrInstSrc *result_value;
9626		if (node->data.break_expr.expr) {
9627		ResultLocPeer *peer_result = create_peer_result(loop_scope->peer_parent);
9628		loop_scope->peer_parent->peers.append(peer_result);
	2242	return &inst->base;
	2243	}
9629	2244
9630		result_value = ir_gen_node_extra(irb, node->data.break_expr.expr, break_scope,
9631		loop_scope->lval, &peer_result->base);
9632		if (result_value == irb->codegen->invalid_inst_src)
9633		return irb->codegen->invalid_inst_src;
9634		} else {
9635		result_value = ir_build_const_void(irb, break_scope, node);
9636		}
	2245	static IrInstGenAwait ir_build_await_gen(IrAnalyze ira, IrInst *source_instruction,
	2246	IrInstGen frame, ZigType result_type, IrInstGen *result_loc, bool is_nosuspend)
	2247	{
	2248	IrInstGenAwait *instruction = ir_build_inst_gen<IrInstGenAwait>(&ira->new_irb,
	2249	source_instruction->scope, source_instruction->source_node);
	2250	instruction->base.value->type = result_type;
	2251	instruction->frame = frame;
	2252	instruction->result_loc = result_loc;
	2253	instruction->is_nosuspend = is_nosuspend;
9637	2254
9638		IrBasicBlockSrc *dest_block = loop_scope->break_block;
9639		if (!ir_gen_defers_for_block(irb, break_scope, dest_block->scope, nullptr, nullptr))
9640		return irb->codegen->invalid_inst_src;
	2255	ir_ref_inst_gen(frame);
	2256	if (result_loc != nullptr) ir_ref_inst_gen(result_loc);
9641	2257
9642		loop_scope->incoming_blocks->append(irb->current_basic_block);
9643		loop_scope->incoming_values->append(result_value);
9644		return ir_build_br(irb, break_scope, node, dest_block, is_comptime);
	2258	return instruction;
9645	2259	}
9646	2260
9647		static IrInstSrc ir_gen_continue(IrBuilderSrc irb, Scope continue_scope, AstNode node) {
9648		assert(node->type == NodeTypeContinue);
9649
9650		// Search up the scope. We'll find one of these things first:
9651		// * function definition scope or global scope => error, break outside loop
9652		// * defer expression scope => error, cannot break out of defer expression
9653		// * loop scope => OK
	2261	static IrInstGen ir_build_resume_gen(IrAnalyze ira, IrInst source_instr, IrInstGen frame) {
	2262	IrInstGenResume *instruction = ir_build_inst_void<IrInstGenResume>(&ira->new_irb,
	2263	source_instr->scope, source_instr->source_node);
	2264	instruction->frame = frame;
9654	2265
9655		ZigList<ScopeRuntime *> runtime_scopes = {};
	2266	ir_ref_inst_gen(frame);
9656	2267
9657		Scope *search_scope = continue_scope;
9658		ScopeLoop *loop_scope;
9659		for (;;) {
9660		if (search_scope == nullptr \|\| search_scope->id == ScopeIdFnDef) {
9661		if (node->data.continue_expr.name != nullptr) {
9662		add_node_error(irb->codegen, node, buf_sprintf("labeled loop not found: '%s'", buf_ptr(node->data.continue_expr.name)));
9663		return irb->codegen->invalid_inst_src;
9664		} else {
9665		add_node_error(irb->codegen, node, buf_sprintf("continue expression outside loop"));
9666		return irb->codegen->invalid_inst_src;
9667		}
9668		} else if (search_scope->id == ScopeIdDeferExpr) {
9669		add_node_error(irb->codegen, node, buf_sprintf("cannot continue out of defer expression"));
9670		return irb->codegen->invalid_inst_src;
9671		} else if (search_scope->id == ScopeIdLoop) {
9672		ScopeLoop this_loop_scope = (ScopeLoop )search_scope;
9673		if (node->data.continue_expr.name == nullptr \|\|
9674		(this_loop_scope->name != nullptr && buf_eql_buf(node->data.continue_expr.name, this_loop_scope->name)))
9675		{
9676		this_loop_scope->name_used = true;
9677		loop_scope = this_loop_scope;
9678		break;
9679		}
9680		} else if (search_scope->id == ScopeIdRuntime) {
9681		ScopeRuntime scope_runtime = (ScopeRuntime )search_scope;
9682		runtime_scopes.append(scope_runtime);
9683		}
9684		search_scope = search_scope->parent;
9685		}
	2268	return &instruction->base;
	2269	}
9686	2270
9687		IrInstSrc *is_comptime;
9688		if (ir_should_inline(irb->exec, continue_scope)) {
9689		is_comptime = ir_build_const_bool(irb, continue_scope, node, true);
9690		} else {
9691		is_comptime = loop_scope->is_comptime;
9692		}
	2271	static IrInstGen ir_build_spill_begin_gen(IrAnalyze ira, IrInst source_instr, IrInstGen operand,
	2272	SpillId spill_id)
	2273	{
	2274	IrInstGenSpillBegin *instruction = ir_build_inst_void<IrInstGenSpillBegin>(&ira->new_irb,
	2275	source_instr->scope, source_instr->source_node);
	2276	instruction->operand = operand;
	2277	instruction->spill_id = spill_id;
9693	2278
9694		for (size_t i = 0; i < runtime_scopes.length; i += 1) {
9695		ScopeRuntime *scope_runtime = runtime_scopes.at(i);
9696		ir_mark_gen(ir_build_check_runtime_scope(irb, continue_scope, node, scope_runtime->is_comptime, is_comptime));
9697		}
9698		runtime_scopes.deinit();
	2279	ir_ref_inst_gen(operand);
9699	2280
9700		IrBasicBlockSrc *dest_block = loop_scope->continue_block;
9701		if (!ir_gen_defers_for_block(irb, continue_scope, dest_block->scope, nullptr, nullptr))
9702		return irb->codegen->invalid_inst_src;
9703		return ir_mark_gen(ir_build_br(irb, continue_scope, node, dest_block, is_comptime));
	2281	return &instruction->base;
9704	2282	}
9705	2283
9706		static IrInstSrc ir_gen_error_type(IrBuilderSrc irb, Scope scope, AstNode node) {
9707		assert(node->type == NodeTypeErrorType);
9708		return ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_global_error_set);
9709		}
	2284	static IrInstGen ir_build_spill_end_gen(IrAnalyze ira, IrInst source_instr, IrInstGenSpillBegin begin,
	2285	ZigType *result_type)
	2286	{
	2287	IrInstGenSpillEnd *instruction = ir_build_inst_gen<IrInstGenSpillEnd>(&ira->new_irb,
	2288	source_instr->scope, source_instr->source_node);
	2289	instruction->base.value->type = result_type;
	2290	instruction->begin = begin;
	2291
	2292	ir_ref_inst_gen(&begin->base);
9710	2293
9711		static IrInstSrc ir_gen_defer(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
9712		assert(node->type == NodeTypeDefer);
	2294	return &instruction->base;
	2295	}
9713	2296
9714		ScopeDefer *defer_child_scope = create_defer_scope(irb->codegen, node, parent_scope);
9715		node->data.defer.child_scope = &defer_child_scope->base;
	2297	static IrInstGen ir_build_vector_extract_elem(IrAnalyze ira, IrInst *source_instruction,
	2298	IrInstGen vector, IrInstGen index)
	2299	{
	2300	IrInstGenVectorExtractElem *instruction = ir_build_inst_gen<IrInstGenVectorExtractElem>(
	2301	&ira->new_irb, source_instruction->scope, source_instruction->source_node);
	2302	instruction->base.value->type = vector->value->type->data.vector.elem_type;
	2303	instruction->vector = vector;
	2304	instruction->index = index;
9716	2305
9717		ScopeDeferExpr *defer_expr_scope = create_defer_expr_scope(irb->codegen, node, parent_scope);
9718		node->data.defer.expr_scope = &defer_expr_scope->base;
	2306	ir_ref_inst_gen(vector);
	2307	ir_ref_inst_gen(index);
9719	2308
9720		return ir_build_const_void(irb, parent_scope, node);
	2309	return &instruction->base;
9721	2310	}
9722	2311
9723		static IrInstSrc ir_gen_slice(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval, ResultLoc *result_loc) {
9724		assert(node->type == NodeTypeSliceExpr);
9725
9726		AstNodeSliceExpr *slice_expr = &node->data.slice_expr;
9727		AstNode *array_node = slice_expr->array_ref_expr;
9728		AstNode *start_node = slice_expr->start;
9729		AstNode *end_node = slice_expr->end;
9730		AstNode *sentinel_node = slice_expr->sentinel;
	2312	static IrInstGen ir_build_wasm_memory_size_gen(IrAnalyze ira, IrInst source_instr, IrInstGen index) {
	2313	IrInstGenWasmMemorySize *instruction = ir_build_inst_gen<IrInstGenWasmMemorySize>(&ira->new_irb,
	2314	source_instr->scope, source_instr->source_node);
	2315	instruction->base.value->type = ira->codegen->builtin_types.entry_u32;
	2316	instruction->index = index;
9731	2317
9732		IrInstSrc *ptr_value = ir_gen_node_extra(irb, array_node, scope, LValPtr, nullptr);
9733		if (ptr_value == irb->codegen->invalid_inst_src)
9734		return irb->codegen->invalid_inst_src;
	2318	ir_ref_inst_gen(index);
9735	2319
9736		IrInstSrc *start_value = ir_gen_node(irb, start_node, scope);
9737		if (start_value == irb->codegen->invalid_inst_src)
9738		return irb->codegen->invalid_inst_src;
	2320	return &instruction->base;
	2321	}
9739	2322
9740		IrInstSrc *end_value;
9741		if (end_node) {
9742		end_value = ir_gen_node(irb, end_node, scope);
9743		if (end_value == irb->codegen->invalid_inst_src)
9744		return irb->codegen->invalid_inst_src;
9745		} else {
9746		end_value = nullptr;
9747		}
	2323	static IrInstGen ir_build_wasm_memory_grow_gen(IrAnalyze ira, IrInst source_instr, IrInstGen index, IrInstGen *delta) {
	2324	IrInstGenWasmMemoryGrow *instruction = ir_build_inst_gen<IrInstGenWasmMemoryGrow>(&ira->new_irb,
	2325	source_instr->scope, source_instr->source_node);
	2326	instruction->base.value->type = ira->codegen->builtin_types.entry_i32;
	2327	instruction->index = index;
	2328	instruction->delta = delta;
9748	2329
9749		IrInstSrc *sentinel_value;
9750		if (sentinel_node) {
9751		sentinel_value = ir_gen_node(irb, sentinel_node, scope);
9752		if (sentinel_value == irb->codegen->invalid_inst_src)
9753		return irb->codegen->invalid_inst_src;
9754		} else {
9755		sentinel_value = nullptr;
9756		}
	2330	ir_ref_inst_gen(index);
	2331	ir_ref_inst_gen(delta);
9757	2332
9758		IrInstSrc *slice = ir_build_slice_src(irb, scope, node, ptr_value, start_value, end_value,
9759		sentinel_value, true, result_loc);
9760		return ir_lval_wrap(irb, scope, slice, lval, result_loc);
	2333	return &instruction->base;
9761	2334	}
9762	2335
9763		static IrInstSrc ir_gen_catch(IrBuilderSrc irb, Scope parent_scope, AstNode node, LVal lval,
9764		ResultLoc *result_loc)
	2336	static Error parse_asm_template(IrAnalyze ira, AstNode source_node, Buf *asm_template,
	2337	ZigList<AsmToken> *tok_list)
9765	2338	{
9766		assert(node->type == NodeTypeCatchExpr);
	2339	// TODO Connect the errors in this function back up to the actual source location
	2340	// rather than just the token. https://github.com/ziglang/zig/issues/2080
	2341	enum State {
	2342	StateStart,
	2343	StatePercent,
	2344	StateTemplate,
	2345	StateVar,
	2346	};
	2347
	2348	assert(tok_list->length == 0);
	2349
	2350	AsmToken *cur_tok = nullptr;
9767	2351
9768		AstNode *op1_node = node->data.unwrap_err_expr.op1;
9769		AstNode *op2_node = node->data.unwrap_err_expr.op2;
9770		AstNode *var_node = node->data.unwrap_err_expr.symbol;
	2352	enum State state = StateStart;
9771	2353
9772		if (op2_node->type == NodeTypeUnreachable) {
9773		if (var_node != nullptr) {
9774		assert(var_node->type == NodeTypeSymbol);
9775		Buf *var_name = var_node->data.symbol_expr.symbol;
9776		add_node_error(irb->codegen, var_node, buf_sprintf("unused variable: '%s'", buf_ptr(var_name)));
9777		return irb->codegen->invalid_inst_src;
	2354	for (size_t i = 0; i < buf_len(asm_template); i += 1) {
	2355	uint8_t c = ((uint8_t)buf_ptr(asm_template) + i);
	2356	switch (state) {
	2357	case StateStart:
	2358	if (c == '%') {
	2359	tok_list->add_one();
	2360	cur_tok = &tok_list->last();
	2361	cur_tok->id = AsmTokenIdPercent;
	2362	cur_tok->start = i;
	2363	state = StatePercent;
	2364	} else {
	2365	tok_list->add_one();
	2366	cur_tok = &tok_list->last();
	2367	cur_tok->id = AsmTokenIdTemplate;
	2368	cur_tok->start = i;
	2369	state = StateTemplate;
	2370	}
	2371	break;
	2372	case StatePercent:
	2373	if (c == '%') {
	2374	cur_tok->end = i;
	2375	state = StateStart;
	2376	} else if (c == '[') {
	2377	cur_tok->id = AsmTokenIdVar;
	2378	state = StateVar;
	2379	} else if (c == '=') {
	2380	cur_tok->id = AsmTokenIdUniqueId;
	2381	cur_tok->end = i;
	2382	state = StateStart;
	2383	} else {
	2384	add_node_error(ira->codegen, source_node,
	2385	buf_create_from_str("expected a '%' or '['"));
	2386	return ErrorSemanticAnalyzeFail;
	2387	}
	2388	break;
	2389	case StateTemplate:
	2390	if (c == '%') {
	2391	cur_tok->end = i;
	2392	i -= 1;
	2393	cur_tok = nullptr;
	2394	state = StateStart;
	2395	}
	2396	break;
	2397	case StateVar:
	2398	if (c == ']') {
	2399	cur_tok->end = i;
	2400	state = StateStart;
	2401	} else if ((c >= 'a' && c <= 'z') \|\|
	2402	(c >= '0' && c <= '9') \|\|
	2403	(c == '_'))
	2404	{
	2405	// do nothing
	2406	} else {
	2407	add_node_error(ira->codegen, source_node,
	2408	buf_sprintf("invalid substitution character: '%c'", c));
	2409	return ErrorSemanticAnalyzeFail;
	2410	}
	2411	break;
9778	2412	}
9779		return ir_gen_catch_unreachable(irb, parent_scope, node, op1_node, lval, result_loc);
9780		}
9781
9782
9783		ScopeExpr *spill_scope = create_expr_scope(irb->codegen, op1_node, parent_scope);
9784		spill_scope->spill_harder = true;
9785
9786		IrInstSrc *err_union_ptr = ir_gen_node_extra(irb, op1_node, &spill_scope->base, LValPtr, nullptr);
9787		if (err_union_ptr == irb->codegen->invalid_inst_src)
9788		return irb->codegen->invalid_inst_src;
9789
9790		IrInstSrc *is_err = ir_build_test_err_src(irb, parent_scope, node, err_union_ptr, true, false);
9791
9792		IrInstSrc *is_comptime;
9793		if (ir_should_inline(irb->exec, parent_scope)) {
9794		is_comptime = ir_build_const_bool(irb, parent_scope, node, true);
9795		} else {
9796		is_comptime = ir_build_test_comptime(irb, parent_scope, node, is_err);
9797		}
9798
9799		IrBasicBlockSrc *ok_block = ir_create_basic_block(irb, parent_scope, "UnwrapErrOk");
9800		IrBasicBlockSrc *err_block = ir_create_basic_block(irb, parent_scope, "UnwrapErrError");
9801		IrBasicBlockSrc *end_block = ir_create_basic_block(irb, parent_scope, "UnwrapErrEnd");
9802		IrInstSrc *cond_br_inst = ir_build_cond_br(irb, parent_scope, node, is_err, err_block, ok_block, is_comptime);
9803
9804		ResultLocPeerParent *peer_parent = ir_build_binary_result_peers(irb, cond_br_inst, ok_block, end_block, result_loc,
9805		is_comptime);
9806
9807		ir_set_cursor_at_end_and_append_block(irb, err_block);
9808		Scope *subexpr_scope = create_runtime_scope(irb->codegen, node, &spill_scope->base, is_comptime);
9809		Scope *err_scope;
9810		if (var_node) {
9811		assert(var_node->type == NodeTypeSymbol);
9812		Buf *var_name = var_node->data.symbol_expr.symbol;
9813		bool is_const = true;
9814		bool is_shadowable = false;
9815		ZigVar *var = ir_create_var(irb, node, subexpr_scope, var_name,
9816		is_const, is_const, is_shadowable, is_comptime);
9817		err_scope = var->child_scope;
9818		IrInstSrc *err_ptr = ir_build_unwrap_err_code_src(irb, err_scope, node, err_union_ptr);
9819		IrInstSrc *err_value = ir_build_load_ptr(irb, err_scope, var_node, err_ptr);
9820		build_decl_var_and_init(irb, err_scope, var_node, var, err_value, buf_ptr(var_name), is_comptime);
9821		} else {
9822		err_scope = subexpr_scope;
9823		}
9824		IrInstSrc *err_result = ir_gen_node_extra(irb, op2_node, err_scope, LValNone, &peer_parent->peers.at(0)->base);
9825		if (err_result == irb->codegen->invalid_inst_src)
9826		return irb->codegen->invalid_inst_src;
9827		IrBasicBlockSrc *after_err_block = irb->current_basic_block;
9828		if (!instr_is_unreachable(err_result))
9829		ir_mark_gen(ir_build_br(irb, parent_scope, node, end_block, is_comptime));
9830
9831		ir_set_cursor_at_end_and_append_block(irb, ok_block);
9832		IrInstSrc *unwrapped_ptr = ir_build_unwrap_err_payload_src(irb, parent_scope, node, err_union_ptr, false, false);
9833		IrInstSrc *unwrapped_payload = ir_build_load_ptr(irb, parent_scope, node, unwrapped_ptr);
9834		ir_build_end_expr(irb, parent_scope, node, unwrapped_payload, &peer_parent->peers.at(1)->base);
9835		IrBasicBlockSrc *after_ok_block = irb->current_basic_block;
9836		ir_build_br(irb, parent_scope, node, end_block, is_comptime);
9837
9838		ir_set_cursor_at_end_and_append_block(irb, end_block);
9839		IrInstSrc *incoming_values = heap::c_allocator.allocate<IrInstSrc >(2);
9840		incoming_values[0] = err_result;
9841		incoming_values[1] = unwrapped_payload;
9842		IrBasicBlockSrc *incoming_blocks = heap::c_allocator.allocate<IrBasicBlockSrc >(2);
9843		incoming_blocks[0] = after_err_block;
9844		incoming_blocks[1] = after_ok_block;
9845		IrInstSrc *phi = ir_build_phi(irb, parent_scope, node, 2, incoming_blocks, incoming_values, peer_parent);
9846		return ir_lval_wrap(irb, parent_scope, phi, lval, result_loc);
9847		}
9848
9849		static bool render_instance_name_recursive(CodeGen codegen, Buf name, Scope outer_scope, Scope inner_scope) {
9850		if (inner_scope == nullptr \|\| inner_scope == outer_scope) return false;
9851		bool need_comma = render_instance_name_recursive(codegen, name, outer_scope, inner_scope->parent);
9852		if (inner_scope->id != ScopeIdVarDecl)
9853		return need_comma;
9854
9855		ScopeVarDecl var_scope = (ScopeVarDecl )inner_scope;
9856		if (need_comma)
9857		buf_append_char(name, ',');
9858		// TODO: const ptr reinterpret here to make the var type agree with the value?
9859		render_const_value(codegen, name, var_scope->var->const_value);
9860		return true;
9861		}
9862
9863		static Buf get_anon_type_name(CodeGen codegen, IrExecutableSrc exec, const char kind_name,
9864		Scope scope, AstNode source_node, Buf *out_bare_name)
9865		{
9866		if (exec != nullptr && exec->name) {
9867		ZigType *import = get_scope_import(scope);
9868		Buf *namespace_name = buf_alloc();
9869		append_namespace_qualification(codegen, namespace_name, import);
9870		buf_append_buf(namespace_name, exec->name);
9871		buf_init_from_buf(out_bare_name, exec->name);
9872		return namespace_name;
9873		} else if (exec != nullptr && exec->name_fn != nullptr) {
9874		Buf *name = buf_alloc();
9875		buf_append_buf(name, &exec->name_fn->symbol_name);
9876		buf_appendf(name, "(");
9877		render_instance_name_recursive(codegen, name, &exec->name_fn->fndef_scope->base, exec->begin_scope);
9878		buf_appendf(name, ")");
9879		buf_init_from_buf(out_bare_name, name);
9880		return name;
9881		} else {
9882		ZigType *import = get_scope_import(scope);
9883		Buf *namespace_name = buf_alloc();
9884		append_namespace_qualification(codegen, namespace_name, import);
9885		buf_appendf(namespace_name, "%s:%" ZIG_PRI_usize ":%" ZIG_PRI_usize, kind_name,
9886		source_node->line + 1, source_node->column + 1);
9887		buf_init_from_buf(out_bare_name, namespace_name);
9888		return namespace_name;
9889	2413	}
9890		}
9891	2414
9892		static IrInstSrc ir_gen_container_decl(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
9893		assert(node->type == NodeTypeContainerDecl);
9894
9895		ContainerKind kind = node->data.container_decl.kind;
9896		Buf *bare_name = buf_alloc();
9897		Buf *name = get_anon_type_name(irb->codegen, irb->exec, container_string(kind), parent_scope, node, bare_name);
9898
9899		ContainerLayout layout = node->data.container_decl.layout;
9900		ZigType *container_type = get_partial_container_type(irb->codegen, parent_scope,
9901		kind, node, buf_ptr(name), bare_name, layout);
9902		ScopeDecls *child_scope = get_container_scope(container_type);
9903
9904		for (size_t i = 0; i < node->data.container_decl.decls.length; i += 1) {
9905		AstNode *child_node = node->data.container_decl.decls.at(i);
9906		scan_decls(irb->codegen, child_scope, child_node);
	2415	switch (state) {
	2416	case StateStart:
	2417	break;
	2418	case StatePercent:
	2419	case StateVar:
	2420	add_node_error(ira->codegen, source_node, buf_sprintf("unexpected end of assembly template"));
	2421	return ErrorSemanticAnalyzeFail;
	2422	case StateTemplate:
	2423	cur_tok->end = buf_len(asm_template);
	2424	break;
9907	2425	}
9908
9909		TldContainer *tld_container = heap::c_allocator.create<TldContainer>();
9910		init_tld(&tld_container->base, TldIdContainer, bare_name, VisibModPub, node, parent_scope);
9911		tld_container->type_entry = container_type;
9912		tld_container->decls_scope = child_scope;
9913		irb->codegen->resolve_queue.append(&tld_container->base);
9914
9915		// Add this to the list to mark as invalid if analyzing this exec fails.
9916		irb->exec->tld_list.append(&tld_container->base);
9917
9918		return ir_build_const_type(irb, parent_scope, node, container_type);
	2426	return ErrorNone;
9919	2427	}
9920	2428
9921	2429	// errors should be populated with set1's values
...	...	@@ -10003,497 +2511,6 @@ static ZigType make_err_set_with_one_item(CodeGen g, Scope *parent_scope, AstN
10003	2511	return err_set_type;
10004	2512	}
10005	2513
10006		static AstNode ast_field_to_symbol_node(AstNode err_set_field_node) {
10007		if (err_set_field_node->type == NodeTypeSymbol) {
10008		return err_set_field_node;
10009		} else if (err_set_field_node->type == NodeTypeErrorSetField) {
10010		assert(err_set_field_node->data.err_set_field.field_name->type == NodeTypeSymbol);
10011		return err_set_field_node->data.err_set_field.field_name;
10012		} else {
10013		return err_set_field_node;
10014		}
10015		}
10016
10017		static IrInstSrc ir_gen_err_set_decl(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
10018		assert(node->type == NodeTypeErrorSetDecl);
10019
10020		uint32_t err_count = node->data.err_set_decl.decls.length;
10021
10022		Buf bare_name = BUF_INIT;
10023		Buf *type_name = get_anon_type_name(irb->codegen, irb->exec, "error", parent_scope, node, &bare_name);
10024		ZigType *err_set_type = new_type_table_entry(ZigTypeIdErrorSet);
10025		buf_init_from_buf(&err_set_type->name, type_name);
10026		err_set_type->data.error_set.err_count = err_count;
10027		err_set_type->size_in_bits = irb->codegen->builtin_types.entry_global_error_set->size_in_bits;
10028		err_set_type->abi_align = irb->codegen->builtin_types.entry_global_error_set->abi_align;
10029		err_set_type->abi_size = irb->codegen->builtin_types.entry_global_error_set->abi_size;
10030		err_set_type->data.error_set.errors = heap::c_allocator.allocate<ErrorTableEntry *>(err_count);
10031
10032		size_t errors_count = irb->codegen->errors_by_index.length + err_count;
10033		ErrorTableEntry *errors = heap::c_allocator.allocate<ErrorTableEntry >(errors_count);
10034
10035		for (uint32_t i = 0; i < err_count; i += 1) {
10036		AstNode *field_node = node->data.err_set_decl.decls.at(i);
10037		AstNode *symbol_node = ast_field_to_symbol_node(field_node);
10038		Buf *err_name = symbol_node->data.symbol_expr.symbol;
10039		ErrorTableEntry *err = heap::c_allocator.create<ErrorTableEntry>();
10040		err->decl_node = field_node;
10041		buf_init_from_buf(&err->name, err_name);
10042
10043		auto existing_entry = irb->codegen->error_table.put_unique(err_name, err);
10044		if (existing_entry) {
10045		err->value = existing_entry->value->value;
10046		} else {
10047		size_t error_value_count = irb->codegen->errors_by_index.length;
10048		assert((uint32_t)error_value_count < (((uint32_t)1) << (uint32_t)irb->codegen->err_tag_type->data.integral.bit_count));
10049		err->value = error_value_count;
10050		irb->codegen->errors_by_index.append(err);
10051		}
10052		err_set_type->data.error_set.errors[i] = err;
10053
10054		ErrorTableEntry *prev_err = errors[err->value];
10055		if (prev_err != nullptr) {
10056		ErrorMsg *msg = add_node_error(irb->codegen, ast_field_to_symbol_node(err->decl_node),
10057		buf_sprintf("duplicate error: '%s'", buf_ptr(&err->name)));
10058		add_error_note(irb->codegen, msg, ast_field_to_symbol_node(prev_err->decl_node),
10059		buf_sprintf("other error here"));
10060		return irb->codegen->invalid_inst_src;
10061		}
10062		errors[err->value] = err;
10063		}
10064		heap::c_allocator.deallocate(errors, errors_count);
10065		return ir_build_const_type(irb, parent_scope, node, err_set_type);
10066		}
10067
10068		static IrInstSrc ir_gen_fn_proto(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
10069		assert(node->type == NodeTypeFnProto);
10070
10071		size_t param_count = node->data.fn_proto.params.length;
10072		IrInstSrc *param_types = heap::c_allocator.allocate<IrInstSrc>(param_count);
10073
10074		bool is_var_args = false;
10075		for (size_t i = 0; i < param_count; i += 1) {
10076		AstNode *param_node = node->data.fn_proto.params.at(i);
10077		if (param_node->data.param_decl.is_var_args) {
10078		is_var_args = true;
10079		break;
10080		}
10081		if (param_node->data.param_decl.anytype_token == nullptr) {
10082		AstNode *type_node = param_node->data.param_decl.type;
10083		IrInstSrc *type_value = ir_gen_node(irb, type_node, parent_scope);
10084		if (type_value == irb->codegen->invalid_inst_src)
10085		return irb->codegen->invalid_inst_src;
10086		param_types[i] = type_value;
10087		} else {
10088		param_types[i] = nullptr;
10089		}
10090		}
10091
10092		IrInstSrc *align_value = nullptr;
10093		if (node->data.fn_proto.align_expr != nullptr) {
10094		align_value = ir_gen_node(irb, node->data.fn_proto.align_expr, parent_scope);
10095		if (align_value == irb->codegen->invalid_inst_src)
10096		return irb->codegen->invalid_inst_src;
10097		}
10098
10099		IrInstSrc *callconv_value = nullptr;
10100		if (node->data.fn_proto.callconv_expr != nullptr) {
10101		callconv_value = ir_gen_node(irb, node->data.fn_proto.callconv_expr, parent_scope);
10102		if (callconv_value == irb->codegen->invalid_inst_src)
10103		return irb->codegen->invalid_inst_src;
10104		}
10105
10106		IrInstSrc *return_type;
10107		if (node->data.fn_proto.return_type == nullptr) {
10108		return_type = ir_build_const_type(irb, parent_scope, node, irb->codegen->builtin_types.entry_void);
10109		} else {
10110		return_type = ir_gen_node(irb, node->data.fn_proto.return_type, parent_scope);
10111		if (return_type == irb->codegen->invalid_inst_src)
10112		return irb->codegen->invalid_inst_src;
10113		}
10114
10115		return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, callconv_value, return_type, is_var_args);
10116		}
10117
10118		static IrInstSrc ir_gen_resume(IrBuilderSrc irb, Scope scope, AstNode node) {
10119		assert(node->type == NodeTypeResume);
10120
10121		IrInstSrc *target_inst = ir_gen_node_extra(irb, node->data.resume_expr.expr, scope, LValPtr, nullptr);
10122		if (target_inst == irb->codegen->invalid_inst_src)
10123		return irb->codegen->invalid_inst_src;
10124
10125		return ir_build_resume_src(irb, scope, node, target_inst);
10126		}
10127
10128		static IrInstSrc ir_gen_await_expr(IrBuilderSrc irb, Scope scope, AstNode node, LVal lval,
10129		ResultLoc *result_loc)
10130		{
10131		assert(node->type == NodeTypeAwaitExpr);
10132
10133		bool is_nosuspend = get_scope_nosuspend(scope) != nullptr;
10134
10135		AstNode *expr_node = node->data.await_expr.expr;
10136		if (expr_node->type == NodeTypeFnCallExpr && expr_node->data.fn_call_expr.modifier == CallModifierBuiltin) {
10137		AstNode *fn_ref_expr = expr_node->data.fn_call_expr.fn_ref_expr;
10138		Buf *name = fn_ref_expr->data.symbol_expr.symbol;
10139		auto entry = irb->codegen->builtin_fn_table.maybe_get(name);
10140		if (entry != nullptr) {
10141		BuiltinFnEntry *builtin_fn = entry->value;
10142		if (builtin_fn->id == BuiltinFnIdAsyncCall) {
10143		return ir_gen_async_call(irb, scope, node, expr_node, lval, result_loc);
10144		}
10145		}
10146		}
10147
10148		ZigFn *fn_entry = exec_fn_entry(irb->exec);
10149		if (!fn_entry) {
10150		add_node_error(irb->codegen, node, buf_sprintf("await outside function definition"));
10151		return irb->codegen->invalid_inst_src;
10152		}
10153		ScopeSuspend *existing_suspend_scope = get_scope_suspend(scope);
10154		if (existing_suspend_scope) {
10155		if (!existing_suspend_scope->reported_err) {
10156		ErrorMsg *msg = add_node_error(irb->codegen, node, buf_sprintf("cannot await inside suspend block"));
10157		add_error_note(irb->codegen, msg, existing_suspend_scope->base.source_node, buf_sprintf("suspend block here"));
10158		existing_suspend_scope->reported_err = true;
10159		}
10160		return irb->codegen->invalid_inst_src;
10161		}
10162
10163		IrInstSrc *target_inst = ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr);
10164		if (target_inst == irb->codegen->invalid_inst_src)
10165		return irb->codegen->invalid_inst_src;
10166
10167		IrInstSrc *await_inst = ir_build_await_src(irb, scope, node, target_inst, result_loc, is_nosuspend);
10168		return ir_lval_wrap(irb, scope, await_inst, lval, result_loc);
10169		}
10170
10171		static IrInstSrc ir_gen_suspend(IrBuilderSrc irb, Scope parent_scope, AstNode node) {
10172		assert(node->type == NodeTypeSuspend);
10173
10174		ZigFn *fn_entry = exec_fn_entry(irb->exec);
10175		if (!fn_entry) {
10176		add_node_error(irb->codegen, node, buf_sprintf("suspend outside function definition"));
10177		return irb->codegen->invalid_inst_src;
10178		}
10179		if (get_scope_nosuspend(parent_scope) != nullptr) {
10180		add_node_error(irb->codegen, node, buf_sprintf("suspend in nosuspend scope"));
10181		return irb->codegen->invalid_inst_src;
10182		}
10183
10184		ScopeSuspend *existing_suspend_scope = get_scope_suspend(parent_scope);
10185		if (existing_suspend_scope) {
10186		if (!existing_suspend_scope->reported_err) {
10187		ErrorMsg *msg = add_node_error(irb->codegen, node, buf_sprintf("cannot suspend inside suspend block"));
10188		add_error_note(irb->codegen, msg, existing_suspend_scope->base.source_node, buf_sprintf("other suspend block here"));
10189		existing_suspend_scope->reported_err = true;
10190		}
10191		return irb->codegen->invalid_inst_src;
10192		}
10193
10194		IrInstSrcSuspendBegin *begin = ir_build_suspend_begin_src(irb, parent_scope, node);
10195		ScopeSuspend *suspend_scope = create_suspend_scope(irb->codegen, node, parent_scope);
10196		Scope *child_scope = &suspend_scope->base;
10197		IrInstSrc *susp_res = ir_gen_node(irb, node->data.suspend.block, child_scope);
10198		if (susp_res == irb->codegen->invalid_inst_src)
10199		return irb->codegen->invalid_inst_src;
10200		ir_mark_gen(ir_build_check_statement_is_void(irb, child_scope, node->data.suspend.block, susp_res));
10201
10202		return ir_mark_gen(ir_build_suspend_finish_src(irb, parent_scope, node, begin));
10203		}
10204
10205		static IrInstSrc ir_gen_node_raw(IrBuilderSrc irb, AstNode node, Scope scope,
10206		LVal lval, ResultLoc *result_loc)
10207		{
10208		assert(scope);
10209		switch (node->type) {
10210		case NodeTypeStructValueField:
10211		case NodeTypeParamDecl:
10212		case NodeTypeUsingNamespace:
10213		case NodeTypeSwitchProng:
10214		case NodeTypeSwitchRange:
10215		case NodeTypeStructField:
10216		case NodeTypeErrorSetField:
10217		case NodeTypeFnDef:
10218		case NodeTypeTestDecl:
10219		zig_unreachable();
10220		case NodeTypeBlock:
10221		return ir_gen_block(irb, scope, node, lval, result_loc);
10222		case NodeTypeGroupedExpr:
10223		return ir_gen_node_raw(irb, node->data.grouped_expr, scope, lval, result_loc);
10224		case NodeTypeBinOpExpr:
10225		return ir_gen_bin_op(irb, scope, node, lval, result_loc);
10226		case NodeTypeIntLiteral:
10227		return ir_lval_wrap(irb, scope, ir_gen_int_lit(irb, scope, node), lval, result_loc);
10228		case NodeTypeFloatLiteral:
10229		return ir_lval_wrap(irb, scope, ir_gen_float_lit(irb, scope, node), lval, result_loc);
10230		case NodeTypeCharLiteral:
10231		return ir_lval_wrap(irb, scope, ir_gen_char_lit(irb, scope, node), lval, result_loc);
10232		case NodeTypeSymbol:
10233		return ir_gen_symbol(irb, scope, node, lval, result_loc);
10234		case NodeTypeFnCallExpr:
10235		return ir_gen_fn_call(irb, scope, node, lval, result_loc);
10236		case NodeTypeIfBoolExpr:
10237		return ir_gen_if_bool_expr(irb, scope, node, lval, result_loc);
10238		case NodeTypePrefixOpExpr:
10239		return ir_gen_prefix_op_expr(irb, scope, node, lval, result_loc);
10240		case NodeTypeContainerInitExpr:
10241		return ir_gen_container_init_expr(irb, scope, node, lval, result_loc);
10242		case NodeTypeVariableDeclaration:
10243		return ir_gen_var_decl(irb, scope, node);
10244		case NodeTypeWhileExpr:
10245		return ir_gen_while_expr(irb, scope, node, lval, result_loc);
10246		case NodeTypeForExpr:
10247		return ir_gen_for_expr(irb, scope, node, lval, result_loc);
10248		case NodeTypeArrayAccessExpr:
10249		return ir_gen_array_access(irb, scope, node, lval, result_loc);
10250		case NodeTypeReturnExpr:
10251		return ir_gen_return(irb, scope, node, lval, result_loc);
10252		case NodeTypeFieldAccessExpr:
10253		{
10254		IrInstSrc *ptr_instruction = ir_gen_field_access(irb, scope, node);
10255		if (ptr_instruction == irb->codegen->invalid_inst_src)
10256		return ptr_instruction;
10257		if (lval == LValPtr \|\| lval == LValAssign)
10258		return ptr_instruction;
10259
10260		IrInstSrc *load_ptr = ir_build_load_ptr(irb, scope, node, ptr_instruction);
10261		return ir_expr_wrap(irb, scope, load_ptr, result_loc);
10262		}
10263		case NodeTypePtrDeref: {
10264		AstNode *expr_node = node->data.ptr_deref_expr.target;
10265
10266		LVal child_lval = lval;
10267		if (child_lval == LValAssign)
10268		child_lval = LValPtr;
10269
10270		IrInstSrc *value = ir_gen_node_extra(irb, expr_node, scope, child_lval, nullptr);
10271		if (value == irb->codegen->invalid_inst_src)
10272		return value;
10273
10274		// We essentially just converted any lvalue from &(x.) to (&x).;
10275		// this inhibits checking that x is a pointer later, so we directly
10276		// record whether the pointer check is needed
10277		IrInstSrc *un_op = ir_build_un_op_lval(irb, scope, node, IrUnOpDereference, value, lval, result_loc);
10278		return ir_expr_wrap(irb, scope, un_op, result_loc);
10279		}
10280		case NodeTypeUnwrapOptional: {
10281		AstNode *expr_node = node->data.unwrap_optional.expr;
10282
10283		IrInstSrc *maybe_ptr = ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr);
10284		if (maybe_ptr == irb->codegen->invalid_inst_src)
10285		return irb->codegen->invalid_inst_src;
10286
10287		IrInstSrc *unwrapped_ptr = ir_build_optional_unwrap_ptr(irb, scope, node, maybe_ptr, true );
10288		if (lval == LValPtr \|\| lval == LValAssign)
10289		return unwrapped_ptr;
10290
10291		IrInstSrc *load_ptr = ir_build_load_ptr(irb, scope, node, unwrapped_ptr);
10292		return ir_expr_wrap(irb, scope, load_ptr, result_loc);
10293		}
10294		case NodeTypeBoolLiteral:
10295		return ir_lval_wrap(irb, scope, ir_gen_bool_literal(irb, scope, node), lval, result_loc);
10296		case NodeTypeArrayType:
10297		return ir_lval_wrap(irb, scope, ir_gen_array_type(irb, scope, node), lval, result_loc);
10298		case NodeTypePointerType:
10299		return ir_lval_wrap(irb, scope, ir_gen_pointer_type(irb, scope, node), lval, result_loc);
10300		case NodeTypeAnyFrameType:
10301		return ir_lval_wrap(irb, scope, ir_gen_anyframe_type(irb, scope, node), lval, result_loc);
10302		case NodeTypeStringLiteral:
10303		return ir_lval_wrap(irb, scope, ir_gen_string_literal(irb, scope, node), lval, result_loc);
10304		case NodeTypeUndefinedLiteral:
10305		return ir_lval_wrap(irb, scope, ir_gen_undefined_literal(irb, scope, node), lval, result_loc);
10306		case NodeTypeAsmExpr:
10307		return ir_lval_wrap(irb, scope, ir_gen_asm_expr(irb, scope, node), lval, result_loc);
10308		case NodeTypeNullLiteral:
10309		return ir_lval_wrap(irb, scope, ir_gen_null_literal(irb, scope, node), lval, result_loc);
10310		case NodeTypeIfErrorExpr:
10311		return ir_gen_if_err_expr(irb, scope, node, lval, result_loc);
10312		case NodeTypeIfOptional:
10313		return ir_gen_if_optional_expr(irb, scope, node, lval, result_loc);
10314		case NodeTypeSwitchExpr:
10315		return ir_gen_switch_expr(irb, scope, node, lval, result_loc);
10316		case NodeTypeCompTime:
10317		return ir_expr_wrap(irb, scope, ir_gen_comptime(irb, scope, node, lval), result_loc);
10318		case NodeTypeNoSuspend:
10319		return ir_expr_wrap(irb, scope, ir_gen_nosuspend(irb, scope, node, lval), result_loc);
10320		case NodeTypeErrorType:
10321		return ir_lval_wrap(irb, scope, ir_gen_error_type(irb, scope, node), lval, result_loc);
10322		case NodeTypeBreak:
10323		return ir_lval_wrap(irb, scope, ir_gen_break(irb, scope, node), lval, result_loc);
10324		case NodeTypeContinue:
10325		return ir_lval_wrap(irb, scope, ir_gen_continue(irb, scope, node), lval, result_loc);
10326		case NodeTypeUnreachable:
10327		return ir_build_unreachable(irb, scope, node);
10328		case NodeTypeDefer:
10329		return ir_lval_wrap(irb, scope, ir_gen_defer(irb, scope, node), lval, result_loc);
10330		case NodeTypeSliceExpr:
10331		return ir_gen_slice(irb, scope, node, lval, result_loc);
10332		case NodeTypeCatchExpr:
10333		return ir_gen_catch(irb, scope, node, lval, result_loc);
10334		case NodeTypeContainerDecl:
10335		return ir_lval_wrap(irb, scope, ir_gen_container_decl(irb, scope, node), lval, result_loc);
10336		case NodeTypeFnProto:
10337		return ir_lval_wrap(irb, scope, ir_gen_fn_proto(irb, scope, node), lval, result_loc);
10338		case NodeTypeErrorSetDecl:
10339		return ir_lval_wrap(irb, scope, ir_gen_err_set_decl(irb, scope, node), lval, result_loc);
10340		case NodeTypeResume:
10341		return ir_lval_wrap(irb, scope, ir_gen_resume(irb, scope, node), lval, result_loc);
10342		case NodeTypeAwaitExpr:
10343		return ir_gen_await_expr(irb, scope, node, lval, result_loc);
10344		case NodeTypeSuspend:
10345		return ir_lval_wrap(irb, scope, ir_gen_suspend(irb, scope, node), lval, result_loc);
10346		case NodeTypeEnumLiteral:
10347		return ir_lval_wrap(irb, scope, ir_gen_enum_literal(irb, scope, node), lval, result_loc);
10348		case NodeTypeInferredArrayType:
10349		add_node_error(irb->codegen, node,
10350		buf_sprintf("inferred array size invalid here"));
10351		return irb->codegen->invalid_inst_src;
10352		case NodeTypeAnyTypeField:
10353		return ir_lval_wrap(irb, scope,
10354		ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_anytype), lval, result_loc);
10355		}
10356		zig_unreachable();
10357		}
10358
10359		static ResultLoc *no_result_loc(void) {
10360		ResultLocNone *result_loc_none = heap::c_allocator.create<ResultLocNone>();
10361		result_loc_none->base.id = ResultLocIdNone;
10362		return &result_loc_none->base;
10363		}
10364
10365		static IrInstSrc ir_gen_node_extra(IrBuilderSrc irb, AstNode node, Scope scope, LVal lval,
10366		ResultLoc *result_loc)
10367		{
10368		if (lval == LValAssign) {
10369		switch (node->type) {
10370		case NodeTypeStructValueField:
10371		case NodeTypeParamDecl:
10372		case NodeTypeUsingNamespace:
10373		case NodeTypeSwitchProng:
10374		case NodeTypeSwitchRange:
10375		case NodeTypeStructField:
10376		case NodeTypeErrorSetField:
10377		case NodeTypeFnDef:
10378		case NodeTypeTestDecl:
10379		zig_unreachable();
10380
10381		// cannot be assigned to
10382		case NodeTypeBlock:
10383		case NodeTypeGroupedExpr:
10384		case NodeTypeBinOpExpr:
10385		case NodeTypeIntLiteral:
10386		case NodeTypeFloatLiteral:
10387		case NodeTypeCharLiteral:
10388		case NodeTypeIfBoolExpr:
10389		case NodeTypeContainerInitExpr:
10390		case NodeTypeVariableDeclaration:
10391		case NodeTypeWhileExpr:
10392		case NodeTypeForExpr:
10393		case NodeTypeReturnExpr:
10394		case NodeTypeBoolLiteral:
10395		case NodeTypeArrayType:
10396		case NodeTypePointerType:
10397		case NodeTypeAnyFrameType:
10398		case NodeTypeStringLiteral:
10399		case NodeTypeUndefinedLiteral:
10400		case NodeTypeAsmExpr:
10401		case NodeTypeNullLiteral:
10402		case NodeTypeIfErrorExpr:
10403		case NodeTypeIfOptional:
10404		case NodeTypeSwitchExpr:
10405		case NodeTypeCompTime:
10406		case NodeTypeNoSuspend:
10407		case NodeTypeErrorType:
10408		case NodeTypeBreak:
10409		case NodeTypeContinue:
10410		case NodeTypeUnreachable:
10411		case NodeTypeDefer:
10412		case NodeTypeSliceExpr:
10413		case NodeTypeCatchExpr:
10414		case NodeTypeContainerDecl:
10415		case NodeTypeFnProto:
10416		case NodeTypeErrorSetDecl:
10417		case NodeTypeResume:
10418		case NodeTypeAwaitExpr:
10419		case NodeTypeSuspend:
10420		case NodeTypeEnumLiteral:
10421		case NodeTypeInferredArrayType:
10422		case NodeTypeAnyTypeField:
10423		case NodeTypePrefixOpExpr:
10424		add_node_error(irb->codegen, node,
10425		buf_sprintf("invalid left-hand side to assignment"));
10426		return irb->codegen->invalid_inst_src;
10427
10428		// @field can be assigned to
10429		case NodeTypeFnCallExpr:
10430		if (node->data.fn_call_expr.modifier == CallModifierBuiltin) {
10431		AstNode *fn_ref_expr = node->data.fn_call_expr.fn_ref_expr;
10432		Buf *name = fn_ref_expr->data.symbol_expr.symbol;
10433		auto entry = irb->codegen->builtin_fn_table.maybe_get(name);
10434
10435		if (!entry) {
10436		add_node_error(irb->codegen, node,
10437		buf_sprintf("invalid builtin function: '%s'", buf_ptr(name)));
10438		return irb->codegen->invalid_inst_src;
10439		}
10440
10441		if (entry->value->id == BuiltinFnIdField) {
10442		break;
10443		}
10444		}
10445		add_node_error(irb->codegen, node,
10446		buf_sprintf("invalid left-hand side to assignment"));
10447		return irb->codegen->invalid_inst_src;
10448
10449
10450		// can be assigned to
10451		case NodeTypeUnwrapOptional:
10452		case NodeTypePtrDeref:
10453		case NodeTypeFieldAccessExpr:
10454		case NodeTypeArrayAccessExpr:
10455		case NodeTypeSymbol:
10456		break;
10457		}
10458		}
10459		if (result_loc == nullptr) {
10460		// Create a result location indicating there is none - but if one gets created
10461		// it will be properly distributed.
10462		result_loc = no_result_loc();
10463		ir_build_reset_result(irb, scope, node, result_loc);
10464		}
10465		Scope *child_scope;
10466		if (irb->exec->is_inline \|\|
10467		(irb->exec->fn_entry != nullptr && irb->exec->fn_entry->child_scope == scope))
10468		{
10469		child_scope = scope;
10470		} else {
10471		child_scope = &create_expr_scope(irb->codegen, node, scope)->base;
10472		}
10473		IrInstSrc *result = ir_gen_node_raw(irb, node, child_scope, lval, result_loc);
10474		if (result == irb->codegen->invalid_inst_src) {
10475		if (irb->exec->first_err_trace_msg == nullptr) {
10476		irb->exec->first_err_trace_msg = irb->codegen->trace_err;
10477		}
10478		}
10479		return result;
10480		}
10481
10482		static IrInstSrc ir_gen_node(IrBuilderSrc irb, AstNode node, Scope scope) {
10483		return ir_gen_node_extra(irb, node, scope, LValNone, nullptr);
10484		}
10485
10486		static void invalidate_exec(IrExecutableSrc exec, ErrorMsg msg) {
10487		if (exec->first_err_trace_msg != nullptr)
10488		return;
10489
10490		exec->first_err_trace_msg = msg;
10491
10492		for (size_t i = 0; i < exec->tld_list.length; i += 1) {
10493		exec->tld_list.items[i]->resolution = TldResolutionInvalid;
10494		}
10495		}
10496
10497	2514	static void invalidate_exec_gen(IrExecutableGen exec, ErrorMsg msg) {
10498	2515	if (exec->first_err_trace_msg != nullptr)
10499	2516	return;
...	...	@@ -10509,78 +2526,6 @@ static void invalidate_exec_gen(IrExecutableGen exec, ErrorMsg msg) {
10509	2526	}
10510	2527
10511	2528
10512		bool ir_gen(CodeGen codegen, AstNode node, Scope scope, IrExecutableSrc ir_executable) {
10513		assert(node->owner);
10514
10515		IrBuilderSrc ir_builder = {0};
10516		IrBuilderSrc *irb = &ir_builder;
10517
10518		irb->codegen = codegen;
10519		irb->exec = ir_executable;
10520		irb->main_block_node = node;
10521
10522		IrBasicBlockSrc *entry_block = ir_create_basic_block(irb, scope, "Entry");
10523		ir_set_cursor_at_end_and_append_block(irb, entry_block);
10524		// Entry block gets a reference because we enter it to begin.
10525		ir_ref_bb(irb->current_basic_block);
10526
10527		IrInstSrc *result = ir_gen_node_extra(irb, node, scope, LValNone, nullptr);
10528
10529		if (result == irb->codegen->invalid_inst_src)
10530		return false;
10531
10532		if (irb->exec->first_err_trace_msg != nullptr) {
10533		codegen->trace_err = irb->exec->first_err_trace_msg;
10534		return false;
10535		}
10536
10537		if (!instr_is_unreachable(result)) {
10538		ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, result->base.source_node, result, nullptr));
10539		// no need for save_err_ret_addr because this cannot return error
10540		ResultLocReturn *result_loc_ret = heap::c_allocator.create<ResultLocReturn>();
10541		result_loc_ret->base.id = ResultLocIdReturn;
10542		ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
10543		ir_mark_gen(ir_build_end_expr(irb, scope, node, result, &result_loc_ret->base));
10544		ir_mark_gen(ir_build_return_src(irb, scope, result->base.source_node, result));
10545		}
10546
10547		return true;
10548		}
10549
10550		bool ir_gen_fn(CodeGen codegen, ZigFn fn_entry) {
10551		assert(fn_entry);
10552
10553		IrExecutableSrc *ir_executable = fn_entry->ir_executable;
10554		AstNode *body_node = fn_entry->body_node;
10555
10556		assert(fn_entry->child_scope);
10557
10558		return ir_gen(codegen, body_node, fn_entry->child_scope, ir_executable);
10559		}
10560
10561		static void ir_add_call_stack_errors_gen(CodeGen codegen, IrExecutableGen exec, ErrorMsg *err_msg, int limit) {
10562		if (!exec \|\| !exec->source_node \|\| limit < 0) return;
10563		add_error_note(codegen, err_msg, exec->source_node, buf_sprintf("called from here"));
10564
10565		ir_add_call_stack_errors_gen(codegen, exec->parent_exec, err_msg, limit - 1);
10566		}
10567
10568		static void ir_add_call_stack_errors(CodeGen codegen, IrExecutableSrc exec, ErrorMsg *err_msg, int limit) {
10569		if (!exec \|\| !exec->source_node \|\| limit < 0) return;
10570		add_error_note(codegen, err_msg, exec->source_node, buf_sprintf("called from here"));
10571
10572		ir_add_call_stack_errors_gen(codegen, exec->parent_exec, err_msg, limit - 1);
10573		}
10574
10575		static ErrorMsg exec_add_error_node(CodeGen codegen, IrExecutableSrc exec, AstNode source_node, Buf *msg) {
10576		ErrorMsg *err_msg = add_node_error(codegen, source_node, msg);
10577		invalidate_exec(exec, err_msg);
10578		if (exec->parent_exec) {
10579		ir_add_call_stack_errors(codegen, exec, err_msg, 10);
10580		}
10581		return err_msg;
10582		}
10583
10584	2529	static ErrorMsg exec_add_error_node_gen(CodeGen codegen, IrExecutableGen exec, AstNode source_node, Buf *msg) {
10585	2530	ErrorMsg *err_msg = add_node_error(codegen, source_node, msg);
10586	2531	invalidate_exec_gen(exec, err_msg);
...	...	@@ -10605,11 +2550,6 @@ static ErrorMsg ir_add_error(IrAnalyze ira, IrInst source_instruction, Buf m
10605	2550	return ir_add_error_node(ira, source_instruction->source_node, msg);
10606	2551	}
10607	2552
10608		static void ir_assert_impl(bool ok, IrInst source_instruction, char const file, unsigned int line) {
10609		if (ok) return;
10610		src_assert_impl(ok, source_instruction->source_node, file, line);
10611		}
10612
10613	2553	static void ir_assert_gen_impl(bool ok, IrInstGen source_instruction, char const file, unsigned int line) {
10614	2554	if (ok) return;
10615	2555	src_assert_impl(ok, source_instruction->base.source_node, file, line);
...	...	@@ -13665,8 +5605,6 @@ Error ir_eval_const_value(CodeGen codegen, Scope scope, AstNode *node,
13665	5605	}
13666	5606
13667	5607	if (codegen->verbose_ir) {
13668		fprintf(stderr, "\nSource: ");
13669		ast_render(stderr, node, 4);
13670	5608	fprintf(stderr, "\n{ // (IR)\n");
13671	5609	ir_print_src(codegen, stderr, ir_executable, 2);
13672	5610	fprintf(stderr, "}\n");
...	...	@@ -19482,7 +11420,7 @@ static IrInstGen ir_analyze_instruction_extern(IrAnalyze ira, IrInstSrcExtern
19482	11420	}
19483	11421
19484	11422	static bool exec_has_err_ret_trace(CodeGen g, IrExecutableSrc exec) {
19485		ZigFn *fn_entry = exec_fn_entry(exec);
	11423	ZigFn *fn_entry = exec->fn_entry;
19486	11424	return fn_entry != nullptr && fn_entry->calls_or_awaits_errorable_fn && g->have_err_ret_tracing;
19487	11425	}
19488	11426
...	...	@@ -20319,7 +12257,7 @@ static bool ir_analyze_fn_call_inline_arg(IrAnalyze ira, AstNode fn_proto_node
20319	12257	assert(param_decl_node->type == NodeTypeParamDecl);
20320	12258
20321	12259	IrInstGen *casted_arg;
20322		if (param_decl_node->data.param_decl.anytype_token == nullptr) {
	12260	if (param_decl_node->data.param_decl.anytype_token == 0) {
20323	12261	AstNode *param_type_node = param_decl_node->data.param_decl.type;
20324	12262	ZigType param_type = ir_analyze_type_expr(ira, exec_scope, param_type_node);
20325	12263	if (type_is_invalid(param_type))
...	...	@@ -20362,7 +12300,7 @@ static bool ir_analyze_fn_call_generic_arg(IrAnalyze ira, AstNode fn_proto_nod
20362	12300	casted_arg = arg;
20363	12301	param_info_type = arg->value->type;
20364	12302	} else {
20365		if (param_decl_node->data.param_decl.anytype_token == nullptr) {
	12303	if (param_decl_node->data.param_decl.anytype_token == 0) {
20366	12304	AstNode *param_type_node = param_decl_node->data.param_decl.type;
20367	12305	ZigType param_type = ir_analyze_type_expr(ira, child_scope, param_type_node);
20368	12306	if (type_is_invalid(param_type))
...	...	@@ -23593,6 +15531,25 @@ static IrInstGen ir_analyze_instruction_slice_type(IrAnalyze ira, IrInstSrcSli
23593	15531	return result;
23594	15532	}
23595	15533
	15534	static size_t find_asm_index(CodeGen g, AstNode node, AsmToken tok, Buf src_template) {
	15535	const char *ptr = buf_ptr(src_template) + tok->start + 2;
	15536	size_t len = tok->end - tok->start - 2;
	15537	size_t result = 0;
	15538	for (size_t i = 0; i < node->data.asm_expr.output_list.length; i += 1, result += 1) {
	15539	AsmOutput *asm_output = node->data.asm_expr.output_list.at(i);
	15540	if (buf_eql_mem(asm_output->asm_symbolic_name, ptr, len)) {
	15541	return result;
	15542	}
	15543	}
	15544	for (size_t i = 0; i < node->data.asm_expr.input_list.length; i += 1, result += 1) {
	15545	AsmInput *asm_input = node->data.asm_expr.input_list.at(i);
	15546	if (buf_eql_mem(asm_input->asm_symbolic_name, ptr, len)) {
	15547	return result;
	15548	}
	15549	}
	15550	return SIZE_MAX;
	15551	}
	15552
23596	15553	static IrInstGen ir_analyze_instruction_asm(IrAnalyze ira, IrInstSrcAsm *asm_instruction) {
23597	15554	Error err;
23598	15555
...	...	@@ -27066,8 +19023,10 @@ static IrInstGen ir_analyze_instruction_c_import(IrAnalyze ira, IrInstSrcCImpo
27066	19023	return ira->codegen->invalid_inst_gen;
27067	19024
27068	19025	ZigPackage *cur_scope_pkg = scope_package(instruction->base.base.scope);
27069		Buf *namespace_name = buf_sprintf("%s.cimport:%" ZIG_PRI_usize ":%" ZIG_PRI_usize,
27070		buf_ptr(&cur_scope_pkg->pkg_path), node->line + 1, node->column + 1);
	19026	RootStruct *root_struct = node->owner->data.structure.root_struct;
	19027	TokenLoc tok_loc = root_struct->token_locs[node->main_token];
	19028	Buf *namespace_name = buf_sprintf("%s.cimport:%" PRIu32 ":%" PRIu32,
	19029	buf_ptr(&cur_scope_pkg->pkg_path), tok_loc.line + 1, tok_loc.column + 1);
27071	19030
27072	19031	ZigPackage *cimport_pkg = new_anonymous_package();
27073	19032	cimport_pkg->package_table.put(buf_create_from_str("builtin"), ira->codegen->compile_var_package);
...	...	@@ -27095,12 +19054,14 @@ static IrInstGen ir_analyze_instruction_c_import(IrAnalyze ira, IrInstSrcCImpo
27095	19054	}
27096	19055	for (size_t i = 0; i < errors_len; i += 1) {
27097	19056	Stage2ErrorMsg *clang_err = &errors_ptr[i];
27098		// Clang can emit "too many errors, stopping now", in which case `source` and `filename_ptr` are null
	19057	// Clang can emit "too many errors, stopping now", in which case
	19058	// `source` and `filename_ptr` are null
27099	19059	if (clang_err->source && clang_err->filename_ptr) {
27100	19060	ErrorMsg *err_msg = err_msg_create_with_offset(
27101	19061	clang_err->filename_ptr ?
27102		buf_create_from_mem(clang_err->filename_ptr, clang_err->filename_len) : buf_alloc(),
27103		clang_err->line, clang_err->column, clang_err->offset, clang_err->source,
	19062	buf_create_from_mem(clang_err->filename_ptr, clang_err->filename_len) :
	19063	buf_alloc(),
	19064	clang_err->offset, clang_err->source,
27104	19065	buf_create_from_mem(clang_err->msg_ptr, clang_err->msg_len));
27105	19066	err_msg_add_note(parent_err_msg, err_msg);
27106	19067	}
...	...	@@ -27225,7 +19186,7 @@ static IrInstGen ir_analyze_instruction_embed_file(IrAnalyze ira, IrInstSrcEmb
27225	19186	}
27226	19187
27227	19188	IrInstGen *result = ir_const(ira, &instruction->base.base, nullptr);
27228		init_const_str_lit(ira->codegen, result->value, file_contents);
	19189	init_const_str_lit(ira->codegen, result->value, file_contents, true);
27229	19190	return result;
27230	19191	}
27231	19192
...	...	@@ -31851,6 +23812,22 @@ static IrInstGen ir_analyze_instruction_has_decl(IrAnalyze ira, IrInstSrcHasDe
31851	23812	return ir_const_bool(ira, &instruction->base.base, true);
31852	23813	}
31853	23814
	23815	static void populate_invalid_variable_in_scope(CodeGen g, Scope scope, AstNode node, Buf var_name) {
	23816	ScopeDecls *scope_decls = nullptr;
	23817	while (scope != nullptr) {
	23818	if (scope->id == ScopeIdDecls) {
	23819	scope_decls = reinterpret_cast<ScopeDecls *>(scope);
	23820	}
	23821	scope = scope->parent;
	23822	}
	23823	TldVar *tld_var = heap::c_allocator.create<TldVar>();
	23824	init_tld(&tld_var->base, TldIdVar, var_name, VisibModPub, node, &scope_decls->base);
	23825	tld_var->base.resolution = TldResolutionInvalid;
	23826	tld_var->var = add_variable(g, node, &scope_decls->base, var_name, false,
	23827	g->invalid_inst_gen->value, &tld_var->base, g->builtin_types.entry_invalid);
	23828	scope_decls->decl_table.put(var_name, &tld_var->base);
	23829	}
	23830
31854	23831	static IrInstGen ir_analyze_instruction_undeclared_ident(IrAnalyze ira, IrInstSrcUndeclaredIdent *instruction) {
31855	23832	// put a variable of same name with invalid type in global scope
31856	23833	// so that future references to this same name will find a variable with an invalid type
...	...	@@ -32169,7 +24146,8 @@ static IrInstGen ir_analyze_instruction_src(IrAnalyze ira, IrInstSrcSrc *instr
32169	24146	fields[0]->special = ConstValSpecialStatic;
32170	24147
32171	24148	ZigType *import = instruction->base.base.source_node->owner;
32172		Buf *path = import->data.structure.root_struct->path;
	24149	RootStruct *root_struct = import->data.structure.root_struct;
	24150	Buf *path = root_struct->path;
32173	24151	ZigValue *file_name = create_const_str_lit(ira->codegen, path)->data.x_ptr.data.ref.pointee;
32174	24152	init_const_slice(ira->codegen, fields[0], file_name, 0, buf_len(path), true);
32175	24153	fields[0]->type = u8_slice;
...	...	@@ -32182,17 +24160,20 @@ static IrInstGen ir_analyze_instruction_src(IrAnalyze ira, IrInstSrcSrc *instr
32182	24160	init_const_slice(ira->codegen, fields[1], fn_name, 0, buf_len(&fn_entry->symbol_name), true);
32183	24161	fields[1]->type = u8_slice;
32184	24162
	24163
	24164	TokenLoc tok_loc = root_struct->token_locs[instruction->base.base.source_node->main_token];
	24165
32185	24166	// line: u32
32186	24167	ensure_field_index(source_location_type, "line", 2);
32187	24168	fields[2]->special = ConstValSpecialStatic;
32188	24169	fields[2]->type = ira->codegen->builtin_types.entry_u32;
32189		bigint_init_unsigned(&fields[2]->data.x_bigint, instruction->base.base.source_node->line + 1);
	24170	bigint_init_unsigned(&fields[2]->data.x_bigint, tok_loc.line + 1);
32190	24171
32191	24172	// column: u32
32192	24173	ensure_field_index(source_location_type, "column", 3);
32193	24174	fields[3]->special = ConstValSpecialStatic;
32194	24175	fields[3]->type = ira->codegen->builtin_types.entry_u32;
32195		bigint_init_unsigned(&fields[3]->data.x_bigint, instruction->base.base.source_node->column + 1);
	24176	bigint_init_unsigned(&fields[3]->data.x_bigint, tok_loc.column + 1);
32196	24177
32197	24178	return ir_const_move(ira, &instruction->base.base, result);
32198	24179	}
...	...	@@ -32539,20 +24520,6 @@ ZigType ir_analyze(CodeGen codegen, IrExecutableSrc *old_exec, IrExecutableGen
32539	24520	old_instruction->src();
32540	24521	fprintf(stderr, "~ ");
32541	24522	ir_print_inst_src(codegen, stderr, old_instruction, 0);
32542		bool want_break = false;
32543		if (ira->break_debug_id == old_instruction->base.debug_id) {
32544		want_break = true;
32545		} else if (old_instruction->base.source_node != nullptr) {
32546		for (size_t i = 0; i < dbg_ir_breakpoints_count; i += 1) {
32547		if (dbg_ir_breakpoints_buf[i].line == old_instruction->base.source_node->line + 1 &&
32548		buf_ends_with_str(old_instruction->base.source_node->owner->data.structure.root_struct->path,
32549		dbg_ir_breakpoints_buf[i].src_file))
32550		{
32551		want_break = true;
32552		}
32553		}
32554		}
32555		if (want_break) BREAKPOINT;
32556	24523	}
32557	24524	IrInstGen *new_instruction = ir_analyze_instruction_base(ira, old_instruction);
32558	24525	if (new_instruction != nullptr) {
...	...	@@ -33541,11 +25508,3 @@ void IrAnalyze::dump() {
33541	25508	ir_print_basic_block_gen(this->codegen, stderr, this->new_irb.current_basic_block, 1);
33542	25509	}
33543	25510	}
33544
33545		void dbg_ir_break(const char *src_file, uint32_t line) {
33546		dbg_ir_breakpoints_buf[dbg_ir_breakpoints_count] = {src_file, line};
33547		dbg_ir_breakpoints_count += 1;
33548		}
33549		void dbg_ir_clear(void) {
33550		dbg_ir_breakpoints_count = 0;
33551		}

src/stage1/ir.hpp-7

...	...	@@ -10,9 +10,6 @@
10	10
11	11	#include "all_types.hpp"
12	12
13		bool ir_gen(CodeGen g, AstNode node, Scope scope, IrExecutableSrc ir_executable);
14		bool ir_gen_fn(CodeGen g, ZigFn fn_entry);
15
16	13	IrInstGen ir_create_alloca(CodeGen g, Scope scope, AstNode source_node, ZigFn *fn,
17	14	ZigType var_type, const char name_hint);
18	15
...	...	@@ -33,8 +30,4 @@ struct IrAnalyze;
33	30	ZigValue const_ptr_pointee(IrAnalyze ira, CodeGen codegen, ZigValue const_val,
34	31	AstNode *source_node);
35	32
36		// for debugging purposes
37		void dbg_ir_break(const char *src_file, uint32_t line);
38		void dbg_ir_clear(void);
39
40	33	#endif

src/stage1/parser.cpp+911-590

...	...	@@ -16,28 +16,34 @@
16	16
17	17	struct ParseContext {
18	18	Buf *buf;
19		size_t current_token;
20		ZigList<Token> *tokens;
	19	// Shortcut to `owner->data.structure.root_struct->token_ids`.
	20	TokenId *token_ids;
	21	// Shortcut to `owner->data.structure.root_struct->token_locs`.
	22	TokenLoc *token_locs;
21	23	ZigType *owner;
	24	TokenIndex current_token;
22	25	ErrColor err_color;
	26	// Shortcut to `owner->data.structure.root_struct->token_count`.
	27	uint32_t token_count;
23	28	};
24	29
25	30	struct PtrPayload {
26		Token *asterisk;
27		Token *payload;
	31	TokenIndex asterisk;
	32	TokenIndex payload;
28	33	};
29	34
30	35	struct PtrIndexPayload {
31		Token *asterisk;
32		Token *payload;
33		Token *index;
	36	TokenIndex asterisk;
	37	TokenIndex payload;
	38	TokenIndex index;
34	39	};
35	40
36	41	static AstNode ast_parse_root(ParseContext pc);
37	42	static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc);
38	43	static AstNode ast_parse_test_decl(ParseContext pc);
39	44	static AstNode ast_parse_top_level_comptime(ParseContext pc);
40		static AstNode ast_parse_top_level_decl(ParseContext pc, VisibMod visib_mod, Buf *doc_comments);
	45	static AstNode ast_parse_top_level_decl(ParseContext pc, VisibMod visib_mod,
	46	TokenIndex doc_comments);
41	47	static AstNode ast_parse_fn_proto(ParseContext pc);
42	48	static AstNode ast_parse_var_decl(ParseContext pc);
43	49	static AstNode ast_parse_container_field(ParseContext pc);
...	...	@@ -87,8 +93,8 @@ static AsmOutput ast_parse_asm_output_item(ParseContext pc);
87	93	static AstNode ast_parse_asm_input(ParseContext pc);
88	94	static AsmInput ast_parse_asm_input_item(ParseContext pc);
89	95	static AstNode ast_parse_asm_clobbers(ParseContext pc);
90		static Token ast_parse_break_label(ParseContext pc);
91		static Token ast_parse_block_label(ParseContext pc);
	96	static TokenIndex ast_parse_break_label(ParseContext *pc);
	97	static TokenIndex ast_parse_block_label(ParseContext *pc);
92	98	static AstNode ast_parse_field_init(ParseContext pc);
93	99	static AstNode ast_parse_while_continue_expr(ParseContext pc);
94	100	static AstNode ast_parse_link_section(ParseContext pc);
...	...	@@ -98,7 +104,7 @@ static AstNode ast_parse_param_type(ParseContext pc);
98	104	static AstNode ast_parse_if_prefix(ParseContext pc);
99	105	static AstNode ast_parse_while_prefix(ParseContext pc);
100	106	static AstNode ast_parse_for_prefix(ParseContext pc);
101		static Token ast_parse_payload(ParseContext pc);
	107	static TokenIndex ast_parse_payload(ParseContext *pc);
102	108	static Optional<PtrPayload> ast_parse_ptr_payload(ParseContext *pc);
103	109	static Optional<PtrIndexPayload> ast_parse_ptr_index_payload(ParseContext *pc);
104	110	static AstNode ast_parse_switch_prong(ParseContext pc);
...	...	@@ -122,27 +128,25 @@ static AstNode ast_parse_byte_align(ParseContext pc);
122	128
123	129	ATTRIBUTE_PRINTF(3, 4)
124	130	ATTRIBUTE_NORETURN
125		static void ast_error(ParseContext pc, Token token, const char *format, ...) {
	131	static void ast_error(ParseContext pc, TokenIndex token, const char format, ...) {
126	132	va_list ap;
127	133	va_start(ap, format);
128	134	Buf *msg = buf_vprintf(format, ap);
129	135	va_end(ap);
130	136
131
132		ErrorMsg *err = err_msg_create_with_line(pc->owner->data.structure.root_struct->path,
133		token->start_line, token->start_column,
134		pc->owner->data.structure.root_struct->source_code,
135		pc->owner->data.structure.root_struct->line_offsets, msg);
136		err->line_start = token->start_line;
137		err->column_start = token->start_column;
	137	RootStruct *root_struct = pc->owner->data.structure.root_struct;
	138	assert(token < root_struct->token_count);
	139	uint32_t byte_offset = root_struct->token_locs[token].offset;
	140	ErrorMsg *err = err_msg_create_with_offset(root_struct->path,
	141	byte_offset, buf_ptr(root_struct->source_code), msg);
138	142
139	143	print_err_msg(err, pc->err_color);
140	144	exit(EXIT_FAILURE);
141	145	}
142	146
143	147	ATTRIBUTE_NORETURN
144		static void ast_invalid_token_error(ParseContext pc, Token token) {
145		ast_error(pc, token, "invalid token: '%s'", token_name(token->id));
	148	static void ast_invalid_token_error(ParseContext *pc, TokenIndex token) {
	149	ast_error(pc, token, "invalid token: '%s'", token_name(pc->token_ids[token]));
146	150	}
147	151
148	152	static AstNode ast_create_node_no_line_info(ParseContext pc, NodeType type) {
...	...	@@ -152,48 +156,44 @@ static AstNode ast_create_node_no_line_info(ParseContext pc, NodeType type) {
152	156	return node;
153	157	}
154	158
155		static AstNode ast_create_node(ParseContext pc, NodeType type, Token *first_token) {
	159	static AstNode ast_create_node(ParseContext pc, NodeType type, TokenIndex first_token) {
156	160	assert(first_token);
157	161	AstNode *node = ast_create_node_no_line_info(pc, type);
158		node->line = first_token->start_line;
159		node->column = first_token->start_column;
	162	node->main_token = first_token;
160	163	return node;
161	164	}
162	165
163	166	static AstNode ast_create_node_copy_line_info(ParseContext pc, NodeType type, AstNode *from) {
164	167	assert(from);
165	168	AstNode *node = ast_create_node_no_line_info(pc, type);
166		node->line = from->line;
167		node->column = from->column;
	169	node->main_token = from->main_token;
168	170	return node;
169	171	}
170	172
171		static Token peek_token_i(ParseContext pc, size_t i) {
172		return &pc->tokens->at(pc->current_token + i);
173		}
174
175		static Token peek_token(ParseContext pc) {
176		return peek_token_i(pc, 0);
	173	static TokenIndex peek_token(ParseContext *pc) {
	174	return pc->current_token;
177	175	}
178	176
179		static Token eat_token(ParseContext pc) {
180		Token *res = peek_token(pc);
	177	static TokenIndex eat_token(ParseContext *pc) {
	178	TokenIndex res = peek_token(pc);
181	179	pc->current_token += 1;
182	180	return res;
183	181	}
184	182
185		static Token eat_token_if(ParseContext pc, TokenId id) {
186		Token *res = peek_token(pc);
187		if (res->id == id)
	183	static TokenIndex eat_token_if(ParseContext *pc, TokenId id) {
	184	TokenIndex res = peek_token(pc);
	185	if (pc->token_ids[res] == id) {
188	186	return eat_token(pc);
	187	}
189	188
190		return nullptr;
	189	return 0;
191	190	}
192	191
193		static Token expect_token(ParseContext pc, TokenId id) {
194		Token *res = eat_token(pc);
195		if (res->id != id)
196		ast_error(pc, res, "expected token '%s', found '%s'", token_name(id), token_name(res->id));
	192	static TokenIndex expect_token(ParseContext *pc, TokenId id) {
	193	TokenIndex res = eat_token(pc);
	194	TokenId actual_id = pc->token_ids[res];
	195	if (actual_id != id)
	196	ast_error(pc, res, "expected token '%s', found '%s'", token_name(id), token_name(actual_id));
197	197
198	198	return res;
199	199	}
...	...	@@ -202,23 +202,23 @@ static void put_back_token(ParseContext *pc) {
202	202	pc->current_token -= 1;
203	203	}
204	204
205		static Buf token_buf(Token token) {
206		if (token == nullptr)
	205	static Buf token_buf(ParseContext pc, TokenIndex token) {
	206	if (token == 0)
207	207	return nullptr;
208		assert(token->id == TokenIdStringLiteral \|\| token->id == TokenIdMultilineStringLiteral \|\| token->id == TokenIdSymbol);
209		return &token->data.str_lit.str;
210		}
211	208
212		static BigInt token_bigint(Token token) {
213		assert(token->id == TokenIdIntLiteral);
214		return &token->data.int_lit.bigint;
	209	RootStruct *root_struct = pc->owner->data.structure.root_struct;
	210	if (root_struct->token_ids[token] == TokenIdIdentifier) {
	211	return token_identifier_buf(root_struct, token);
	212	} else if (root_struct->token_ids[token] == TokenIdStringLiteral) {
	213	return token_string_literal_buf(root_struct, token);
	214	} else {
	215	zig_unreachable();
	216	}
215	217	}
216	218
217		static AstNode token_symbol(ParseContext pc, Token *token) {
218		assert(token->id == TokenIdSymbol);
219		AstNode *res = ast_create_node(pc, NodeTypeSymbol, token);
220		res->data.symbol_expr.symbol = token_buf(token);
221		return res;
	219	static AstNode token_identifier(ParseContext pc, TokenIndex token) {
	220	assert(pc->token_ids[token] == TokenIdIdentifier);
	221	return ast_create_node(pc, NodeTypeIdentifier, token);
222	222	}
223	223
224	224	// (Rule SEP)* Rule?
...	...	@@ -231,7 +231,7 @@ static ZigList<T > ast_parse_list(ParseContext pc, TokenId sep, T (parser)(P
231	231	break;
232	232
233	233	res.append(curr);
234		if (eat_token_if(pc, sep) == nullptr)
	234	if (eat_token_if(pc, sep) == 0)
235	235	break;
236	236	}
237	237
...	...	@@ -355,22 +355,22 @@ static AstNode ast_parse_if_expr_helper(ParseContext pc, AstNode (body_parse
355	355	return nullptr;
356	356
357	357	AstNode *body = ast_expect(pc, body_parser);
358		Token *err_payload = nullptr;
	358	TokenIndex err_payload = 0;
359	359	AstNode *else_body = nullptr;
360		if (eat_token_if(pc, TokenIdKeywordElse) != nullptr) {
	360	if (eat_token_if(pc, TokenIdKeywordElse) != 0) {
361	361	err_payload = ast_parse_payload(pc);
362	362	else_body = ast_expect(pc, body_parser);
363	363	}
364	364
365	365	assert(res->type == NodeTypeIfOptional);
366		if (err_payload != nullptr) {
	366	if (err_payload != 0) {
367	367	AstNodeTestExpr old = res->data.test_expr;
368	368	res->type = NodeTypeIfErrorExpr;
369	369	res->data.if_err_expr.target_node = old.target_node;
370	370	res->data.if_err_expr.var_is_ptr = old.var_is_ptr;
371	371	res->data.if_err_expr.var_symbol = old.var_symbol;
372	372	res->data.if_err_expr.then_node = body;
373		res->data.if_err_expr.err_symbol = token_buf(err_payload);
	373	res->data.if_err_expr.err_symbol = token_buf(pc, err_payload);
374	374	res->data.if_err_expr.else_node = else_body;
375	375	return res;
376	376	}
...	...	@@ -395,22 +395,22 @@ static AstNode *ast_parse_loop_expr_helper(
395	395	AstNode (for_parser)(ParseContext *),
396	396	AstNode (while_parser)(ParseContext *)
397	397	) {
398		Token *inline_token = eat_token_if(pc, TokenIdKeywordInline);
	398	TokenIndex inline_token = eat_token_if(pc, TokenIdKeywordInline);
399	399	AstNode *for_expr = for_parser(pc);
400	400	if (for_expr != nullptr) {
401	401	assert(for_expr->type == NodeTypeForExpr);
402		for_expr->data.for_expr.is_inline = inline_token != nullptr;
	402	for_expr->data.for_expr.is_inline = inline_token != 0;
403	403	return for_expr;
404	404	}
405	405
406	406	AstNode *while_expr = while_parser(pc);
407	407	if (while_expr != nullptr) {
408	408	assert(while_expr->type == NodeTypeWhileExpr);
409		while_expr->data.while_expr.is_inline = inline_token != nullptr;
	409	while_expr->data.while_expr.is_inline = inline_token != 0;
410	410	return while_expr;
411	411	}
412	412
413		if (inline_token != nullptr)
	413	if (inline_token != 0)
414	414	ast_invalid_token_error(pc, peek_token(pc));
415	415	return nullptr;
416	416	}
...	...	@@ -423,7 +423,7 @@ static AstNode ast_parse_for_expr_helper(ParseContext pc, AstNode (body_pars
423	423
424	424	AstNode *body = ast_expect(pc, body_parser);
425	425	AstNode *else_body = nullptr;
426		if (eat_token_if(pc, TokenIdKeywordElse) != nullptr)
	426	if (eat_token_if(pc, TokenIdKeywordElse) != 0)
427	427	else_body = ast_expect(pc, body_parser);
428	428
429	429	assert(res->type == NodeTypeForExpr);
...	...	@@ -439,24 +439,24 @@ static AstNode ast_parse_while_expr_helper(ParseContext pc, AstNode (body_pa
439	439	return nullptr;
440	440
441	441	AstNode *body = ast_expect(pc, body_parser);
442		Token *err_payload = nullptr;
	442	TokenIndex err_payload = 0;
443	443	AstNode *else_body = nullptr;
444		if (eat_token_if(pc, TokenIdKeywordElse) != nullptr) {
	444	if (eat_token_if(pc, TokenIdKeywordElse) != 0) {
445	445	err_payload = ast_parse_payload(pc);
446	446	else_body = ast_expect(pc, body_parser);
447	447	}
448	448
449	449	assert(res->type == NodeTypeWhileExpr);
450	450	res->data.while_expr.body = body;
451		res->data.while_expr.err_symbol = token_buf(err_payload);
	451	res->data.while_expr.err_symbol = token_buf(pc, err_payload);
452	452	res->data.while_expr.else_node = else_body;
453	453	return res;
454	454	}
455	455
456	456	template<TokenId id, BinOpType op>
457	457	AstNode ast_parse_bin_op_simple(ParseContext pc) {
458		Token *op_token = eat_token_if(pc, id);
459		if (op_token == nullptr)
	458	TokenIndex op_token = eat_token_if(pc, id);
	459	if (op_token == 0)
460	460	return nullptr;
461	461
462	462	AstNode *res = ast_create_node(pc, NodeTypeBinOpExpr, op_token);
...	...	@@ -464,20 +464,25 @@ AstNode ast_parse_bin_op_simple(ParseContext pc) {
464	464	return res;
465	465	}
466	466
467		AstNode ast_parse(Buf buf, ZigList<Token> tokens, ZigType owner, ErrColor err_color) {
	467	AstNode ast_parse(Buf buf, ZigType *owner, ErrColor err_color) {
	468	RootStruct *root_struct = owner->data.structure.root_struct;
	469
468	470	ParseContext pc = {};
469	471	pc.err_color = err_color;
470	472	pc.owner = owner;
471	473	pc.buf = buf;
472		pc.tokens = tokens;
	474	pc.token_ids = root_struct->token_ids;
	475	pc.token_locs = root_struct->token_locs;
	476	pc.token_count = root_struct->token_count;
	477	pc.current_token = 1; // Skip over the first (invalid) token.
473	478	return ast_parse_root(&pc);
474	479	}
475	480
476	481	// Root <- skip ContainerMembers eof
477	482	static AstNode ast_parse_root(ParseContext pc) {
478		Token *first = peek_token(pc);
	483	TokenIndex first = peek_token(pc);
479	484	AstNodeContainerDecl members = ast_parse_container_members(pc);
480		if (pc->current_token != pc->tokens->length - 1)
	485	if (pc->current_token != pc->token_count - 1)
481	486	ast_invalid_token_error(pc, peek_token(pc));
482	487
483	488	AstNode *node = ast_create_node(pc, NodeTypeContainerDecl, first);
...	...	@@ -486,70 +491,26 @@ static AstNode ast_parse_root(ParseContext pc) {
486	491	node->data.container_decl.layout = ContainerLayoutAuto;
487	492	node->data.container_decl.kind = ContainerKindStruct;
488	493	node->data.container_decl.is_root = true;
489		if (buf_len(&members.doc_comments) != 0) {
490		node->data.container_decl.doc_comments = members.doc_comments;
491		}
	494	node->data.container_decl.doc_comments = members.doc_comments;
492	495
493	496	return node;
494	497	}
495	498
496		static Token ast_parse_multiline_string_literal(ParseContext pc, Buf *buf) {
497		Token *first_str_token = nullptr;
498		Token *str_token = nullptr;
499		while ((str_token = eat_token_if(pc, TokenIdMultilineStringLiteral))) {
500		if (first_str_token == nullptr) {
501		first_str_token = str_token;
502		}
503		if (buf->list.length == 0) {
504		buf_resize(buf, 0);
505		}
506		buf_append_buf(buf, token_buf(str_token));
507
508		// Ignore inline comments
509		size_t cur_token = pc->current_token;
510		while (eat_token_if(pc, TokenIdDocComment));
511
512		// Lookahead to see if there's another multilne string literal,
513		// if not, we have to revert back to before the doc comment
514		if (peek_token(pc)->id != TokenIdMultilineStringLiteral) {
515		pc->current_token = cur_token;
516		} else {
517		buf_append_char(buf, '\n'); // Add a newline between comments
518		}
	499	static TokenIndex ast_parse_multi_tok(ParseContext *pc, TokenId token_id) {
	500	TokenIndex first_token = eat_token_if(pc, token_id);
	501	TokenIndex token = first_token;
	502	while (token != 0) {
	503	token = eat_token_if(pc, token_id);
519	504	}
520		return first_str_token;
	505	return first_token;
521	506	}
522	507
523		static Token ast_parse_doc_comments(ParseContext pc, Buf *buf) {
524		Token *first_doc_token = nullptr;
525		Token *doc_token = nullptr;
526		while ((doc_token = eat_token_if(pc, TokenIdDocComment))) {
527		if (first_doc_token == nullptr) {
528		first_doc_token = doc_token;
529		}
530		if (buf->list.length == 0) {
531		buf_resize(buf, 0);
532		}
533		// chops off '///' but leaves '\n'
534		buf_append_mem(buf, buf_ptr(pc->buf) + doc_token->start_pos + 3,
535		doc_token->end_pos - doc_token->start_pos - 3);
536		}
537		return first_doc_token;
	508	static TokenIndex ast_parse_doc_comments(ParseContext *pc) {
	509	return ast_parse_multi_tok(pc, TokenIdDocComment);
538	510	}
539	511
540		static void ast_parse_container_doc_comments(ParseContext pc, Buf buf) {
541		if (buf_len(buf) != 0 && peek_token(pc)->id == TokenIdContainerDocComment) {
542		buf_append_char(buf, '\n');
543		}
544		Token *doc_token = nullptr;
545		while ((doc_token = eat_token_if(pc, TokenIdContainerDocComment))) {
546		if (buf->list.length == 0) {
547		buf_resize(buf, 0);
548		}
549		// chops off '//!' but leaves '\n'
550		buf_append_mem(buf, buf_ptr(pc->buf) + doc_token->start_pos + 3,
551		doc_token->end_pos - doc_token->start_pos - 3);
552		}
	512	static TokenIndex ast_parse_container_doc_comments(ParseContext *pc) {
	513	return ast_parse_multi_tok(pc, TokenIdContainerDocComment);
553	514	}
554	515
555	516	enum ContainerFieldState {
...	...	@@ -570,14 +531,11 @@ enum ContainerFieldState {
570	531	// /
571	532	static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
572	533	AstNodeContainerDecl res = {};
573		Buf tld_doc_comment_buf = BUF_INIT;
574		buf_resize(&tld_doc_comment_buf, 0);
575	534	ContainerFieldState field_state = ContainerFieldStateNone;
576		Token *first_token = nullptr;
	535	TokenIndex first_token = 0;
	536	res.doc_comments = ast_parse_container_doc_comments(pc);
577	537	for (;;) {
578		ast_parse_container_doc_comments(pc, &tld_doc_comment_buf);
579
580		Token *peeked_token = peek_token(pc);
	538	TokenIndex peeked_token = peek_token(pc);
581	539
582	540	AstNode *test_decl = ast_parse_test_decl(pc);
583	541	if (test_decl != nullptr) {
...	...	@@ -599,15 +557,14 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
599	557	continue;
600	558	}
601	559
602		Buf doc_comment_buf = BUF_INIT;
603		ast_parse_doc_comments(pc, &doc_comment_buf);
	560	TokenIndex first_doc_token = ast_parse_doc_comments(pc);
604	561
605	562	peeked_token = peek_token(pc);
606	563
607		Token *visib_token = eat_token_if(pc, TokenIdKeywordPub);
608		VisibMod visib_mod = visib_token != nullptr ? VisibModPub : VisibModPrivate;
	564	TokenIndex visib_token = eat_token_if(pc, TokenIdKeywordPub);
	565	VisibMod visib_mod = (visib_token != 0) ? VisibModPub : VisibModPrivate;
609	566
610		AstNode *top_level_decl = ast_parse_top_level_decl(pc, visib_mod, &doc_comment_buf);
	567	AstNode *top_level_decl = ast_parse_top_level_decl(pc, visib_mod, first_doc_token);
611	568	if (top_level_decl != nullptr) {
612	569	if (field_state == ContainerFieldStateSeen) {
613	570	field_state = ContainerFieldStateEnd;
...	...	@@ -617,11 +574,11 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
617	574	continue;
618	575	}
619	576
620		if (visib_token != nullptr) {
	577	if (visib_token != 0) {
621	578	ast_error(pc, peek_token(pc), "expected function or variable declaration after pub");
622	579	}
623	580
624		Token *comptime_token = eat_token_if(pc, TokenIdKeywordCompTime);
	581	TokenIndex comptime_token = eat_token_if(pc, TokenIdKeywordCompTime);
625	582
626	583	AstNode *container_field = ast_parse_container_field(pc);
627	584	if (container_field != nullptr) {
...	...	@@ -636,10 +593,10 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
636	593	}
637	594
638	595	assert(container_field->type == NodeTypeStructField);
639		container_field->data.struct_field.doc_comments = doc_comment_buf;
	596	container_field->data.struct_field.doc_comments = first_doc_token;
640	597	container_field->data.struct_field.comptime_token = comptime_token;
641	598	res.fields.append(container_field);
642		if (eat_token_if(pc, TokenIdComma) != nullptr) {
	599	if (eat_token_if(pc, TokenIdComma) != 0) {
643	600	continue;
644	601	} else {
645	602	break;
...	...	@@ -648,33 +605,32 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
648	605
649	606	break;
650	607	}
651		res.doc_comments = tld_doc_comment_buf;
652	608	return res;
653	609	}
654	610
655	611	// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block
656	612	static AstNode ast_parse_test_decl(ParseContext pc) {
657		Token *test = eat_token_if(pc, TokenIdKeywordTest);
658		if (test == nullptr)
	613	TokenIndex test = eat_token_if(pc, TokenIdKeywordTest);
	614	if (test == 0)
659	615	return nullptr;
660	616
661		Token *name = eat_token_if(pc, TokenIdStringLiteral);
	617	TokenIndex name = eat_token_if(pc, TokenIdStringLiteral);
662	618	AstNode *block = ast_expect(pc, ast_parse_block);
663	619	AstNode *res = ast_create_node(pc, NodeTypeTestDecl, test);
664		res->data.test_decl.name = name ? token_buf(name) : nullptr;
	620	res->data.test_decl.name = name ? token_buf(pc, name) : nullptr;
665	621	res->data.test_decl.body = block;
666	622	return res;
667	623	}
668	624
669	625	// TopLevelComptime <- KEYWORD_comptime BlockExpr
670	626	static AstNode ast_parse_top_level_comptime(ParseContext pc) {
671		Token *comptime = eat_token_if(pc, TokenIdKeywordCompTime);
672		if (comptime == nullptr)
	627	TokenIndex comptime = eat_token_if(pc, TokenIdKeywordCompTime);
	628	if (comptime == 0)
673	629	return nullptr;
674	630
675	631	// 1 token lookahead because it could be a comptime struct field
676		Token *lbrace = peek_token(pc);
677		if (lbrace->id != TokenIdLBrace) {
	632	TokenIndex lbrace = peek_token(pc);
	633	if (pc->token_ids[lbrace] != TokenIdLBrace) {
678	634	put_back_token(pc);
679	635	return nullptr;
680	636	}
...	...	@@ -689,39 +645,40 @@ static AstNode ast_parse_top_level_comptime(ParseContext pc) {
689	645	// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)
690	646	// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl
691	647	// / KEYWORD_use Expr SEMICOLON
692		static AstNode ast_parse_top_level_decl(ParseContext pc, VisibMod visib_mod, Buf *doc_comments) {
693		Token *first = eat_token_if(pc, TokenIdKeywordExport);
694		if (first == nullptr)
	648	static AstNode ast_parse_top_level_decl(ParseContext pc, VisibMod visib_mod,
	649	TokenIndex doc_comments)
	650	{
	651	TokenIndex first = eat_token_if(pc, TokenIdKeywordExport);
	652	if (first == 0)
695	653	first = eat_token_if(pc, TokenIdKeywordExtern);
696		if (first == nullptr)
	654	if (first == 0)
697	655	first = eat_token_if(pc, TokenIdKeywordInline);
698		if (first == nullptr)
	656	if (first == 0)
699	657	first = eat_token_if(pc, TokenIdKeywordNoInline);
700		if (first != nullptr) {
701		Token *lib_name = nullptr;
702		if (first->id == TokenIdKeywordExtern)
	658	if (first != 0) {
	659	TokenIndex lib_name = 0;
	660	if (pc->token_ids[first] == TokenIdKeywordExtern)
703	661	lib_name = eat_token_if(pc, TokenIdStringLiteral);
704	662
705		if (first->id != TokenIdKeywordNoInline && first->id != TokenIdKeywordInline) {
706		Token *thread_local_kw = eat_token_if(pc, TokenIdKeywordThreadLocal);
	663	if (pc->token_ids[first] != TokenIdKeywordNoInline && pc->token_ids[first] != TokenIdKeywordInline) {
	664	TokenIndex thread_local_kw = eat_token_if(pc, TokenIdKeywordThreadLocal);
707	665	AstNode *var_decl = ast_parse_var_decl(pc);
708	666	if (var_decl != nullptr) {
709	667	assert(var_decl->type == NodeTypeVariableDeclaration);
710		if (first->id == TokenIdKeywordExtern && var_decl->data.variable_declaration.expr != nullptr) {
	668	if (pc->token_ids[first] == TokenIdKeywordExtern && var_decl->data.variable_declaration.expr != nullptr) {
711	669	ast_error(pc, first, "extern variables have no initializers");
712	670	}
713		var_decl->line = first->start_line;
714		var_decl->column = first->start_column;
	671	var_decl->main_token = first;
715	672	var_decl->data.variable_declaration.threadlocal_tok = thread_local_kw;
716	673	var_decl->data.variable_declaration.visib_mod = visib_mod;
717		var_decl->data.variable_declaration.doc_comments = *doc_comments;
718		var_decl->data.variable_declaration.is_extern = first->id == TokenIdKeywordExtern;
719		var_decl->data.variable_declaration.is_export = first->id == TokenIdKeywordExport;
720		var_decl->data.variable_declaration.lib_name = token_buf(lib_name);
	674	var_decl->data.variable_declaration.doc_comments = doc_comments;
	675	var_decl->data.variable_declaration.is_extern = pc->token_ids[first] == TokenIdKeywordExtern;
	676	var_decl->data.variable_declaration.is_export = pc->token_ids[first] == TokenIdKeywordExport;
	677	var_decl->data.variable_declaration.lib_name = token_buf(pc, lib_name);
721	678	return var_decl;
722	679	}
723	680
724		if (thread_local_kw != nullptr)
	681	if (thread_local_kw != 0)
725	682	put_back_token(pc);
726	683	}
727	684
...	...	@@ -732,14 +689,13 @@ static AstNode ast_parse_top_level_decl(ParseContext pc, VisibMod visib_mod, B
732	689	expect_token(pc, TokenIdSemicolon);
733	690
734	691	assert(fn_proto->type == NodeTypeFnProto);
735		fn_proto->line = first->start_line;
736		fn_proto->column = first->start_column;
	692	fn_proto->main_token = first;
737	693	fn_proto->data.fn_proto.visib_mod = visib_mod;
738		fn_proto->data.fn_proto.doc_comments = *doc_comments;
	694	fn_proto->data.fn_proto.doc_comments = doc_comments;
739	695	if (!fn_proto->data.fn_proto.is_extern)
740		fn_proto->data.fn_proto.is_extern = first->id == TokenIdKeywordExtern;
741		fn_proto->data.fn_proto.is_export = first->id == TokenIdKeywordExport;
742		switch (first->id) {
	696	fn_proto->data.fn_proto.is_extern = pc->token_ids[first] == TokenIdKeywordExtern;
	697	fn_proto->data.fn_proto.is_export = pc->token_ids[first] == TokenIdKeywordExport;
	698	switch (pc->token_ids[first]) {
743	699	case TokenIdKeywordInline:
744	700	fn_proto->data.fn_proto.fn_inline = FnInlineAlways;
745	701	break;
...	...	@@ -750,7 +706,7 @@ static AstNode ast_parse_top_level_decl(ParseContext pc, VisibMod visib_mod, B
750	706	fn_proto->data.fn_proto.fn_inline = FnInlineAuto;
751	707	break;
752	708	}
753		fn_proto->data.fn_proto.lib_name = token_buf(lib_name);
	709	fn_proto->data.fn_proto.lib_name = token_buf(pc, lib_name);
754	710
755	711	AstNode *res = fn_proto;
756	712	if (body != nullptr) {
...	...	@@ -769,17 +725,17 @@ static AstNode ast_parse_top_level_decl(ParseContext pc, VisibMod visib_mod, B
769	725	ast_invalid_token_error(pc, peek_token(pc));
770	726	}
771	727
772		Token *thread_local_kw = eat_token_if(pc, TokenIdKeywordThreadLocal);
	728	TokenIndex thread_local_kw = eat_token_if(pc, TokenIdKeywordThreadLocal);
773	729	AstNode *var_decl = ast_parse_var_decl(pc);
774	730	if (var_decl != nullptr) {
775	731	assert(var_decl->type == NodeTypeVariableDeclaration);
776	732	var_decl->data.variable_declaration.visib_mod = visib_mod;
777		var_decl->data.variable_declaration.doc_comments = *doc_comments;
	733	var_decl->data.variable_declaration.doc_comments = doc_comments;
778	734	var_decl->data.variable_declaration.threadlocal_tok = thread_local_kw;
779	735	return var_decl;
780	736	}
781	737
782		if (thread_local_kw != nullptr)
	738	if (thread_local_kw != 0)
783	739	put_back_token(pc);
784	740
785	741	AstNode *fn_proto = ast_parse_fn_proto(pc);
...	...	@@ -790,7 +746,7 @@ static AstNode ast_parse_top_level_decl(ParseContext pc, VisibMod visib_mod, B
790	746
791	747	assert(fn_proto->type == NodeTypeFnProto);
792	748	fn_proto->data.fn_proto.visib_mod = visib_mod;
793		fn_proto->data.fn_proto.doc_comments = *doc_comments;
	749	fn_proto->data.fn_proto.doc_comments = doc_comments;
794	750	AstNode *res = fn_proto;
795	751	if (body != nullptr) {
796	752	res = ast_create_node_copy_line_info(pc, NodeTypeFnDef, fn_proto);
...	...	@@ -802,8 +758,8 @@ static AstNode ast_parse_top_level_decl(ParseContext pc, VisibMod visib_mod, B
802	758	return res;
803	759	}
804	760
805		Token *usingnamespace = eat_token_if(pc, TokenIdKeywordUsingNamespace);
806		if (usingnamespace != nullptr) {
	761	TokenIndex usingnamespace = eat_token_if(pc, TokenIdKeywordUsingNamespace);
	762	if (usingnamespace != 0) {
807	763	AstNode *expr = ast_expect(pc, ast_parse_expr);
808	764	expect_token(pc, TokenIdSemicolon);
809	765
...	...	@@ -818,12 +774,12 @@ static AstNode ast_parse_top_level_decl(ParseContext pc, VisibMod visib_mod, B
818	774
819	775	// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_anytype / TypeExpr)
820	776	static AstNode ast_parse_fn_proto(ParseContext pc) {
821		Token *first = eat_token_if(pc, TokenIdKeywordFn);
822		if (first == nullptr) {
	777	TokenIndex first = eat_token_if(pc, TokenIdKeywordFn);
	778	if (first == 0) {
823	779	return nullptr;
824	780	}
825	781
826		Token *identifier = eat_token_if(pc, TokenIdSymbol);
	782	TokenIndex identifier = eat_token_if(pc, TokenIdIdentifier);
827	783	expect_token(pc, TokenIdLParen);
828	784	ZigList<AstNode *> params = ast_parse_list(pc, TokenIdComma, ast_parse_param_decl);
829	785	expect_token(pc, TokenIdRParen);
...	...	@@ -831,29 +787,29 @@ static AstNode ast_parse_fn_proto(ParseContext pc) {
831	787	AstNode *align_expr = ast_parse_byte_align(pc);
832	788	AstNode *section_expr = ast_parse_link_section(pc);
833	789	AstNode *callconv_expr = ast_parse_callconv(pc);
834		Token *exmark = nullptr;
	790	TokenIndex exmark = 0;
835	791	AstNode *return_type = nullptr;
836	792
837	793	exmark = eat_token_if(pc, TokenIdBang);
838	794	return_type = ast_parse_type_expr(pc);
839	795	if (return_type == nullptr) {
840		Token *next = peek_token(pc);
	796	TokenIndex next = peek_token(pc);
841	797	ast_error(
842	798	pc,
843	799	next,
844	800	"expected return type (use 'void' to return nothing), found: '%s'",
845		token_name(next->id)
	801	token_name(pc->token_ids[next])
846	802	);
847	803	}
848	804
849	805	AstNode *res = ast_create_node(pc, NodeTypeFnProto, first);
850	806	res->data.fn_proto = {};
851		res->data.fn_proto.name = token_buf(identifier);
	807	res->data.fn_proto.name = token_buf(pc, identifier);
852	808	res->data.fn_proto.params = params;
853	809	res->data.fn_proto.align_expr = align_expr;
854	810	res->data.fn_proto.section_expr = section_expr;
855	811	res->data.fn_proto.callconv_expr = callconv_expr;
856		res->data.fn_proto.auto_err_set = exmark != nullptr;
	812	res->data.fn_proto.auto_err_set = exmark != 0;
857	813	res->data.fn_proto.return_type = return_type;
858	814
859	815	for (size_t i = 0; i < params.length; i++) {
...	...	@@ -869,28 +825,28 @@ static AstNode ast_parse_fn_proto(ParseContext pc) {
869	825
870	826	// VarDecl <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? LinkSection? (EQUAL Expr)? SEMICOLON
871	827	static AstNode ast_parse_var_decl(ParseContext pc) {
872		Token *mut_kw = eat_token_if(pc, TokenIdKeywordConst);
873		if (mut_kw == nullptr)
	828	TokenIndex mut_kw = eat_token_if(pc, TokenIdKeywordConst);
	829	if (mut_kw == 0)
874	830	mut_kw = eat_token_if(pc, TokenIdKeywordVar);
875		if (mut_kw == nullptr)
	831	if (mut_kw == 0)
876	832	return nullptr;
877	833
878		Token *identifier = expect_token(pc, TokenIdSymbol);
	834	TokenIndex identifier = expect_token(pc, TokenIdIdentifier);
879	835	AstNode *type_expr = nullptr;
880		if (eat_token_if(pc, TokenIdColon) != nullptr)
	836	if (eat_token_if(pc, TokenIdColon) != 0)
881	837	type_expr = ast_expect(pc, ast_parse_type_expr);
882	838
883	839	AstNode *align_expr = ast_parse_byte_align(pc);
884	840	AstNode *section_expr = ast_parse_link_section(pc);
885	841	AstNode *expr = nullptr;
886		if (eat_token_if(pc, TokenIdEq) != nullptr)
	842	if (eat_token_if(pc, TokenIdEq) != 0)
887	843	expr = ast_expect(pc, ast_parse_expr);
888	844
889	845	expect_token(pc, TokenIdSemicolon);
890	846
891	847	AstNode *res = ast_create_node(pc, NodeTypeVariableDeclaration, mut_kw);
892		res->data.variable_declaration.is_const = mut_kw->id == TokenIdKeywordConst;
893		res->data.variable_declaration.symbol = token_buf(identifier);
	848	res->data.variable_declaration.is_const = pc->token_ids[mut_kw] == TokenIdKeywordConst;
	849	res->data.variable_declaration.symbol = token_buf(pc, identifier);
894	850	res->data.variable_declaration.type = type_expr;
895	851	res->data.variable_declaration.align_expr = align_expr;
896	852	res->data.variable_declaration.section_expr = section_expr;
...	...	@@ -900,14 +856,14 @@ static AstNode ast_parse_var_decl(ParseContext pc) {
900	856
901	857	// ContainerField <- KEYWORD_comptime? IDENTIFIER (COLON TypeExpr ByteAlign?)? (EQUAL Expr)?
902	858	static AstNode ast_parse_container_field(ParseContext pc) {
903		Token *identifier = eat_token_if(pc, TokenIdSymbol);
904		if (identifier == nullptr)
	859	TokenIndex identifier = eat_token_if(pc, TokenIdIdentifier);
	860	if (identifier == 0)
905	861	return nullptr;
906	862
907	863	AstNode *type_expr = nullptr;
908		if (eat_token_if(pc, TokenIdColon) != nullptr) {
909		Token *anytype_tok = eat_token_if(pc, TokenIdKeywordAnyType);
910		if (anytype_tok != nullptr) {
	864	if (eat_token_if(pc, TokenIdColon) != 0) {
	865	TokenIndex anytype_tok = eat_token_if(pc, TokenIdKeywordAnyType);
	866	if (anytype_tok != 0) {
911	867	type_expr = ast_create_node(pc, NodeTypeAnyTypeField, anytype_tok);
912	868	} else {
913	869	type_expr = ast_expect(pc, ast_parse_type_expr);
...	...	@@ -915,11 +871,11 @@ static AstNode ast_parse_container_field(ParseContext pc) {
915	871	}
916	872	AstNode *align_expr = ast_parse_byte_align(pc);
917	873	AstNode *expr = nullptr;
918		if (eat_token_if(pc, TokenIdEq) != nullptr)
	874	if (eat_token_if(pc, TokenIdEq) != 0)
919	875	expr = ast_expect(pc, ast_parse_expr);
920	876
921	877	AstNode *res = ast_create_node(pc, NodeTypeStructField, identifier);
922		res->data.struct_field.name = token_buf(identifier);
	878	res->data.struct_field.name = token_buf(pc, identifier);
923	879	res->data.struct_field.type = type_expr;
924	880	res->data.struct_field.value = expr;
925	881	res->data.struct_field.align_expr = align_expr;
...	...	@@ -938,52 +894,52 @@ static AstNode ast_parse_container_field(ParseContext pc) {
938	894	// / SwitchExpr
939	895	// / AssignExpr SEMICOLON
940	896	static AstNode ast_parse_statement(ParseContext pc) {
941		Token *comptime = eat_token_if(pc, TokenIdKeywordCompTime);
	897	TokenIndex comptime = eat_token_if(pc, TokenIdKeywordCompTime);
942	898	AstNode *var_decl = ast_parse_var_decl(pc);
943	899	if (var_decl != nullptr) {
944	900	assert(var_decl->type == NodeTypeVariableDeclaration);
945		var_decl->data.variable_declaration.is_comptime = comptime != nullptr;
	901	var_decl->data.variable_declaration.is_comptime = comptime != 0;
946	902	return var_decl;
947	903	}
948	904
949		if (comptime != nullptr) {
	905	if (comptime != 0) {
950	906	AstNode *statement = ast_expect(pc, ast_parse_block_expr_statement);
951	907	AstNode *res = ast_create_node(pc, NodeTypeCompTime, comptime);
952	908	res->data.comptime_expr.expr = statement;
953	909	return res;
954	910	}
955	911
956		Token *nosuspend = eat_token_if(pc, TokenIdKeywordNoSuspend);
957		if (nosuspend != nullptr) {
	912	TokenIndex nosuspend = eat_token_if(pc, TokenIdKeywordNoSuspend);
	913	if (nosuspend != 0) {
958	914	AstNode *statement = ast_expect(pc, ast_parse_block_expr_statement);
959	915	AstNode *res = ast_create_node(pc, NodeTypeNoSuspend, nosuspend);
960	916	res->data.nosuspend_expr.expr = statement;
961	917	return res;
962	918	}
963	919
964		Token *suspend = eat_token_if(pc, TokenIdKeywordSuspend);
965		if (suspend != nullptr) {
	920	TokenIndex suspend = eat_token_if(pc, TokenIdKeywordSuspend);
	921	if (suspend != 0) {
966	922	AstNode *statement = ast_expect(pc, ast_parse_block_expr_statement);
967	923	AstNode *res = ast_create_node(pc, NodeTypeSuspend, suspend);
968	924	res->data.suspend.block = statement;
969	925	return res;
970	926	}
971	927
972		Token *defer = eat_token_if(pc, TokenIdKeywordDefer);
973		if (defer == nullptr)
	928	TokenIndex defer = eat_token_if(pc, TokenIdKeywordDefer);
	929	if (defer == 0)
974	930	defer = eat_token_if(pc, TokenIdKeywordErrdefer);
975		if (defer != nullptr) {
976		Token *payload = (defer->id == TokenIdKeywordErrdefer) ?
977		ast_parse_payload(pc) : nullptr;
	931	if (defer != 0) {
	932	TokenIndex payload = (pc->token_ids[defer] == TokenIdKeywordErrdefer) ?
	933	ast_parse_payload(pc) : 0;
978	934	AstNode *statement = ast_expect(pc, ast_parse_block_expr_statement);
979	935	AstNode *res = ast_create_node(pc, NodeTypeDefer, defer);
980	936
981	937	res->data.defer.kind = ReturnKindUnconditional;
982	938	res->data.defer.expr = statement;
983		if (defer->id == TokenIdKeywordErrdefer) {
	939	if (pc->token_ids[defer] == TokenIdKeywordErrdefer) {
984	940	res->data.defer.kind = ReturnKindError;
985		if (payload != nullptr)
986		res->data.defer.err_payload = token_symbol(pc, payload);
	941	if (payload != 0)
	942	res->data.defer.err_payload = token_identifier(pc, payload);
987	943	}
988	944	return res;
989	945	}
...	...	@@ -1025,13 +981,13 @@ static AstNode ast_parse_if_statement(ParseContext pc) {
1025	981	}
1026	982
1027	983	if (body == nullptr) {
1028		Token *tok = eat_token(pc);
1029		ast_error(pc, tok, "expected if body, found '%s'", token_name(tok->id));
	984	TokenIndex tok = eat_token(pc);
	985	ast_error(pc, tok, "expected if body, found '%s'", token_name(pc->token_ids[tok]));
1030	986	}
1031	987
1032		Token *err_payload = nullptr;
	988	TokenIndex err_payload = 0;
1033	989	AstNode *else_body = nullptr;
1034		if (eat_token_if(pc, TokenIdKeywordElse) != nullptr) {
	990	if (eat_token_if(pc, TokenIdKeywordElse) != 0) {
1035	991	err_payload = ast_parse_payload(pc);
1036	992	else_body = ast_expect(pc, ast_parse_statement);
1037	993	}
...	...	@@ -1040,14 +996,14 @@ static AstNode ast_parse_if_statement(ParseContext pc) {
1040	996	expect_token(pc, TokenIdSemicolon);
1041	997
1042	998	assert(res->type == NodeTypeIfOptional);
1043		if (err_payload != nullptr) {
	999	if (err_payload != 0) {
1044	1000	AstNodeTestExpr old = res->data.test_expr;
1045	1001	res->type = NodeTypeIfErrorExpr;
1046	1002	res->data.if_err_expr.target_node = old.target_node;
1047	1003	res->data.if_err_expr.var_is_ptr = old.var_is_ptr;
1048	1004	res->data.if_err_expr.var_symbol = old.var_symbol;
1049	1005	res->data.if_err_expr.then_node = body;
1050		res->data.if_err_expr.err_symbol = token_buf(err_payload);
	1006	res->data.if_err_expr.err_symbol = token_buf(pc, err_payload);
1051	1007	res->data.if_err_expr.else_node = else_body;
1052	1008	return res;
1053	1009	}
...	...	@@ -1068,11 +1024,11 @@ static AstNode ast_parse_if_statement(ParseContext pc) {
1068	1024
1069	1025	// LabeledStatement <- BlockLabel? (Block / LoopStatement)
1070	1026	static AstNode ast_parse_labeled_statement(ParseContext pc) {
1071		Token *label = ast_parse_block_label(pc);
	1027	TokenIndex label = ast_parse_block_label(pc);
1072	1028	AstNode *block = ast_parse_block(pc);
1073	1029	if (block != nullptr) {
1074	1030	assert(block->type == NodeTypeBlock);
1075		block->data.block.name = token_buf(label);
	1031	block->data.block.name = token_buf(pc, label);
1076	1032	return block;
1077	1033	}
1078	1034
...	...	@@ -1080,10 +1036,10 @@ static AstNode ast_parse_labeled_statement(ParseContext pc) {
1080	1036	if (loop != nullptr) {
1081	1037	switch (loop->type) {
1082	1038	case NodeTypeForExpr:
1083		loop->data.for_expr.name = token_buf(label);
	1039	loop->data.for_expr.name = token_buf(pc, label);
1084	1040	break;
1085	1041	case NodeTypeWhileExpr:
1086		loop->data.while_expr.name = token_buf(label);
	1042	loop->data.while_expr.name = token_buf(pc, label);
1087	1043	break;
1088	1044	default:
1089	1045	zig_unreachable();
...	...	@@ -1091,29 +1047,29 @@ static AstNode ast_parse_labeled_statement(ParseContext pc) {
1091	1047	return loop;
1092	1048	}
1093	1049
1094		if (label != nullptr)
	1050	if (label != 0)
1095	1051	ast_invalid_token_error(pc, peek_token(pc));
1096	1052	return nullptr;
1097	1053	}
1098	1054
1099	1055	// LoopStatement <- KEYWORD_inline? (ForStatement / WhileStatement)
1100	1056	static AstNode ast_parse_loop_statement(ParseContext pc) {
1101		Token *inline_token = eat_token_if(pc, TokenIdKeywordInline);
	1057	TokenIndex inline_token = eat_token_if(pc, TokenIdKeywordInline);
1102	1058	AstNode *for_statement = ast_parse_for_statement(pc);
1103	1059	if (for_statement != nullptr) {
1104	1060	assert(for_statement->type == NodeTypeForExpr);
1105		for_statement->data.for_expr.is_inline = inline_token != nullptr;
	1061	for_statement->data.for_expr.is_inline = inline_token != 0;
1106	1062	return for_statement;
1107	1063	}
1108	1064
1109	1065	AstNode *while_statement = ast_parse_while_statement(pc);
1110	1066	if (while_statement != nullptr) {
1111	1067	assert(while_statement->type == NodeTypeWhileExpr);
1112		while_statement->data.while_expr.is_inline = inline_token != nullptr;
	1068	while_statement->data.while_expr.is_inline = inline_token != 0;
1113	1069	return while_statement;
1114	1070	}
1115	1071
1116		if (inline_token != nullptr)
	1072	if (inline_token != 0)
1117	1073	ast_invalid_token_error(pc, peek_token(pc));
1118	1074	return nullptr;
1119	1075	}
...	...	@@ -1134,12 +1090,12 @@ static AstNode ast_parse_for_statement(ParseContext pc) {
1134	1090	}
1135	1091
1136	1092	if (body == nullptr) {
1137		Token *tok = eat_token(pc);
1138		ast_error(pc, tok, "expected loop body, found '%s'", token_name(tok->id));
	1093	TokenIndex tok = eat_token(pc);
	1094	ast_error(pc, tok, "expected loop body, found '%s'", token_name(pc->token_ids[tok]));
1139	1095	}
1140	1096
1141	1097	AstNode *else_body = nullptr;
1142		if (eat_token_if(pc, TokenIdKeywordElse) != nullptr) {
	1098	if (eat_token_if(pc, TokenIdKeywordElse) != 0) {
1143	1099	else_body = ast_expect(pc, ast_parse_statement);
1144	1100	}
1145	1101
...	...	@@ -1168,13 +1124,13 @@ static AstNode ast_parse_while_statement(ParseContext pc) {
1168	1124	}
1169	1125
1170	1126	if (body == nullptr) {
1171		Token *tok = eat_token(pc);
1172		ast_error(pc, tok, "expected loop body, found '%s'", token_name(tok->id));
	1127	TokenIndex tok = eat_token(pc);
	1128	ast_error(pc, tok, "expected loop body, found '%s'", token_name(pc->token_ids[tok]));
1173	1129	}
1174	1130
1175		Token *err_payload = nullptr;
	1131	TokenIndex err_payload = 0;
1176	1132	AstNode *else_body = nullptr;
1177		if (eat_token_if(pc, TokenIdKeywordElse) != nullptr) {
	1133	if (eat_token_if(pc, TokenIdKeywordElse) != 0) {
1178	1134	err_payload = ast_parse_payload(pc);
1179	1135	else_body = ast_expect(pc, ast_parse_statement);
1180	1136	}
...	...	@@ -1184,7 +1140,7 @@ static AstNode ast_parse_while_statement(ParseContext pc) {
1184	1140
1185	1141	assert(res->type == NodeTypeWhileExpr);
1186	1142	res->data.while_expr.body = body;
1187		res->data.while_expr.err_symbol = token_buf(err_payload);
	1143	res->data.while_expr.err_symbol = token_buf(pc, err_payload);
1188	1144	res->data.while_expr.else_node = else_body;
1189	1145	return res;
1190	1146	}
...	...	@@ -1209,11 +1165,11 @@ static AstNode ast_parse_block_expr_statement(ParseContext pc) {
1209	1165
1210	1166	// BlockExpr <- BlockLabel? Block
1211	1167	static AstNode ast_parse_block_expr(ParseContext pc) {
1212		Token *label = ast_parse_block_label(pc);
1213		if (label != nullptr) {
	1168	TokenIndex label = ast_parse_block_label(pc);
	1169	if (label != 0) {
1214	1170	AstNode *res = ast_expect(pc, ast_parse_block);
1215	1171	assert(res->type == NodeTypeBlock);
1216		res->data.block.name = token_buf(label);
	1172	res->data.block.name = token_buf(pc, label);
1217	1173	return res;
1218	1174	}
1219	1175
...	...	@@ -1230,8 +1186,8 @@ static AstNode ast_parse_expr(ParseContext pc) {
1230	1186	return ast_parse_prefix_op_expr(
1231	1187	pc,
1232	1188	[](ParseContext *context) {
1233		Token *try_token = eat_token_if(context, TokenIdKeywordTry);
1234		if (try_token != nullptr) {
	1189	TokenIndex try_token = eat_token_if(context, TokenIdKeywordTry);
	1190	if (try_token != 0) {
1235	1191	AstNode *res = ast_create_node(context, NodeTypeReturnExpr, try_token);
1236	1192	res->data.return_expr.kind = ReturnKindError;
1237	1193	return res;
...	...	@@ -1318,72 +1274,72 @@ static AstNode ast_parse_primary_expr(ParseContext pc) {
1318	1274	if (if_expr != nullptr)
1319	1275	return if_expr;
1320	1276
1321		Token *break_token = eat_token_if(pc, TokenIdKeywordBreak);
1322		if (break_token != nullptr) {
1323		Token *label = ast_parse_break_label(pc);
	1277	TokenIndex break_token = eat_token_if(pc, TokenIdKeywordBreak);
	1278	if (break_token != 0) {
	1279	TokenIndex label = ast_parse_break_label(pc);
1324	1280	AstNode *expr = ast_parse_expr(pc);
1325	1281
1326	1282	AstNode *res = ast_create_node(pc, NodeTypeBreak, break_token);
1327		res->data.break_expr.name = token_buf(label);
	1283	res->data.break_expr.name = token_buf(pc, label);
1328	1284	res->data.break_expr.expr = expr;
1329	1285	return res;
1330	1286	}
1331	1287
1332		Token *comptime = eat_token_if(pc, TokenIdKeywordCompTime);
1333		if (comptime != nullptr) {
	1288	TokenIndex comptime = eat_token_if(pc, TokenIdKeywordCompTime);
	1289	if (comptime != 0) {
1334	1290	AstNode *expr = ast_expect(pc, ast_parse_expr);
1335	1291	AstNode *res = ast_create_node(pc, NodeTypeCompTime, comptime);
1336	1292	res->data.comptime_expr.expr = expr;
1337	1293	return res;
1338	1294	}
1339	1295
1340		Token *nosuspend = eat_token_if(pc, TokenIdKeywordNoSuspend);
1341		if (nosuspend != nullptr) {
	1296	TokenIndex nosuspend = eat_token_if(pc, TokenIdKeywordNoSuspend);
	1297	if (nosuspend != 0) {
1342	1298	AstNode *expr = ast_expect(pc, ast_parse_expr);
1343	1299	AstNode *res = ast_create_node(pc, NodeTypeNoSuspend, nosuspend);
1344	1300	res->data.nosuspend_expr.expr = expr;
1345	1301	return res;
1346	1302	}
1347	1303
1348		Token *continue_token = eat_token_if(pc, TokenIdKeywordContinue);
1349		if (continue_token != nullptr) {
1350		Token *label = ast_parse_break_label(pc);
	1304	TokenIndex continue_token = eat_token_if(pc, TokenIdKeywordContinue);
	1305	if (continue_token != 0) {
	1306	TokenIndex label = ast_parse_break_label(pc);
1351	1307	AstNode *res = ast_create_node(pc, NodeTypeContinue, continue_token);
1352		res->data.continue_expr.name = token_buf(label);
	1308	res->data.continue_expr.name = token_buf(pc, label);
1353	1309	return res;
1354	1310	}
1355	1311
1356		Token *resume = eat_token_if(pc, TokenIdKeywordResume);
1357		if (resume != nullptr) {
	1312	TokenIndex resume = eat_token_if(pc, TokenIdKeywordResume);
	1313	if (resume != 0) {
1358	1314	AstNode *expr = ast_expect(pc, ast_parse_expr);
1359	1315	AstNode *res = ast_create_node(pc, NodeTypeResume, resume);
1360	1316	res->data.resume_expr.expr = expr;
1361	1317	return res;
1362	1318	}
1363	1319
1364		Token *return_token = eat_token_if(pc, TokenIdKeywordReturn);
1365		if (return_token != nullptr) {
	1320	TokenIndex return_token = eat_token_if(pc, TokenIdKeywordReturn);
	1321	if (return_token != 0) {
1366	1322	AstNode *expr = ast_parse_expr(pc);
1367	1323	AstNode *res = ast_create_node(pc, NodeTypeReturnExpr, return_token);
1368	1324	res->data.return_expr.expr = expr;
1369	1325	return res;
1370	1326	}
1371	1327
1372		Token *label = ast_parse_block_label(pc);
	1328	TokenIndex label = ast_parse_block_label(pc);
1373	1329	AstNode *loop = ast_parse_loop_expr(pc);
1374	1330	if (loop != nullptr) {
1375	1331	switch (loop->type) {
1376	1332	case NodeTypeForExpr:
1377		loop->data.for_expr.name = token_buf(label);
	1333	loop->data.for_expr.name = token_buf(pc, label);
1378	1334	break;
1379	1335	case NodeTypeWhileExpr:
1380		loop->data.while_expr.name = token_buf(label);
	1336	loop->data.while_expr.name = token_buf(pc, label);
1381	1337	break;
1382	1338	default:
1383	1339	zig_unreachable();
1384	1340	}
1385	1341	return loop;
1386		} else if (label != nullptr) {
	1342	} else if (label != 0) {
1387	1343	// Restore the tokens that we eaten by ast_parse_block_label.
1388	1344	put_back_token(pc);
1389	1345	put_back_token(pc);
...	...	@@ -1407,8 +1363,8 @@ static AstNode ast_parse_if_expr(ParseContext pc) {
1407	1363
1408	1364	// Block <- LBRACE Statement* RBRACE
1409	1365	static AstNode ast_parse_block(ParseContext pc) {
1410		Token *lbrace = eat_token_if(pc, TokenIdLBrace);
1411		if (lbrace == nullptr)
	1366	TokenIndex lbrace = eat_token_if(pc, TokenIdLBrace);
	1367	if (lbrace == 0)
1412	1368	return nullptr;
1413	1369
1414	1370	ZigList<AstNode *> statements = {};
...	...	@@ -1462,8 +1418,8 @@ static AstNode ast_parse_curly_suffix_expr(ParseContext pc) {
1462	1418	// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE
1463	1419	// / LBRACE RBRACE
1464	1420	static AstNode ast_parse_init_list(ParseContext pc) {
1465		Token *lbrace = eat_token_if(pc, TokenIdLBrace);
1466		if (lbrace == nullptr)
	1421	TokenIndex lbrace = eat_token_if(pc, TokenIdLBrace);
	1422	if (lbrace == 0)
1467	1423	return nullptr;
1468	1424
1469	1425	AstNode *first = ast_parse_field_init(pc);
...	...	@@ -1472,7 +1428,7 @@ static AstNode ast_parse_init_list(ParseContext pc) {
1472	1428	res->data.container_init_expr.kind = ContainerInitKindStruct;
1473	1429	res->data.container_init_expr.entries.append(first);
1474	1430
1475		while (eat_token_if(pc, TokenIdComma) != nullptr) {
	1431	while (eat_token_if(pc, TokenIdComma) != 0) {
1476	1432	AstNode *field_init = ast_parse_field_init(pc);
1477	1433	if (field_init == nullptr)
1478	1434	break;
...	...	@@ -1490,7 +1446,7 @@ static AstNode ast_parse_init_list(ParseContext pc) {
1490	1446	if (first != nullptr) {
1491	1447	res->data.container_init_expr.entries.append(first);
1492	1448
1493		while (eat_token_if(pc, TokenIdComma) != nullptr) {
	1449	while (eat_token_if(pc, TokenIdComma) != 0) {
1494	1450	AstNode *expr = ast_parse_expr(pc);
1495	1451	if (expr == nullptr)
1496	1452	break;
...	...	@@ -1535,7 +1491,7 @@ static AstNode ast_parse_error_union_expr(ParseContext pc) {
1535	1491	// <- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments
1536	1492	// / PrimaryTypeExpr (SuffixOp / FnCallArguments)*
1537	1493	static AstNode ast_parse_suffix_expr(ParseContext pc) {
1538		Token *async_token = eat_token_if(pc, TokenIdKeywordAsync);
	1494	TokenIndex async_token = eat_token_if(pc, TokenIdKeywordAsync);
1539	1495	if (async_token) {
1540	1496	AstNode *child = ast_expect(pc, ast_parse_primary_type_expr);
1541	1497	while (true) {
...	...	@@ -1652,43 +1608,21 @@ static AstNode ast_parse_suffix_expr(ParseContext pc) {
1652	1608	// / STRINGLITERAL
1653	1609	// / SwitchExpr
1654	1610	static AstNode ast_parse_primary_type_expr(ParseContext pc) {
1655		// TODO: This is not in line with the grammar.
1656		// Because the prev stage 1 tokenizer does not parse
1657		// @[a-zA-Z_][a-zA-Z0-9_] as one token, it has to do a
1658		// hack, where it accepts '@' (IDENTIFIER / KEYWORD_export /
1659		// KEYWORD_extern).
1660		// I'd say that it's better if '@' is part of the builtin
1661		// identifier token.
1662		Token *at_sign = eat_token_if(pc, TokenIdAtSign);
1663		if (at_sign != nullptr) {
1664		Buf *name;
1665		Token *token;
1666		if ((token = eat_token_if(pc, TokenIdKeywordExport)) != nullptr) {
1667		name = buf_create_from_str("export");
1668		} else if ((token = eat_token_if(pc, TokenIdKeywordExtern)) != nullptr) {
1669		name = buf_create_from_str("extern");
1670		} else {
1671		token = expect_token(pc, TokenIdSymbol);
1672		name = token_buf(token);
1673		}
1674
	1611	TokenIndex builtin_tok = eat_token_if(pc, TokenIdBuiltin);
	1612	if (builtin_tok != 0) {
1675	1613	AstNode *res = ast_expect(pc, ast_parse_fn_call_arguments);
1676		AstNode *name_sym = ast_create_node(pc, NodeTypeSymbol, token);
1677		name_sym->data.symbol_expr.symbol = name;
	1614	AstNode *name_sym = ast_create_node(pc, NodeTypeIdentifier, builtin_tok);
1678	1615
1679	1616	assert(res->type == NodeTypeFnCallExpr);
1680		res->line = at_sign->start_line;
1681		res->column = at_sign->start_column;
	1617	res->main_token = builtin_tok;
1682	1618	res->data.fn_call_expr.fn_ref_expr = name_sym;
1683	1619	res->data.fn_call_expr.modifier = CallModifierBuiltin;
1684	1620	return res;
1685	1621	}
1686	1622
1687		Token *char_lit = eat_token_if(pc, TokenIdCharLiteral);
1688		if (char_lit != nullptr) {
1689		AstNode *res = ast_create_node(pc, NodeTypeCharLiteral, char_lit);
1690		res->data.char_literal.value = char_lit->data.char_lit.c;
1691		return res;
	1623	TokenIndex char_lit = eat_token_if(pc, TokenIdCharLiteral);
	1624	if (char_lit != 0) {
	1625	return ast_create_node(pc, NodeTypeCharLiteral, char_lit);
1692	1626	}
1693	1627
1694	1628	AstNode *container_decl = ast_parse_container_decl(pc);
...	...	@@ -1703,12 +1637,9 @@ static AstNode ast_parse_primary_type_expr(ParseContext pc) {
1703	1637	if (error_set_decl != nullptr)
1704	1638	return error_set_decl;
1705	1639
1706		Token *float_lit = eat_token_if(pc, TokenIdFloatLiteral);
1707		if (float_lit != nullptr) {
1708		AstNode *res = ast_create_node(pc, NodeTypeFloatLiteral, float_lit);
1709		res->data.float_literal.bigfloat = &float_lit->data.float_lit.bigfloat;
1710		res->data.float_literal.overflow = float_lit->data.float_lit.overflow;
1711		return res;
	1640	TokenIndex float_lit = eat_token_if(pc, TokenIdFloatLiteral);
	1641	if (float_lit != 0) {
	1642	return ast_create_node(pc, NodeTypeFloatLiteral, float_lit);
1712	1643	}
1713	1644
1714	1645	AstNode *fn_proto = ast_parse_fn_proto(pc);
...	...	@@ -1723,86 +1654,77 @@ static AstNode ast_parse_primary_type_expr(ParseContext pc) {
1723	1654	if (labeled_type_expr != nullptr)
1724	1655	return labeled_type_expr;
1725	1656
1726		Token *identifier = eat_token_if(pc, TokenIdSymbol);
1727		if (identifier != nullptr)
1728		return token_symbol(pc, identifier);
	1657	TokenIndex identifier = eat_token_if(pc, TokenIdIdentifier);
	1658	if (identifier != 0)
	1659	return token_identifier(pc, identifier);
1729	1660
1730	1661	AstNode *if_type_expr = ast_parse_if_type_expr(pc);
1731	1662	if (if_type_expr != nullptr)
1732	1663	return if_type_expr;
1733	1664
1734		Token *int_lit = eat_token_if(pc, TokenIdIntLiteral);
1735		if (int_lit != nullptr) {
1736		AstNode *res = ast_create_node(pc, NodeTypeIntLiteral, int_lit);
1737		res->data.int_literal.bigint = &int_lit->data.int_lit.bigint;
1738		return res;
	1665	TokenIndex int_lit = eat_token_if(pc, TokenIdIntLiteral);
	1666	if (int_lit != 0) {
	1667	return ast_create_node(pc, NodeTypeIntLiteral, int_lit);
1739	1668	}
1740	1669
1741		Token *comptime = eat_token_if(pc, TokenIdKeywordCompTime);
1742		if (comptime != nullptr) {
	1670	TokenIndex comptime = eat_token_if(pc, TokenIdKeywordCompTime);
	1671	if (comptime != 0) {
1743	1672	AstNode *expr = ast_expect(pc, ast_parse_type_expr);
1744	1673	AstNode *res = ast_create_node(pc, NodeTypeCompTime, comptime);
1745	1674	res->data.comptime_expr.expr = expr;
1746	1675	return res;
1747	1676	}
1748	1677
1749		Token *error = eat_token_if(pc, TokenIdKeywordError);
1750		if (error != nullptr) {
1751		Token *dot = expect_token(pc, TokenIdDot);
1752		Token *name = expect_token(pc, TokenIdSymbol);
	1678	TokenIndex error = eat_token_if(pc, TokenIdKeywordError);
	1679	if (error != 0) {
	1680	TokenIndex dot = expect_token(pc, TokenIdDot);
	1681	TokenIndex name = expect_token(pc, TokenIdIdentifier);
1753	1682	AstNode *left = ast_create_node(pc, NodeTypeErrorType, error);
1754	1683	AstNode *res = ast_create_node(pc, NodeTypeFieldAccessExpr, dot);
1755	1684	res->data.field_access_expr.struct_expr = left;
1756		res->data.field_access_expr.field_name = token_buf(name);
	1685	res->data.field_access_expr.field_name = token_buf(pc, name);
1757	1686	return res;
1758	1687	}
1759	1688
1760		Token *false_token = eat_token_if(pc, TokenIdKeywordFalse);
1761		if (false_token != nullptr) {
	1689	TokenIndex false_token = eat_token_if(pc, TokenIdKeywordFalse);
	1690	if (false_token != 0) {
1762	1691	AstNode *res = ast_create_node(pc, NodeTypeBoolLiteral, false_token);
1763	1692	res->data.bool_literal.value = false;
1764	1693	return res;
1765	1694	}
1766	1695
1767		Token *null = eat_token_if(pc, TokenIdKeywordNull);
1768		if (null != nullptr)
	1696	TokenIndex null = eat_token_if(pc, TokenIdKeywordNull);
	1697	if (null != 0)
1769	1698	return ast_create_node(pc, NodeTypeNullLiteral, null);
1770	1699
1771		Token *anyframe = eat_token_if(pc, TokenIdKeywordAnyFrame);
1772		if (anyframe != nullptr)
	1700	TokenIndex anyframe = eat_token_if(pc, TokenIdKeywordAnyFrame);
	1701	if (anyframe != 0)
1773	1702	return ast_create_node(pc, NodeTypeAnyFrameType, anyframe);
1774	1703
1775		Token *true_token = eat_token_if(pc, TokenIdKeywordTrue);
1776		if (true_token != nullptr) {
	1704	TokenIndex true_token = eat_token_if(pc, TokenIdKeywordTrue);
	1705	if (true_token != 0) {
1777	1706	AstNode *res = ast_create_node(pc, NodeTypeBoolLiteral, true_token);
1778	1707	res->data.bool_literal.value = true;
1779	1708	return res;
1780	1709	}
1781	1710
1782		Token *undefined = eat_token_if(pc, TokenIdKeywordUndefined);
1783		if (undefined != nullptr)
	1711	TokenIndex undefined = eat_token_if(pc, TokenIdKeywordUndefined);
	1712	if (undefined != 0)
1784	1713	return ast_create_node(pc, NodeTypeUndefinedLiteral, undefined);
1785	1714
1786		Token *unreachable = eat_token_if(pc, TokenIdKeywordUnreachable);
1787		if (unreachable != nullptr)
	1715	TokenIndex unreachable = eat_token_if(pc, TokenIdKeywordUnreachable);
	1716	if (unreachable != 0)
1788	1717	return ast_create_node(pc, NodeTypeUnreachable, unreachable);
1789	1718
1790	1719
1791		Buf *string_buf;
1792		Token *string_lit = eat_token_if(pc, TokenIdStringLiteral);
1793		if (string_lit != nullptr) {
1794		string_buf = token_buf(string_lit);
1795		} else {
1796		Buf multiline_string_buf = BUF_INIT;
1797		string_lit = ast_parse_multiline_string_literal(pc, &multiline_string_buf);
1798		if (string_lit != nullptr) {
1799		string_buf = buf_create_from_buf(&multiline_string_buf);
1800		}
	1720	TokenIndex string_lit = eat_token_if(pc, TokenIdStringLiteral);
	1721	if (string_lit != 0) {
	1722	return ast_create_node(pc, NodeTypeStringLiteral, string_lit);
1801	1723	}
1802		if (string_lit != nullptr) {
1803		AstNode *res = ast_create_node(pc, NodeTypeStringLiteral, string_lit);
1804		res->data.string_literal.buf = string_buf;
1805		return res;
	1724
	1725	TokenIndex multiline_str_lit = ast_parse_multi_tok(pc, TokenIdMultilineStringLiteralLine);
	1726	if (multiline_str_lit != 0) {
	1727	return ast_create_node(pc, NodeTypeStringLiteral, multiline_str_lit);
1806	1728	}
1807	1729
1808	1730	AstNode *switch_expr = ast_parse_switch_expr(pc);
...	...	@@ -1814,22 +1736,21 @@ static AstNode ast_parse_primary_type_expr(ParseContext pc) {
1814	1736
1815	1737	// ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto
1816	1738	static AstNode ast_parse_container_decl(ParseContext pc) {
1817		Token *layout_token = eat_token_if(pc, TokenIdKeywordExtern);
1818		if (layout_token == nullptr)
	1739	TokenIndex layout_token = eat_token_if(pc, TokenIdKeywordExtern);
	1740	if (layout_token == 0)
1819	1741	layout_token = eat_token_if(pc, TokenIdKeywordPacked);
1820	1742
1821	1743	AstNode *res = ast_parse_container_decl_auto(pc);
1822	1744	if (res == nullptr) {
1823		if (layout_token != nullptr)
	1745	if (layout_token != 0)
1824	1746	put_back_token(pc);
1825	1747	return nullptr;
1826	1748	}
1827	1749
1828	1750	assert(res->type == NodeTypeContainerDecl);
1829		if (layout_token != nullptr) {
1830		res->line = layout_token->start_line;
1831		res->column = layout_token->start_column;
1832		res->data.container_decl.layout = layout_token->id == TokenIdKeywordExtern
	1751	if (layout_token != 0) {
	1752	res->main_token = layout_token;
	1753	res->data.container_decl.layout = pc->token_ids[layout_token] == TokenIdKeywordExtern
1833	1754	? ContainerLayoutExtern
1834	1755	: ContainerLayoutPacked;
1835	1756	}
...	...	@@ -1838,28 +1759,27 @@ static AstNode ast_parse_container_decl(ParseContext pc) {
1838	1759
1839	1760	// ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE
1840	1761	static AstNode ast_parse_error_set_decl(ParseContext pc) {
1841		Token *first = eat_token_if(pc, TokenIdKeywordError);
1842		if (first == nullptr)
	1762	TokenIndex first = eat_token_if(pc, TokenIdKeywordError);
	1763	if (first == 0)
1843	1764	return nullptr;
1844		if (eat_token_if(pc, TokenIdLBrace) == nullptr) {
	1765	if (eat_token_if(pc, TokenIdLBrace) == 0) {
1845	1766	put_back_token(pc);
1846	1767	return nullptr;
1847	1768	}
1848	1769
1849	1770	ZigList<AstNode > decls = ast_parse_list<AstNode>(pc, TokenIdComma, [](ParseContext context) {
1850		Buf doc_comment_buf = BUF_INIT;
1851		Token *doc_token = ast_parse_doc_comments(context, &doc_comment_buf);
1852		Token *ident = eat_token_if(context, TokenIdSymbol);
1853		if (ident == nullptr)
	1771	TokenIndex doc_token = ast_parse_doc_comments(context);
	1772	TokenIndex ident = eat_token_if(context, TokenIdIdentifier);
	1773	if (ident == 0)
1854	1774	return (AstNode*)nullptr;
1855	1775
1856		AstNode *symbol_node = token_symbol(context, ident);
1857		if (doc_token == nullptr)
	1776	AstNode *symbol_node = token_identifier(context, ident);
	1777	if (doc_token == 0)
1858	1778	return symbol_node;
1859	1779
1860	1780	AstNode *field_node = ast_create_node(context, NodeTypeErrorSetField, doc_token);
1861	1781	field_node->data.err_set_field.field_name = symbol_node;
1862		field_node->data.err_set_field.doc_comments = doc_comment_buf;
	1782	field_node->data.err_set_field.doc_comments = doc_token;
1863	1783	return field_node;
1864	1784	});
1865	1785	expect_token(pc, TokenIdRBrace);
...	...	@@ -1871,8 +1791,8 @@ static AstNode ast_parse_error_set_decl(ParseContext pc) {
1871	1791
1872	1792	// GroupedExpr <- LPAREN Expr RPAREN
1873	1793	static AstNode ast_parse_grouped_expr(ParseContext pc) {
1874		Token *lparen = eat_token_if(pc, TokenIdLParen);
1875		if (lparen == nullptr)
	1794	TokenIndex lparen = eat_token_if(pc, TokenIdLParen);
	1795	if (lparen == 0)
1876	1796	return nullptr;
1877	1797
1878	1798	AstNode *expr = ast_expect(pc, ast_parse_expr);
...	...	@@ -1892,12 +1812,12 @@ static AstNode ast_parse_if_type_expr(ParseContext pc) {
1892	1812	// <- BlockLabel Block
1893	1813	// / BlockLabel? LoopTypeExpr
1894	1814	static AstNode ast_parse_labeled_type_expr(ParseContext pc) {
1895		Token *label = ast_parse_block_label(pc);
1896		if (label != nullptr) {
	1815	TokenIndex label = ast_parse_block_label(pc);
	1816	if (label != 0) {
1897	1817	AstNode *block = ast_parse_block(pc);
1898	1818	if (block != nullptr) {
1899	1819	assert(block->type == NodeTypeBlock);
1900		block->data.block.name = token_buf(label);
	1820	block->data.block.name = token_buf(pc, label);
1901	1821	return block;
1902	1822	}
1903	1823	}
...	...	@@ -1906,10 +1826,10 @@ static AstNode ast_parse_labeled_type_expr(ParseContext pc) {
1906	1826	if (loop != nullptr) {
1907	1827	switch (loop->type) {
1908	1828	case NodeTypeForExpr:
1909		loop->data.for_expr.name = token_buf(label);
	1829	loop->data.for_expr.name = token_buf(pc, label);
1910	1830	break;
1911	1831	case NodeTypeWhileExpr:
1912		loop->data.while_expr.name = token_buf(label);
	1832	loop->data.while_expr.name = token_buf(pc, label);
1913	1833	break;
1914	1834	default:
1915	1835	zig_unreachable();
...	...	@@ -1917,7 +1837,7 @@ static AstNode ast_parse_labeled_type_expr(ParseContext pc) {
1917	1837	return loop;
1918	1838	}
1919	1839
1920		if (label != nullptr) {
	1840	if (label != 0) {
1921	1841	put_back_token(pc);
1922	1842	put_back_token(pc);
1923	1843	}
...	...	@@ -1945,8 +1865,8 @@ static AstNode ast_parse_while_type_expr(ParseContext pc) {
1945	1865
1946	1866	// SwitchExpr <- KEYWORD_switch LPAREN Expr RPAREN LBRACE SwitchProngList RBRACE
1947	1867	static AstNode ast_parse_switch_expr(ParseContext pc) {
1948		Token *switch_token = eat_token_if(pc, TokenIdKeywordSwitch);
1949		if (switch_token == nullptr)
	1868	TokenIndex switch_token = eat_token_if(pc, TokenIdKeywordSwitch);
	1869	if (switch_token == 0)
1950	1870	return nullptr;
1951	1871
1952	1872	expect_token(pc, TokenIdLParen);
...	...	@@ -1964,11 +1884,11 @@ static AstNode ast_parse_switch_expr(ParseContext pc) {
1964	1884
1965	1885	// AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN STRINGLITERAL AsmOutput? RPAREN
1966	1886	static AstNode ast_parse_asm_expr(ParseContext pc) {
1967		Token *asm_token = eat_token_if(pc, TokenIdKeywordAsm);
1968		if (asm_token == nullptr)
	1887	TokenIndex asm_token = eat_token_if(pc, TokenIdKeywordAsm);
	1888	if (asm_token == 0)
1969	1889	return nullptr;
1970	1890
1971		Token *volatile_token = eat_token_if(pc, TokenIdKeywordVolatile);
	1891	TokenIndex volatile_token = eat_token_if(pc, TokenIdKeywordVolatile);
1972	1892	expect_token(pc, TokenIdLParen);
1973	1893	AstNode *asm_template = ast_expect(pc, ast_parse_expr);
1974	1894	AstNode *res = ast_parse_asm_output(pc);
...	...	@@ -1976,25 +1896,21 @@ static AstNode ast_parse_asm_expr(ParseContext pc) {
1976	1896	res = ast_create_node_no_line_info(pc, NodeTypeAsmExpr);
1977	1897	expect_token(pc, TokenIdRParen);
1978	1898
1979		res->line = asm_token->start_line;
1980		res->column = asm_token->start_column;
	1899	res->main_token = asm_token;
1981	1900	res->data.asm_expr.volatile_token = volatile_token;
1982	1901	res->data.asm_expr.asm_template = asm_template;
1983	1902	return res;
1984	1903	}
1985	1904
1986	1905	static AstNode ast_parse_anon_lit(ParseContext pc) {
1987		Token *period = eat_token_if(pc, TokenIdDot);
1988		if (period == nullptr)
	1906	TokenIndex period = eat_token_if(pc, TokenIdDot);
	1907	if (period == 0)
1989	1908	return nullptr;
1990	1909
1991	1910	// anon enum literal
1992		Token *identifier = eat_token_if(pc, TokenIdSymbol);
1993		if (identifier != nullptr) {
1994		AstNode *res = ast_create_node(pc, NodeTypeEnumLiteral, period);
1995		res->data.enum_literal.period = period;
1996		res->data.enum_literal.identifier = identifier;
1997		return res;
	1911	TokenIndex identifier = eat_token_if(pc, TokenIdIdentifier);
	1912	if (identifier != 0) {
	1913	return ast_create_node(pc, NodeTypeEnumLiteral, period);
1998	1914	}
1999	1915
2000	1916	// anon container literal
...	...	@@ -2007,7 +1923,7 @@ static AstNode ast_parse_anon_lit(ParseContext pc) {
2007	1923
2008	1924	// AsmOutput <- COLON AsmOutputList AsmInput?
2009	1925	static AstNode ast_parse_asm_output(ParseContext pc) {
2010		if (eat_token_if(pc, TokenIdColon) == nullptr)
	1926	if (eat_token_if(pc, TokenIdColon) == 0)
2011	1927	return nullptr;
2012	1928
2013	1929	ZigList<AsmOutput *> output_list = ast_parse_list(pc, TokenIdComma, ast_parse_asm_output_item);
...	...	@@ -2021,20 +1937,20 @@ static AstNode ast_parse_asm_output(ParseContext pc) {
2021	1937
2022	1938	// AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN
2023	1939	static AsmOutput ast_parse_asm_output_item(ParseContext pc) {
2024		if (eat_token_if(pc, TokenIdLBracket) == nullptr)
	1940	if (eat_token_if(pc, TokenIdLBracket) == 0)
2025	1941	return nullptr;
2026	1942
2027		Token *sym_name = expect_token(pc, TokenIdSymbol);
	1943	TokenIndex sym_name = expect_token(pc, TokenIdIdentifier);
2028	1944	expect_token(pc, TokenIdRBracket);
2029	1945
2030		Token *str = eat_token_if(pc, TokenIdMultilineStringLiteral);
2031		if (str == nullptr)
	1946	TokenIndex str = ast_parse_multi_tok(pc, TokenIdMultilineStringLiteralLine);
	1947	if (str == 0)
2032	1948	str = expect_token(pc, TokenIdStringLiteral);
2033	1949	expect_token(pc, TokenIdLParen);
2034	1950
2035		Token *var_name = eat_token_if(pc, TokenIdSymbol);
	1951	TokenIndex var_name = eat_token_if(pc, TokenIdIdentifier);
2036	1952	AstNode *return_type = nullptr;
2037		if (var_name == nullptr) {
	1953	if (var_name == 0) {
2038	1954	expect_token(pc, TokenIdArrow);
2039	1955	return_type = ast_expect(pc, ast_parse_type_expr);
2040	1956	}
...	...	@@ -2042,16 +1958,16 @@ static AsmOutput ast_parse_asm_output_item(ParseContext pc) {
2042	1958	expect_token(pc, TokenIdRParen);
2043	1959
2044	1960	AsmOutput *res = heap::c_allocator.create<AsmOutput>();
2045		res->asm_symbolic_name = token_buf(sym_name);
2046		res->constraint = token_buf(str);
2047		res->variable_name = token_buf(var_name);
	1961	res->asm_symbolic_name = token_buf(pc, sym_name);
	1962	res->constraint = token_buf(pc, str);
	1963	res->variable_name = token_buf(pc, var_name);
2048	1964	res->return_type = return_type;
2049	1965	return res;
2050	1966	}
2051	1967
2052	1968	// AsmInput <- COLON AsmInputList AsmClobbers?
2053	1969	static AstNode ast_parse_asm_input(ParseContext pc) {
2054		if (eat_token_if(pc, TokenIdColon) == nullptr)
	1970	if (eat_token_if(pc, TokenIdColon) == 0)
2055	1971	return nullptr;
2056	1972
2057	1973	ZigList<AsmInput *> input_list = ast_parse_list(pc, TokenIdComma, ast_parse_asm_input_item);
...	...	@@ -2065,37 +1981,35 @@ static AstNode ast_parse_asm_input(ParseContext pc) {
2065	1981
2066	1982	// AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN Expr RPAREN
2067	1983	static AsmInput ast_parse_asm_input_item(ParseContext pc) {
2068		if (eat_token_if(pc, TokenIdLBracket) == nullptr)
	1984	if (eat_token_if(pc, TokenIdLBracket) == 0)
2069	1985	return nullptr;
2070	1986
2071		Token *sym_name = expect_token(pc, TokenIdSymbol);
	1987	TokenIndex sym_name = expect_token(pc, TokenIdIdentifier);
2072	1988	expect_token(pc, TokenIdRBracket);
2073	1989
2074		Token *constraint = eat_token_if(pc, TokenIdMultilineStringLiteral);
2075		if (constraint == nullptr)
2076		constraint = expect_token(pc, TokenIdStringLiteral);
	1990	TokenIndex constraint = expect_token(pc, TokenIdStringLiteral);
2077	1991	expect_token(pc, TokenIdLParen);
2078	1992	AstNode *expr = ast_expect(pc, ast_parse_expr);
2079	1993	expect_token(pc, TokenIdRParen);
2080	1994
2081	1995	AsmInput *res = heap::c_allocator.create<AsmInput>();
2082		res->asm_symbolic_name = token_buf(sym_name);
2083		res->constraint = token_buf(constraint);
	1996	res->asm_symbolic_name = token_buf(pc, sym_name);
	1997	res->constraint = token_buf(pc, constraint);
2084	1998	res->expr = expr;
2085	1999	return res;
2086	2000	}
2087	2001
2088	2002	// AsmClobbers <- COLON StringList
2089	2003	static AstNode ast_parse_asm_clobbers(ParseContext pc) {
2090		if (eat_token_if(pc, TokenIdColon) == nullptr)
	2004	if (eat_token_if(pc, TokenIdColon) == 0)
2091	2005	return nullptr;
2092	2006
2093	2007	ZigList<Buf > clobber_list = ast_parse_list<Buf>(pc, TokenIdComma, [](ParseContext context) {
2094		Token *str = eat_token_if(context, TokenIdStringLiteral);
2095		if (str == nullptr)
2096		str = eat_token_if(context, TokenIdMultilineStringLiteral);
2097		if (str != nullptr)
2098		return token_buf(str);
	2008	TokenIndex str = eat_token_if(context, TokenIdStringLiteral);
	2009	if (str == 0)
	2010	str = ast_parse_multi_tok(context, TokenIdMultilineStringLiteralLine);
	2011	if (str != 0)
	2012	return token_buf(context, str);
2099	2013	return (Buf*)nullptr;
2100	2014	});
2101	2015
...	...	@@ -2105,24 +2019,24 @@ static AstNode ast_parse_asm_clobbers(ParseContext pc) {
2105	2019	}
2106	2020
2107	2021	// BreakLabel <- COLON IDENTIFIER
2108		static Token ast_parse_break_label(ParseContext pc) {
2109		if (eat_token_if(pc, TokenIdColon) == nullptr)
2110		return nullptr;
	2022	static TokenIndex ast_parse_break_label(ParseContext *pc) {
	2023	if (eat_token_if(pc, TokenIdColon) == 0)
	2024	return 0;
2111	2025
2112		return expect_token(pc, TokenIdSymbol);
	2026	return expect_token(pc, TokenIdIdentifier);
2113	2027	}
2114	2028
2115	2029	// BlockLabel <- IDENTIFIER COLON
2116		static Token ast_parse_block_label(ParseContext pc) {
2117		Token *ident = eat_token_if(pc, TokenIdSymbol);
2118		if (ident == nullptr)
2119		return nullptr;
	2030	static TokenIndex ast_parse_block_label(ParseContext *pc) {
	2031	TokenIndex ident = eat_token_if(pc, TokenIdIdentifier);
	2032	if (ident == 0)
	2033	return 0;
2120	2034
2121	2035	// We do 2 token lookahead here, as we don't want to error when
2122	2036	// parsing identifiers.
2123		if (eat_token_if(pc, TokenIdColon) == nullptr) {
	2037	if (eat_token_if(pc, TokenIdColon) == 0) {
2124	2038	put_back_token(pc);
2125		return nullptr;
	2039	return 0;
2126	2040	}
2127	2041
2128	2042	return ident;
...	...	@@ -2130,17 +2044,17 @@ static Token ast_parse_block_label(ParseContext pc) {
2130	2044
2131	2045	// FieldInit <- DOT IDENTIFIER EQUAL Expr
2132	2046	static AstNode ast_parse_field_init(ParseContext pc) {
2133		Token *first = eat_token_if(pc, TokenIdDot);
2134		if (first == nullptr)
	2047	TokenIndex first = eat_token_if(pc, TokenIdDot);
	2048	if (first == 0)
2135	2049	return nullptr;
2136	2050
2137		Token *name = eat_token_if(pc, TokenIdSymbol);
2138		if (name == nullptr) {
	2051	TokenIndex name = eat_token_if(pc, TokenIdIdentifier);
	2052	if (name == 0) {
2139	2053	// Because of anon literals ".{" is also valid.
2140	2054	put_back_token(pc);
2141	2055	return nullptr;
2142	2056	}
2143		if (eat_token_if(pc, TokenIdEq) == nullptr) {
	2057	if (eat_token_if(pc, TokenIdEq) == 0) {
2144	2058	// Because ".Name" can also be intepreted as an enum literal, we should put back
2145	2059	// those two tokens again so that the parser can try to parse them as the enum
2146	2060	// literal later.
...	...	@@ -2151,15 +2065,15 @@ static AstNode ast_parse_field_init(ParseContext pc) {
2151	2065	AstNode *expr = ast_expect(pc, ast_parse_expr);
2152	2066
2153	2067	AstNode *res = ast_create_node(pc, NodeTypeStructValueField, first);
2154		res->data.struct_val_field.name = token_buf(name);
	2068	res->data.struct_val_field.name = token_buf(pc, name);
2155	2069	res->data.struct_val_field.expr = expr;
2156	2070	return res;
2157	2071	}
2158	2072
2159	2073	// WhileContinueExpr <- COLON LPAREN AssignExpr RPAREN
2160	2074	static AstNode ast_parse_while_continue_expr(ParseContext pc) {
2161		Token *first = eat_token_if(pc, TokenIdColon);
2162		if (first == nullptr)
	2075	TokenIndex first = eat_token_if(pc, TokenIdColon);
	2076	if (first == 0)
2163	2077	return nullptr;
2164	2078
2165	2079	expect_token(pc, TokenIdLParen);
...	...	@@ -2170,8 +2084,8 @@ static AstNode ast_parse_while_continue_expr(ParseContext pc) {
2170	2084
2171	2085	// LinkSection <- KEYWORD_linksection LPAREN Expr RPAREN
2172	2086	static AstNode ast_parse_link_section(ParseContext pc) {
2173		Token *first = eat_token_if(pc, TokenIdKeywordLinkSection);
2174		if (first == nullptr)
	2087	TokenIndex first = eat_token_if(pc, TokenIdKeywordLinkSection);
	2088	if (first == 0)
2175	2089	return nullptr;
2176	2090
2177	2091	expect_token(pc, TokenIdLParen);
...	...	@@ -2182,8 +2096,8 @@ static AstNode ast_parse_link_section(ParseContext pc) {
2182	2096
2183	2097	// CallConv <- KEYWORD_callconv LPAREN Expr RPAREN
2184	2098	static AstNode ast_parse_callconv(ParseContext pc) {
2185		Token *first = eat_token_if(pc, TokenIdKeywordCallconv);
2186		if (first == nullptr)
	2099	TokenIndex first = eat_token_if(pc, TokenIdKeywordCallconv);
	2100	if (first == 0)
2187	2101	return nullptr;
2188	2102
2189	2103	expect_token(pc, TokenIdLParen);
...	...	@@ -2194,28 +2108,27 @@ static AstNode ast_parse_callconv(ParseContext pc) {
2194	2108
2195	2109	// ParamDecl <- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType
2196	2110	static AstNode ast_parse_param_decl(ParseContext pc) {
2197		Buf doc_comments = BUF_INIT;
2198		ast_parse_doc_comments(pc, &doc_comments);
	2111	TokenIndex first_doc_comment = ast_parse_doc_comments(pc);
2199	2112
2200		Token *first = eat_token_if(pc, TokenIdKeywordNoAlias);
2201		if (first == nullptr)
	2113	TokenIndex first = eat_token_if(pc, TokenIdKeywordNoAlias);
	2114	if (first == 0)
2202	2115	first = eat_token_if(pc, TokenIdKeywordCompTime);
2203	2116
2204		Token *name = eat_token_if(pc, TokenIdSymbol);
2205		if (name != nullptr) {
2206		if (eat_token_if(pc, TokenIdColon) != nullptr) {
2207		if (first == nullptr)
	2117	TokenIndex name = eat_token_if(pc, TokenIdIdentifier);
	2118	if (name != 0) {
	2119	if (eat_token_if(pc, TokenIdColon) != 0) {
	2120	if (first == 0)
2208	2121	first = name;
2209	2122	} else {
2210	2123	// We put back the ident, so it can be parsed as a ParamType
2211	2124	// later.
2212	2125	put_back_token(pc);
2213		name = nullptr;
	2126	name = 0;
2214	2127	}
2215	2128	}
2216	2129
2217	2130	AstNode *res;
2218		if (first == nullptr) {
	2131	if (first == 0) {
2219	2132	first = peek_token(pc);
2220	2133	res = ast_parse_param_type(pc);
2221	2134	} else {
...	...	@@ -2226,12 +2139,11 @@ static AstNode ast_parse_param_decl(ParseContext pc) {
2226	2139	return nullptr;
2227	2140
2228	2141	assert(res->type == NodeTypeParamDecl);
2229		res->line = first->start_line;
2230		res->column = first->start_column;
2231		res->data.param_decl.name = token_buf(name);
2232		res->data.param_decl.doc_comments = doc_comments;
2233		res->data.param_decl.is_noalias = first->id == TokenIdKeywordNoAlias;
2234		res->data.param_decl.is_comptime = first->id == TokenIdKeywordCompTime;
	2142	res->main_token = first;
	2143	res->data.param_decl.name = token_buf(pc, name);
	2144	res->data.param_decl.doc_comments = first_doc_comment;
	2145	res->data.param_decl.is_noalias = pc->token_ids[first] == TokenIdKeywordNoAlias;
	2146	res->data.param_decl.is_comptime = pc->token_ids[first] == TokenIdKeywordCompTime;
2235	2147	return res;
2236	2148	}
2237	2149
...	...	@@ -2240,15 +2152,15 @@ static AstNode ast_parse_param_decl(ParseContext pc) {
2240	2152	// / DOT3
2241	2153	// / TypeExpr
2242	2154	static AstNode ast_parse_param_type(ParseContext pc) {
2243		Token *anytype_token = eat_token_if(pc, TokenIdKeywordAnyType);
2244		if (anytype_token != nullptr) {
	2155	TokenIndex anytype_token = eat_token_if(pc, TokenIdKeywordAnyType);
	2156	if (anytype_token != 0) {
2245	2157	AstNode *res = ast_create_node(pc, NodeTypeParamDecl, anytype_token);
2246	2158	res->data.param_decl.anytype_token = anytype_token;
2247	2159	return res;
2248	2160	}
2249	2161
2250		Token *dots = eat_token_if(pc, TokenIdEllipsis3);
2251		if (dots != nullptr) {
	2162	TokenIndex dots = eat_token_if(pc, TokenIdEllipsis3);
	2163	if (dots != 0) {
2252	2164	AstNode *res = ast_create_node(pc, NodeTypeParamDecl, dots);
2253	2165	res->data.param_decl.is_var_args = true;
2254	2166	return res;
...	...	@@ -2266,8 +2178,8 @@ static AstNode ast_parse_param_type(ParseContext pc) {
2266	2178
2267	2179	// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload?
2268	2180	static AstNode ast_parse_if_prefix(ParseContext pc) {
2269		Token *first = eat_token_if(pc, TokenIdKeywordIf);
2270		if (first == nullptr)
	2181	TokenIndex first = eat_token_if(pc, TokenIdKeywordIf);
	2182	if (first == 0)
2271	2183	return nullptr;
2272	2184
2273	2185	expect_token(pc, TokenIdLParen);
...	...	@@ -2279,16 +2191,16 @@ static AstNode ast_parse_if_prefix(ParseContext pc) {
2279	2191	AstNode *res = ast_create_node(pc, NodeTypeIfOptional, first);
2280	2192	res->data.test_expr.target_node = condition;
2281	2193	if (opt_payload.unwrap(&payload)) {
2282		res->data.test_expr.var_symbol = token_buf(payload.payload);
2283		res->data.test_expr.var_is_ptr = payload.asterisk != nullptr;
	2194	res->data.test_expr.var_symbol = token_buf(pc, payload.payload);
	2195	res->data.test_expr.var_is_ptr = payload.asterisk != 0;
2284	2196	}
2285	2197	return res;
2286	2198	}
2287	2199
2288	2200	// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
2289	2201	static AstNode ast_parse_while_prefix(ParseContext pc) {
2290		Token *while_token = eat_token_if(pc, TokenIdKeywordWhile);
2291		if (while_token == nullptr)
	2202	TokenIndex while_token = eat_token_if(pc, TokenIdKeywordWhile);
	2203	if (while_token == 0)
2292	2204	return nullptr;
2293	2205
2294	2206	expect_token(pc, TokenIdLParen);
...	...	@@ -2302,8 +2214,8 @@ static AstNode ast_parse_while_prefix(ParseContext pc) {
2302	2214	res->data.while_expr.condition = condition;
2303	2215	res->data.while_expr.continue_expr = continue_expr;
2304	2216	if (opt_payload.unwrap(&payload)) {
2305		res->data.while_expr.var_symbol = token_buf(payload.payload);
2306		res->data.while_expr.var_is_ptr = payload.asterisk != nullptr;
	2217	res->data.while_expr.var_symbol = token_buf(pc, payload.payload);
	2218	res->data.while_expr.var_is_ptr = payload.asterisk != 0;
2307	2219	}
2308	2220
2309	2221	return res;
...	...	@@ -2311,8 +2223,8 @@ static AstNode ast_parse_while_prefix(ParseContext pc) {
2311	2223
2312	2224	// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload
2313	2225	static AstNode ast_parse_for_prefix(ParseContext pc) {
2314		Token *for_token = eat_token_if(pc, TokenIdKeywordFor);
2315		if (for_token == nullptr)
	2226	TokenIndex for_token = eat_token_if(pc, TokenIdKeywordFor);
	2227	if (for_token == 0)
2316	2228	return nullptr;
2317	2229
2318	2230	expect_token(pc, TokenIdLParen);
...	...	@@ -2324,31 +2236,31 @@ static AstNode ast_parse_for_prefix(ParseContext pc) {
2324	2236
2325	2237	AstNode *res = ast_create_node(pc, NodeTypeForExpr, for_token);
2326	2238	res->data.for_expr.array_expr = array_expr;
2327		res->data.for_expr.elem_node = token_symbol(pc, payload.payload);
2328		res->data.for_expr.elem_is_ptr = payload.asterisk != nullptr;
2329		if (payload.index != nullptr)
2330		res->data.for_expr.index_node = token_symbol(pc, payload.index);
	2239	res->data.for_expr.elem_node = token_identifier(pc, payload.payload);
	2240	res->data.for_expr.elem_is_ptr = payload.asterisk != 0;
	2241	if (payload.index != 0)
	2242	res->data.for_expr.index_node = token_identifier(pc, payload.index);
2331	2243
2332	2244	return res;
2333	2245	}
2334	2246
2335	2247	// Payload <- PIPE IDENTIFIER PIPE
2336		static Token ast_parse_payload(ParseContext pc) {
2337		if (eat_token_if(pc, TokenIdBinOr) == nullptr)
2338		return nullptr;
	2248	static TokenIndex ast_parse_payload(ParseContext *pc) {
	2249	if (eat_token_if(pc, TokenIdBinOr) == 0)
	2250	return 0;
2339	2251
2340		Token *res = expect_token(pc, TokenIdSymbol);
	2252	TokenIndex res = expect_token(pc, TokenIdIdentifier);
2341	2253	expect_token(pc, TokenIdBinOr);
2342	2254	return res;
2343	2255	}
2344	2256
2345	2257	// PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE
2346	2258	static Optional<PtrPayload> ast_parse_ptr_payload(ParseContext *pc) {
2347		if (eat_token_if(pc, TokenIdBinOr) == nullptr)
	2259	if (eat_token_if(pc, TokenIdBinOr) == 0)
2348	2260	return Optional<PtrPayload>::none();
2349	2261
2350		Token *asterisk = eat_token_if(pc, TokenIdStar);
2351		Token *payload = expect_token(pc, TokenIdSymbol);
	2262	TokenIndex asterisk = eat_token_if(pc, TokenIdStar);
	2263	TokenIndex payload = expect_token(pc, TokenIdIdentifier);
2352	2264	expect_token(pc, TokenIdBinOr);
2353	2265
2354	2266	PtrPayload res;
...	...	@@ -2359,14 +2271,14 @@ static Optional<PtrPayload> ast_parse_ptr_payload(ParseContext *pc) {
2359	2271
2360	2272	// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE
2361	2273	static Optional<PtrIndexPayload> ast_parse_ptr_index_payload(ParseContext *pc) {
2362		if (eat_token_if(pc, TokenIdBinOr) == nullptr)
	2274	if (eat_token_if(pc, TokenIdBinOr) == 0)
2363	2275	return Optional<PtrIndexPayload>::none();
2364	2276
2365		Token *asterisk = eat_token_if(pc, TokenIdStar);
2366		Token *payload = expect_token(pc, TokenIdSymbol);
2367		Token *index = nullptr;
2368		if (eat_token_if(pc, TokenIdComma) != nullptr)
2369		index = expect_token(pc, TokenIdSymbol);
	2277	TokenIndex asterisk = eat_token_if(pc, TokenIdStar);
	2278	TokenIndex payload = expect_token(pc, TokenIdIdentifier);
	2279	TokenIndex index = 0;
	2280	if (eat_token_if(pc, TokenIdComma) != 0)
	2281	index = expect_token(pc, TokenIdIdentifier);
2370	2282	expect_token(pc, TokenIdBinOr);
2371	2283
2372	2284	PtrIndexPayload res;
...	...	@@ -2390,8 +2302,8 @@ static AstNode ast_parse_switch_prong(ParseContext pc) {
2390	2302	assert(res->type == NodeTypeSwitchProng);
2391	2303	res->data.switch_prong.expr = expr;
2392	2304	if (opt_payload.unwrap(&payload)) {
2393		res->data.switch_prong.var_symbol = token_symbol(pc, payload.payload);
2394		res->data.switch_prong.var_is_ptr = payload.asterisk != nullptr;
	2305	res->data.switch_prong.var_symbol = token_identifier(pc, payload.payload);
	2306	res->data.switch_prong.var_is_ptr = payload.asterisk != 0;
2395	2307	}
2396	2308
2397	2309	return res;
...	...	@@ -2407,7 +2319,7 @@ static AstNode ast_parse_switch_case(ParseContext pc) {
2407	2319	res->data.switch_prong.items.append(first);
2408	2320	res->data.switch_prong.any_items_are_range = first->type == NodeTypeSwitchRange;
2409	2321
2410		while (eat_token_if(pc, TokenIdComma) != nullptr) {
	2322	while (eat_token_if(pc, TokenIdComma) != 0) {
2411	2323	AstNode *item = ast_parse_switch_item(pc);
2412	2324	if (item == nullptr)
2413	2325	break;
...	...	@@ -2419,8 +2331,8 @@ static AstNode ast_parse_switch_case(ParseContext pc) {
2419	2331	return res;
2420	2332	}
2421	2333
2422		Token *else_token = eat_token_if(pc, TokenIdKeywordElse);
2423		if (else_token != nullptr)
	2334	TokenIndex else_token = eat_token_if(pc, TokenIdKeywordElse);
	2335	if (else_token != 0)
2424	2336	return ast_create_node(pc, NodeTypeSwitchProng, else_token);
2425	2337
2426	2338	return nullptr;
...	...	@@ -2432,8 +2344,8 @@ static AstNode ast_parse_switch_item(ParseContext pc) {
2432	2344	if (expr == nullptr)
2433	2345	return nullptr;
2434	2346
2435		Token *dots = eat_token_if(pc, TokenIdEllipsis3);
2436		if (dots != nullptr) {
	2347	TokenIndex dots = eat_token_if(pc, TokenIdEllipsis3);
	2348	if (dots != 0) {
2437	2349	AstNode *expr2 = ast_expect(pc, ast_parse_expr);
2438	2350	AstNode *res = ast_create_node(pc, NodeTypeSwitchRange, dots);
2439	2351	res->data.switch_range.start = expr;
...	...	@@ -2478,9 +2390,9 @@ static AstNode ast_parse_assign_op(ParseContext pc) {
2478	2390	table[TokenIdTimesEq] = BinOpTypeAssignTimes;
2479	2391	table[TokenIdTimesPercentEq] = BinOpTypeAssignTimesWrap;
2480	2392
2481		BinOpType op = table[peek_token(pc)->id];
	2393	BinOpType op = table[pc->token_ids[pc->current_token]];
2482	2394	if (op != BinOpTypeInvalid) {
2483		Token *op_token = eat_token(pc);
	2395	TokenIndex op_token = eat_token(pc);
2484	2396	AstNode *res = ast_create_node(pc, NodeTypeBinOpExpr, op_token);
2485	2397	res->data.bin_op_expr.bin_op = op;
2486	2398	return res;
...	...	@@ -2506,9 +2418,9 @@ static AstNode ast_parse_compare_op(ParseContext pc) {
2506	2418	table[TokenIdCmpLessOrEq] = BinOpTypeCmpLessOrEq;
2507	2419	table[TokenIdCmpGreaterOrEq] = BinOpTypeCmpGreaterOrEq;
2508	2420
2509		BinOpType op = table[peek_token(pc)->id];
	2421	BinOpType op = table[pc->token_ids[pc->current_token]];
2510	2422	if (op != BinOpTypeInvalid) {
2511		Token *op_token = eat_token(pc);
	2423	TokenIndex op_token = eat_token(pc);
2512	2424	AstNode *res = ast_create_node(pc, NodeTypeBinOpExpr, op_token);
2513	2425	res->data.bin_op_expr.bin_op = op;
2514	2426	return res;
...	...	@@ -2530,20 +2442,20 @@ static AstNode ast_parse_bitwise_op(ParseContext pc) {
2530	2442	table[TokenIdBinOr] = BinOpTypeBinOr;
2531	2443	table[TokenIdKeywordOrElse] = BinOpTypeUnwrapOptional;
2532	2444
2533		BinOpType op = table[peek_token(pc)->id];
	2445	BinOpType op = table[pc->token_ids[pc->current_token]];
2534	2446	if (op != BinOpTypeInvalid) {
2535		Token *op_token = eat_token(pc);
	2447	TokenIndex op_token = eat_token(pc);
2536	2448	AstNode *res = ast_create_node(pc, NodeTypeBinOpExpr, op_token);
2537	2449	res->data.bin_op_expr.bin_op = op;
2538	2450	return res;
2539	2451	}
2540	2452
2541		Token *catch_token = eat_token_if(pc, TokenIdKeywordCatch);
2542		if (catch_token != nullptr) {
2543		Token *payload = ast_parse_payload(pc);
	2453	TokenIndex catch_token = eat_token_if(pc, TokenIdKeywordCatch);
	2454	if (catch_token != 0) {
	2455	TokenIndex payload = ast_parse_payload(pc);
2544	2456	AstNode *res = ast_create_node(pc, NodeTypeCatchExpr, catch_token);
2545		if (payload != nullptr)
2546		res->data.unwrap_err_expr.symbol = token_symbol(pc, payload);
	2457	if (payload != 0)
	2458	res->data.unwrap_err_expr.symbol = token_identifier(pc, payload);
2547	2459
2548	2460	return res;
2549	2461	}
...	...	@@ -2559,9 +2471,9 @@ static AstNode ast_parse_bit_shift_op(ParseContext pc) {
2559	2471	table[TokenIdBitShiftLeft] = BinOpTypeBitShiftLeft;
2560	2472	table[TokenIdBitShiftRight] = BinOpTypeBitShiftRight;
2561	2473
2562		BinOpType op = table[peek_token(pc)->id];
	2474	BinOpType op = table[pc->token_ids[pc->current_token]];
2563	2475	if (op != BinOpTypeInvalid) {
2564		Token *op_token = eat_token(pc);
	2476	TokenIndex op_token = eat_token(pc);
2565	2477	AstNode *res = ast_create_node(pc, NodeTypeBinOpExpr, op_token);
2566	2478	res->data.bin_op_expr.bin_op = op;
2567	2479	return res;
...	...	@@ -2584,9 +2496,9 @@ static AstNode ast_parse_addition_op(ParseContext pc) {
2584	2496	table[TokenIdPlusPercent] = BinOpTypeAddWrap;
2585	2497	table[TokenIdMinusPercent] = BinOpTypeSubWrap;
2586	2498
2587		BinOpType op = table[peek_token(pc)->id];
	2499	BinOpType op = table[pc->token_ids[pc->current_token]];
2588	2500	if (op != BinOpTypeInvalid) {
2589		Token *op_token = eat_token(pc);
	2501	TokenIndex op_token = eat_token(pc);
2590	2502	AstNode *res = ast_create_node(pc, NodeTypeBinOpExpr, op_token);
2591	2503	res->data.bin_op_expr.bin_op = op;
2592	2504	return res;
...	...	@@ -2611,9 +2523,9 @@ static AstNode ast_parse_multiply_op(ParseContext pc) {
2611	2523	table[TokenIdStarStar] = BinOpTypeArrayMult;
2612	2524	table[TokenIdTimesPercent] = BinOpTypeMultWrap;
2613	2525
2614		BinOpType op = table[peek_token(pc)->id];
	2526	BinOpType op = table[pc->token_ids[pc->current_token]];
2615	2527	if (op != BinOpTypeInvalid) {
2616		Token *op_token = eat_token(pc);
	2528	TokenIndex op_token = eat_token(pc);
2617	2529	AstNode *res = ast_create_node(pc, NodeTypeBinOpExpr, op_token);
2618	2530	res->data.bin_op_expr.bin_op = op;
2619	2531	return res;
...	...	@@ -2638,23 +2550,23 @@ static AstNode ast_parse_prefix_op(ParseContext pc) {
2638	2550	table[TokenIdMinusPercent] = PrefixOpNegationWrap;
2639	2551	table[TokenIdAmpersand] = PrefixOpAddrOf;
2640	2552
2641		PrefixOp op = table[peek_token(pc)->id];
	2553	PrefixOp op = table[pc->token_ids[pc->current_token]];
2642	2554	if (op != PrefixOpInvalid) {
2643		Token *op_token = eat_token(pc);
	2555	TokenIndex op_token = eat_token(pc);
2644	2556	AstNode *res = ast_create_node(pc, NodeTypePrefixOpExpr, op_token);
2645	2557	res->data.prefix_op_expr.prefix_op = op;
2646	2558	return res;
2647	2559	}
2648	2560
2649		Token *try_token = eat_token_if(pc, TokenIdKeywordTry);
2650		if (try_token != nullptr) {
	2561	TokenIndex try_token = eat_token_if(pc, TokenIdKeywordTry);
	2562	if (try_token != 0) {
2651	2563	AstNode *res = ast_create_node(pc, NodeTypeReturnExpr, try_token);
2652	2564	res->data.return_expr.kind = ReturnKindError;
2653	2565	return res;
2654	2566	}
2655	2567
2656		Token *await = eat_token_if(pc, TokenIdKeywordAwait);
2657		if (await != nullptr) {
	2568	TokenIndex await = eat_token_if(pc, TokenIdKeywordAwait);
	2569	if (await != 0) {
2658	2570	AstNode *res = ast_create_node(pc, NodeTypeAwaitExpr, await);
2659	2571	return res;
2660	2572	}
...	...	@@ -2668,16 +2580,16 @@ static AstNode ast_parse_prefix_op(ParseContext pc) {
2668	2580	// / ArrayTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile)*
2669	2581	// / PtrTypeStart (KEYWORD_align LPAREN Expr (COLON INTEGER COLON INTEGER)? RPAREN / KEYWORD_const / KEYWORD_volatile)*
2670	2582	static AstNode ast_parse_prefix_type_op(ParseContext pc) {
2671		Token *questionmark = eat_token_if(pc, TokenIdQuestion);
2672		if (questionmark != nullptr) {
	2583	TokenIndex questionmark = eat_token_if(pc, TokenIdQuestion);
	2584	if (questionmark != 0) {
2673	2585	AstNode *res = ast_create_node(pc, NodeTypePrefixOpExpr, questionmark);
2674	2586	res->data.prefix_op_expr.prefix_op = PrefixOpOptional;
2675	2587	return res;
2676	2588	}
2677	2589
2678		Token *anyframe = eat_token_if(pc, TokenIdKeywordAnyFrame);
2679		if (anyframe != nullptr) {
2680		if (eat_token_if(pc, TokenIdArrow) != nullptr) {
	2590	TokenIndex anyframe = eat_token_if(pc, TokenIdKeywordAnyFrame);
	2591	if (anyframe != 0) {
	2592	if (eat_token_if(pc, TokenIdArrow) != 0) {
2681	2593	AstNode *res = ast_create_node(pc, NodeTypeAnyFrameType, anyframe);
2682	2594	return res;
2683	2595	}
...	...	@@ -2685,18 +2597,18 @@ static AstNode ast_parse_prefix_type_op(ParseContext pc) {
2685	2597	put_back_token(pc);
2686	2598	}
2687	2599
2688		Token *arr_init_lbracket = eat_token_if(pc, TokenIdLBracket);
2689		if (arr_init_lbracket != nullptr) {
2690		Token *underscore = eat_token_if(pc, TokenIdSymbol);
2691		if (underscore == nullptr) {
	2600	TokenIndex arr_init_lbracket = eat_token_if(pc, TokenIdLBracket);
	2601	if (arr_init_lbracket != 0) {
	2602	TokenIndex underscore = eat_token_if(pc, TokenIdIdentifier);
	2603	if (underscore == 0) {
2692	2604	put_back_token(pc);
2693		} else if (!buf_eql_str(token_buf(underscore), "_")) {
	2605	} else if (!buf_eql_str(token_buf(pc, underscore), "_")) {
2694	2606	put_back_token(pc);
2695	2607	put_back_token(pc);
2696	2608	} else {
2697	2609	AstNode *sentinel = nullptr;
2698		Token *colon = eat_token_if(pc, TokenIdColon);
2699		if (colon != nullptr) {
	2610	TokenIndex colon = eat_token_if(pc, TokenIdColon);
	2611	if (colon != 0) {
2700	2612	sentinel = ast_expect(pc, ast_parse_expr);
2701	2613	}
2702	2614	expect_token(pc, TokenIdRBracket);
...	...	@@ -2715,33 +2627,33 @@ static AstNode ast_parse_prefix_type_op(ParseContext pc) {
2715	2627	if (child == nullptr)
2716	2628	child = ptr;
2717	2629	while (true) {
2718		Token *allowzero_token = eat_token_if(pc, TokenIdKeywordAllowZero);
2719		if (allowzero_token != nullptr) {
	2630	TokenIndex allowzero_token = eat_token_if(pc, TokenIdKeywordAllowZero);
	2631	if (allowzero_token != 0) {
2720	2632	child->data.pointer_type.allow_zero_token = allowzero_token;
2721	2633	continue;
2722	2634	}
2723	2635
2724		if (eat_token_if(pc, TokenIdKeywordAlign) != nullptr) {
	2636	if (eat_token_if(pc, TokenIdKeywordAlign) != 0) {
2725	2637	expect_token(pc, TokenIdLParen);
2726	2638	AstNode *align_expr = ast_expect(pc, ast_parse_expr);
2727	2639	child->data.pointer_type.align_expr = align_expr;
2728		if (eat_token_if(pc, TokenIdColon) != nullptr) {
2729		Token *bit_offset_start = expect_token(pc, TokenIdIntLiteral);
	2640	if (eat_token_if(pc, TokenIdColon) != 0) {
	2641	TokenIndex bit_offset_start = expect_token(pc, TokenIdIntLiteral);
2730	2642	expect_token(pc, TokenIdColon);
2731		Token *host_int_bytes = expect_token(pc, TokenIdIntLiteral);
2732		child->data.pointer_type.bit_offset_start = token_bigint(bit_offset_start);
2733		child->data.pointer_type.host_int_bytes = token_bigint(host_int_bytes);
	2643	TokenIndex host_int_bytes = expect_token(pc, TokenIdIntLiteral);
	2644	child->data.pointer_type.bit_offset_start = bit_offset_start;
	2645	child->data.pointer_type.host_int_bytes = host_int_bytes;
2734	2646	}
2735	2647	expect_token(pc, TokenIdRParen);
2736	2648	continue;
2737	2649	}
2738	2650
2739		if (eat_token_if(pc, TokenIdKeywordConst) != nullptr) {
	2651	if (eat_token_if(pc, TokenIdKeywordConst) != 0) {
2740	2652	child->data.pointer_type.is_const = true;
2741	2653	continue;
2742	2654	}
2743	2655
2744		if (eat_token_if(pc, TokenIdKeywordVolatile) != nullptr) {
	2656	if (eat_token_if(pc, TokenIdKeywordVolatile) != 0) {
2745	2657	child->data.pointer_type.is_volatile = true;
2746	2658	continue;
2747	2659	}
...	...	@@ -2756,8 +2668,8 @@ static AstNode ast_parse_prefix_type_op(ParseContext pc) {
2756	2668	if (array != nullptr) {
2757	2669	assert(array->type == NodeTypeArrayType);
2758	2670	while (true) {
2759		Token *allowzero_token = eat_token_if(pc, TokenIdKeywordAllowZero);
2760		if (allowzero_token != nullptr) {
	2671	TokenIndex allowzero_token = eat_token_if(pc, TokenIdKeywordAllowZero);
	2672	if (allowzero_token != 0) {
2761	2673	array->data.array_type.allow_zero_token = allowzero_token;
2762	2674	continue;
2763	2675	}
...	...	@@ -2768,12 +2680,12 @@ static AstNode ast_parse_prefix_type_op(ParseContext pc) {
2768	2680	continue;
2769	2681	}
2770	2682
2771		if (eat_token_if(pc, TokenIdKeywordConst) != nullptr) {
	2683	if (eat_token_if(pc, TokenIdKeywordConst) != 0) {
2772	2684	array->data.array_type.is_const = true;
2773	2685	continue;
2774	2686	}
2775	2687
2776		if (eat_token_if(pc, TokenIdKeywordVolatile) != nullptr) {
	2688	if (eat_token_if(pc, TokenIdKeywordVolatile) != 0) {
2777	2689	array->data.array_type.is_volatile = true;
2778	2690	continue;
2779	2691	}
...	...	@@ -2793,14 +2705,14 @@ static AstNode ast_parse_prefix_type_op(ParseContext pc) {
2793	2705	// / DOTASTERISK
2794	2706	// / DOTQUESTIONMARK
2795	2707	static AstNode ast_parse_suffix_op(ParseContext pc) {
2796		Token *lbracket = eat_token_if(pc, TokenIdLBracket);
2797		if (lbracket != nullptr) {
	2708	TokenIndex lbracket = eat_token_if(pc, TokenIdLBracket);
	2709	if (lbracket != 0) {
2798	2710	AstNode *start = ast_expect(pc, ast_parse_expr);
2799	2711	AstNode *end = nullptr;
2800		if (eat_token_if(pc, TokenIdEllipsis2) != nullptr) {
	2712	if (eat_token_if(pc, TokenIdEllipsis2) != 0) {
2801	2713	AstNode *sentinel = nullptr;
2802	2714	end = ast_parse_expr(pc);
2803		if (eat_token_if(pc, TokenIdColon) != nullptr) {
	2715	if (eat_token_if(pc, TokenIdColon) != 0) {
2804	2716	sentinel = ast_parse_expr(pc);
2805	2717	}
2806	2718	expect_token(pc, TokenIdRBracket);
...	...	@@ -2819,18 +2731,18 @@ static AstNode ast_parse_suffix_op(ParseContext pc) {
2819	2731	return res;
2820	2732	}
2821	2733
2822		Token *dot_asterisk = eat_token_if(pc, TokenIdDotStar);
2823		if (dot_asterisk != nullptr)
	2734	TokenIndex dot_asterisk = eat_token_if(pc, TokenIdDotStar);
	2735	if (dot_asterisk != 0)
2824	2736	return ast_create_node(pc, NodeTypePtrDeref, dot_asterisk);
2825	2737
2826		Token *dot = eat_token_if(pc, TokenIdDot);
2827		if (dot != nullptr) {
2828		if (eat_token_if(pc, TokenIdQuestion) != nullptr)
	2738	TokenIndex dot = eat_token_if(pc, TokenIdDot);
	2739	if (dot != 0) {
	2740	if (eat_token_if(pc, TokenIdQuestion) != 0)
2829	2741	return ast_create_node(pc, NodeTypeUnwrapOptional, dot);
2830	2742
2831		Token *ident = expect_token(pc, TokenIdSymbol);
	2743	TokenIndex ident = expect_token(pc, TokenIdIdentifier);
2832	2744	AstNode *res = ast_create_node(pc, NodeTypeFieldAccessExpr, dot);
2833		res->data.field_access_expr.field_name = token_buf(ident);
	2745	res->data.field_access_expr.field_name = token_buf(pc, ident);
2834	2746	return res;
2835	2747	}
2836	2748
...	...	@@ -2839,8 +2751,8 @@ static AstNode ast_parse_suffix_op(ParseContext pc) {
2839	2751
2840	2752	// FnCallArguments <- LPAREN ExprList RPAREN
2841	2753	static AstNode ast_parse_fn_call_arguments(ParseContext pc) {
2842		Token *paren = eat_token_if(pc, TokenIdLParen);
2843		if (paren == nullptr)
	2754	TokenIndex paren = eat_token_if(pc, TokenIdLParen);
	2755	if (paren == 0)
2844	2756	return nullptr;
2845	2757
2846	2758	ZigList<AstNode *> params = ast_parse_list(pc, TokenIdComma, ast_parse_expr);
...	...	@@ -2854,14 +2766,14 @@ static AstNode ast_parse_fn_call_arguments(ParseContext pc) {
2854	2766
2855	2767	// ArrayTypeStart <- LBRACKET Expr? RBRACKET
2856	2768	static AstNode ast_parse_array_type_start(ParseContext pc) {
2857		Token *lbracket = eat_token_if(pc, TokenIdLBracket);
2858		if (lbracket == nullptr)
	2769	TokenIndex lbracket = eat_token_if(pc, TokenIdLBracket);
	2770	if (lbracket == 0)
2859	2771	return nullptr;
2860	2772
2861	2773	AstNode *size = ast_parse_expr(pc);
2862	2774	AstNode *sentinel = nullptr;
2863		Token *colon = eat_token_if(pc, TokenIdColon);
2864		if (colon != nullptr) {
	2775	TokenIndex colon = eat_token_if(pc, TokenIdColon);
	2776	if (colon != 0) {
2865	2777	sentinel = ast_expect(pc, ast_parse_expr);
2866	2778	}
2867	2779	expect_token(pc, TokenIdRBracket);
...	...	@@ -2879,10 +2791,10 @@ static AstNode ast_parse_array_type_start(ParseContext pc) {
2879	2791	static AstNode ast_parse_ptr_type_start(ParseContext pc) {
2880	2792	AstNode *sentinel = nullptr;
2881	2793
2882		Token *asterisk = eat_token_if(pc, TokenIdStar);
2883		if (asterisk != nullptr) {
2884		Token *colon = eat_token_if(pc, TokenIdColon);
2885		if (colon != nullptr) {
	2794	TokenIndex asterisk = eat_token_if(pc, TokenIdStar);
	2795	if (asterisk != 0) {
	2796	TokenIndex colon = eat_token_if(pc, TokenIdColon);
	2797	if (colon != 0) {
2886	2798	sentinel = ast_expect(pc, ast_parse_expr);
2887	2799	}
2888	2800	AstNode *res = ast_create_node(pc, NodeTypePointerType, asterisk);
...	...	@@ -2891,10 +2803,10 @@ static AstNode ast_parse_ptr_type_start(ParseContext pc) {
2891	2803	return res;
2892	2804	}
2893	2805
2894		Token *asterisk2 = eat_token_if(pc, TokenIdStarStar);
2895		if (asterisk2 != nullptr) {
2896		Token *colon = eat_token_if(pc, TokenIdColon);
2897		if (colon != nullptr) {
	2806	TokenIndex asterisk2 = eat_token_if(pc, TokenIdStarStar);
	2807	if (asterisk2 != 0) {
	2808	TokenIndex colon = eat_token_if(pc, TokenIdColon);
	2809	if (colon != 0) {
2898	2810	sentinel = ast_expect(pc, ast_parse_expr);
2899	2811	}
2900	2812	AstNode *res = ast_create_node(pc, NodeTypePointerType, asterisk2);
...	...	@@ -2906,15 +2818,15 @@ static AstNode ast_parse_ptr_type_start(ParseContext pc) {
2906	2818	return res;
2907	2819	}
2908	2820
2909		Token *lbracket = eat_token_if(pc, TokenIdLBracket);
2910		if (lbracket != nullptr) {
2911		Token *star = eat_token_if(pc, TokenIdStar);
2912		if (star == nullptr) {
	2821	TokenIndex lbracket = eat_token_if(pc, TokenIdLBracket);
	2822	if (lbracket != 0) {
	2823	TokenIndex star = eat_token_if(pc, TokenIdStar);
	2824	if (star == 0) {
2913	2825	put_back_token(pc);
2914	2826	} else {
2915		Token *c_tok = eat_token_if(pc, TokenIdSymbol);
2916		if (c_tok != nullptr) {
2917		if (!buf_eql_str(token_buf(c_tok), "c")) {
	2827	TokenIndex c_tok = eat_token_if(pc, TokenIdIdentifier);
	2828	if (c_tok != 0) {
	2829	if (!buf_eql_str(token_buf(pc, c_tok), "c")) {
2918	2830	put_back_token(pc); // c symbol
2919	2831	} else {
2920	2832	expect_token(pc, TokenIdRBracket);
...	...	@@ -2924,8 +2836,8 @@ static AstNode ast_parse_ptr_type_start(ParseContext pc) {
2924	2836	}
2925	2837	}
2926	2838
2927		Token *colon = eat_token_if(pc, TokenIdColon);
2928		if (colon != nullptr) {
	2839	TokenIndex colon = eat_token_if(pc, TokenIdColon);
	2840	if (colon != 0) {
2929	2841	sentinel = ast_expect(pc, ast_parse_expr);
2930	2842	}
2931	2843	expect_token(pc, TokenIdRBracket);
...	...	@@ -2951,9 +2863,7 @@ static AstNode ast_parse_container_decl_auto(ParseContext pc) {
2951	2863
2952	2864	res->data.container_decl.fields = members.fields;
2953	2865	res->data.container_decl.decls = members.decls;
2954		if (buf_len(&members.doc_comments) != 0) {
2955		res->data.container_decl.doc_comments = members.doc_comments;
2956		}
	2866	res->data.container_decl.doc_comments = members.doc_comments;
2957	2867	return res;
2958	2868	}
2959	2869
...	...	@@ -2963,8 +2873,8 @@ static AstNode ast_parse_container_decl_auto(ParseContext pc) {
2963	2873	// / KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / Expr) RPAREN)?
2964	2874	// / KEYWORD_opaque
2965	2875	static AstNode ast_parse_container_decl_type(ParseContext pc) {
2966		Token *first = eat_token_if(pc, TokenIdKeywordStruct);
2967		if (first != nullptr) {
	2876	TokenIndex first = eat_token_if(pc, TokenIdKeywordStruct);
	2877	if (first != 0) {
2968	2878	AstNode *res = ast_create_node(pc, NodeTypeContainerDecl, first);
2969	2879	res->data.container_decl.init_arg_expr = nullptr;
2970	2880	res->data.container_decl.kind = ContainerKindStruct;
...	...	@@ -2972,7 +2882,7 @@ static AstNode ast_parse_container_decl_type(ParseContext pc) {
2972	2882	}
2973	2883
2974	2884	first = eat_token_if(pc, TokenIdKeywordOpaque);
2975		if (first != nullptr) {
	2885	if (first != 0) {
2976	2886	AstNode *res = ast_create_node(pc, NodeTypeContainerDecl, first);
2977	2887	res->data.container_decl.init_arg_expr = nullptr;
2978	2888	res->data.container_decl.kind = ContainerKindOpaque;
...	...	@@ -2980,9 +2890,9 @@ static AstNode ast_parse_container_decl_type(ParseContext pc) {
2980	2890	}
2981	2891
2982	2892	first = eat_token_if(pc, TokenIdKeywordEnum);
2983		if (first != nullptr) {
	2893	if (first != 0) {
2984	2894	AstNode *init_arg_expr = nullptr;
2985		if (eat_token_if(pc, TokenIdLParen) != nullptr) {
	2895	if (eat_token_if(pc, TokenIdLParen) != 0) {
2986	2896	init_arg_expr = ast_expect(pc, ast_parse_expr);
2987	2897	expect_token(pc, TokenIdRParen);
2988	2898	}
...	...	@@ -2993,13 +2903,13 @@ static AstNode ast_parse_container_decl_type(ParseContext pc) {
2993	2903	}
2994	2904
2995	2905	first = eat_token_if(pc, TokenIdKeywordUnion);
2996		if (first != nullptr) {
	2906	if (first != 0) {
2997	2907	AstNode *init_arg_expr = nullptr;
2998	2908	bool auto_enum = false;
2999		if (eat_token_if(pc, TokenIdLParen) != nullptr) {
3000		if (eat_token_if(pc, TokenIdKeywordEnum) != nullptr) {
	2909	if (eat_token_if(pc, TokenIdLParen) != 0) {
	2910	if (eat_token_if(pc, TokenIdKeywordEnum) != 0) {
3001	2911	auto_enum = true;
3002		if (eat_token_if(pc, TokenIdLParen) != nullptr) {
	2912	if (eat_token_if(pc, TokenIdLParen) != 0) {
3003	2913	init_arg_expr = ast_expect(pc, ast_parse_expr);
3004	2914	expect_token(pc, TokenIdRParen);
3005	2915	}
...	...	@@ -3022,7 +2932,7 @@ static AstNode ast_parse_container_decl_type(ParseContext pc) {
3022	2932
3023	2933	// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN
3024	2934	static AstNode ast_parse_byte_align(ParseContext pc) {
3025		if (eat_token_if(pc, TokenIdKeywordAlign) == nullptr)
	2935	if (eat_token_if(pc, TokenIdKeywordAlign) == 0)
3026	2936	return nullptr;
3027	2937
3028	2938	expect_token(pc, TokenIdLParen);
...	...	@@ -3103,7 +3013,7 @@ void ast_visit_node_children(AstNode node, void (visit)(AstNode *, void cont
3103	3013	case NodeTypeCharLiteral:
3104	3014	// none
3105	3015	break;
3106		case NodeTypeSymbol:
	3016	case NodeTypeIdentifier:
3107	3017	// none
3108	3018	break;
3109	3019	case NodeTypePrefixOpExpr:
...	...	@@ -3264,3 +3174,414 @@ void ast_visit_node_children(AstNode node, void (visit)(AstNode *, void cont
3264	3174	break;
3265	3175	}
3266	3176	}
	3177
	3178	Error source_string_literal_buf(const char source, Buf out, size_t *bad_index) {
	3179	size_t byte_offset = 0;
	3180
	3181	assert(source[byte_offset] == '"');
	3182	byte_offset += 1;
	3183
	3184	buf_resize(out, 0);
	3185
	3186	uint32_t codepoint;
	3187
	3188	enum {
	3189	StateStart,
	3190	StateBackslash,
	3191	StateUnicodeLBrace,
	3192	StateUnicodeDigit,
	3193	} state = StateStart;
	3194	for (;;byte_offset += 1) {
	3195	switch (state) {
	3196	case StateStart: switch (source[byte_offset]) {
	3197	case '\\':
	3198	state = StateBackslash;
	3199	continue;
	3200	case '\n':
	3201	*bad_index = byte_offset;
	3202	return ErrorInvalidCharacter;
	3203	case '"':
	3204	return ErrorNone;
	3205	default:
	3206	buf_append_char(out, source[byte_offset]);
	3207	continue;
	3208	}
	3209	case StateBackslash: switch (source[byte_offset]) {
	3210	case 'n':
	3211	buf_append_char(out, '\n');
	3212	state = StateStart;
	3213	continue;
	3214	case 'r':
	3215	buf_append_char(out, '\r');
	3216	state = StateStart;
	3217	continue;
	3218	case '\\':
	3219	buf_append_char(out, '\\');
	3220	state = StateStart;
	3221	continue;
	3222	case 't':
	3223	buf_append_char(out, '\t');
	3224	state = StateStart;
	3225	continue;
	3226	case '\'':
	3227	buf_append_char(out, '\'');
	3228	state = StateStart;
	3229	continue;
	3230	case '"':
	3231	buf_append_char(out, '"');
	3232	state = StateStart;
	3233	continue;
	3234	case 'x': {
	3235	byte_offset += 1;
	3236	uint8_t digit1;
	3237	if (source[byte_offset] >= '0' && source[byte_offset] <= '9') {
	3238	digit1 = source[byte_offset] - '0';
	3239	} else if (source[byte_offset] >= 'a' && source[byte_offset] <= 'z') {
	3240	digit1 = source[byte_offset] - 'a' + 10;
	3241	} else if (source[byte_offset] >= 'A' && source[byte_offset] <= 'Z') {
	3242	digit1 = source[byte_offset] - 'A' + 10;
	3243	} else {
	3244	*bad_index = byte_offset;
	3245	return ErrorInvalidCharacter;
	3246	}
	3247
	3248	byte_offset += 1;
	3249	uint8_t digit0;
	3250	if (source[byte_offset] >= '0' && source[byte_offset] <= '9') {
	3251	digit0 = source[byte_offset] - '0';
	3252	} else if (source[byte_offset] >= 'a' && source[byte_offset] <= 'z') {
	3253	digit0 = source[byte_offset] - 'a' + 10;
	3254	} else if (source[byte_offset] >= 'A' && source[byte_offset] <= 'Z') {
	3255	digit0 = source[byte_offset] - 'A' + 10;
	3256	} else {
	3257	*bad_index = byte_offset;
	3258	return ErrorInvalidCharacter;
	3259	}
	3260
	3261	buf_append_char(out, digit1 * 16 + digit0);
	3262	state = StateStart;
	3263	continue;
	3264	}
	3265	case 'u':
	3266	state = StateUnicodeLBrace;
	3267	continue;
	3268	default:
	3269	*bad_index = byte_offset;
	3270	return ErrorInvalidCharacter;
	3271	}
	3272	case StateUnicodeLBrace: switch (source[byte_offset]) {
	3273	case '{':
	3274	state = StateUnicodeDigit;
	3275	codepoint = 0;
	3276	continue;
	3277	default:
	3278	*bad_index = byte_offset;
	3279	return ErrorInvalidCharacter;
	3280	}
	3281	case StateUnicodeDigit: {
	3282	uint8_t digit;
	3283	if (source[byte_offset] >= '0' && source[byte_offset] <= '9') {
	3284	digit = source[byte_offset] - '0';
	3285	} else if (source[byte_offset] >= 'a' && source[byte_offset] <= 'z') {
	3286	digit = source[byte_offset] - 'a' + 10;
	3287	} else if (source[byte_offset] >= 'A' && source[byte_offset] <= 'Z') {
	3288	digit = source[byte_offset] - 'A' + 10;
	3289	} else if (source[byte_offset] == '}') {
	3290	if (codepoint < 0x80) {
	3291	buf_append_char(out, codepoint);
	3292	} else if (codepoint < 0x800) {
	3293	buf_append_char(out, 0xc0 \| (codepoint >> 6));
	3294	buf_append_char(out, 0x80 \| (codepoint & 0x3f));
	3295	} else if (codepoint < 0x10000) {
	3296	buf_append_char(out, 0xe0 \| (codepoint >> 12));
	3297	buf_append_char(out, 0x80 \| ((codepoint >> 6) & 0x3f));
	3298	buf_append_char(out, 0x80 \| (codepoint & 0x3f));
	3299	} else if (codepoint < 0x110000) {
	3300	buf_append_char(out, 0xf0 \| (codepoint >> 18));
	3301	buf_append_char(out, 0x80 \| ((codepoint >> 12) & 0x3f));
	3302	buf_append_char(out, 0x80 \| ((codepoint >> 6) & 0x3f));
	3303	buf_append_char(out, 0x80 \| (codepoint & 0x3f));
	3304	} else {
	3305	*bad_index = byte_offset;
	3306	return ErrorUnicodePointTooLarge;
	3307	}
	3308	state = StateStart;
	3309	continue;
	3310	} else {
	3311	*bad_index = byte_offset;
	3312	return ErrorInvalidCharacter;
	3313	}
	3314	codepoint = codepoint * 16 + digit;
	3315	continue;
	3316	}
	3317	}
	3318	}
	3319	zig_unreachable();
	3320	}
	3321
	3322	static uint32_t utf8_code_point(const uint8_t *bytes) {
	3323	if (bytes[0] <= 0x7f) {
	3324	return bytes[0];
	3325	} else if (bytes[0] >= 0xc0 && bytes[0] <= 0xdf) {
	3326	uint32_t result = bytes[0] & 0x1f;
	3327	result <<= 6;
	3328	result \|= bytes[1] & 0x3f;
	3329	return result;
	3330	} else if (bytes[0] >= 0xe0 && bytes[0] <= 0xef) {
	3331	uint32_t result = bytes[0] & 0xf;
	3332
	3333	result <<= 6;
	3334	result \|= bytes[1] & 0x3f;
	3335
	3336	result <<= 6;
	3337	result \|= bytes[2] & 0x3f;
	3338
	3339	return result;
	3340	} else if (bytes[0] >= 0xf0 && bytes[0] <= 0xf7) {
	3341	uint32_t result = bytes[0] & 0x7;
	3342
	3343	result <<= 6;
	3344	result \|= bytes[1] & 0x3f;
	3345
	3346	result <<= 6;
	3347	result \|= bytes[2] & 0x3f;
	3348
	3349	result <<= 6;
	3350	result \|= bytes[3] & 0x3f;
	3351
	3352	return result;
	3353	} else {
	3354	zig_unreachable();
	3355	}
	3356	}
	3357
	3358	Error source_char_literal(const char source, uint32_t result, size_t *bad_index) {
	3359	if (source[0] != '\\') {
	3360	result = utf8_code_point((const uint8_t )source);
	3361	return ErrorNone;
	3362	}
	3363
	3364	uint32_t byte_offset = 1;
	3365	uint32_t codepoint;
	3366
	3367	enum State {
	3368	StateBackslash,
	3369	StateUnicodeLBrace,
	3370	StateUnicodeDigit,
	3371	} state = StateBackslash;
	3372
	3373	for (;;byte_offset += 1) {
	3374	switch (state) {
	3375	case StateBackslash: switch (source[byte_offset]) {
	3376	case 'n':
	3377	*result = '\n';
	3378	return ErrorNone;
	3379	case 'r':
	3380	*result = '\r';
	3381	return ErrorNone;
	3382	case '\\':
	3383	*result = '\\';
	3384	return ErrorNone;
	3385	case 't':
	3386	*result = '\t';
	3387	return ErrorNone;
	3388	case '\'':
	3389	*result = '\'';
	3390	return ErrorNone;
	3391	case '"':
	3392	*result = '"';
	3393	return ErrorNone;
	3394	case 'x': {
	3395	byte_offset += 1;
	3396	uint8_t digit1;
	3397	if (source[byte_offset] >= '0' && source[byte_offset] <= '9') {
	3398	digit1 = source[byte_offset] - '0';
	3399	} else if (source[byte_offset] >= 'a' && source[byte_offset] <= 'z') {
	3400	digit1 = source[byte_offset] - 'a' + 10;
	3401	} else if (source[byte_offset] >= 'A' && source[byte_offset] <= 'Z') {
	3402	digit1 = source[byte_offset] - 'A' + 10;
	3403	} else {
	3404	*bad_index = byte_offset;
	3405	return ErrorInvalidCharacter;
	3406	}
	3407
	3408	byte_offset += 1;
	3409	uint8_t digit0;
	3410	if (source[byte_offset] >= '0' && source[byte_offset] <= '9') {
	3411	digit0 = source[byte_offset] - '0';
	3412	} else if (source[byte_offset] >= 'a' && source[byte_offset] <= 'z') {
	3413	digit0 = source[byte_offset] - 'a' + 10;
	3414	} else if (source[byte_offset] >= 'A' && source[byte_offset] <= 'Z') {
	3415	digit0 = source[byte_offset] - 'A' + 10;
	3416	} else {
	3417	*bad_index = byte_offset;
	3418	return ErrorInvalidCharacter;
	3419	}
	3420
	3421	result = digit1 16 + digit0;
	3422	return ErrorNone;
	3423	}
	3424	case 'u':
	3425	state = StateUnicodeLBrace;
	3426	continue;
	3427	default:
	3428	*bad_index = byte_offset;
	3429	return ErrorInvalidCharacter;
	3430	}
	3431	case StateUnicodeLBrace: switch (source[byte_offset]) {
	3432	case '{':
	3433	state = StateUnicodeDigit;
	3434	codepoint = 0;
	3435	continue;
	3436	default:
	3437	*bad_index = byte_offset;
	3438	return ErrorInvalidCharacter;
	3439	}
	3440	case StateUnicodeDigit: {
	3441	uint8_t digit;
	3442	if (source[byte_offset] >= '0' && source[byte_offset] <= '9') {
	3443	digit = source[byte_offset] - '0';
	3444	} else if (source[byte_offset] >= 'a' && source[byte_offset] <= 'z') {
	3445	digit = source[byte_offset] - 'a' + 10;
	3446	} else if (source[byte_offset] >= 'A' && source[byte_offset] <= 'Z') {
	3447	digit = source[byte_offset] - 'A' + 10;
	3448	} else if (source[byte_offset] == '}') {
	3449	if (codepoint < 0x110000) {
	3450	*result = codepoint;
	3451	return ErrorNone;
	3452	} else {
	3453	*bad_index = byte_offset;
	3454	return ErrorUnicodePointTooLarge;
	3455	}
	3456	} else {
	3457	*bad_index = byte_offset;
	3458	return ErrorInvalidCharacter;
	3459	}
	3460	codepoint = codepoint * 16 + digit;
	3461	continue;
	3462	}
	3463	}
	3464	}
	3465	}
	3466
	3467
	3468	Buf token_string_literal_buf(RootStruct root_struct, TokenIndex token) {
	3469	Error err;
	3470	assert(root_struct->token_ids[token] == TokenIdStringLiteral);
	3471	const char *source = buf_ptr(root_struct->source_code);
	3472	size_t byte_offset = root_struct->token_locs[token].offset;
	3473	size_t bad_index;
	3474	Buf *str = buf_alloc();
	3475	if ((err = source_string_literal_buf(source + byte_offset, str, &bad_index))) {
	3476	zig_panic("TODO handle string literal parse error");
	3477	}
	3478	return str;
	3479	}
	3480
	3481	Buf token_identifier_buf(RootStruct root_struct, TokenIndex token) {
	3482	Error err;
	3483	const char *source = buf_ptr(root_struct->source_code);
	3484	size_t byte_offset = root_struct->token_locs[token].offset;
	3485	if (root_struct->token_ids[token] == TokenIdBuiltin) {
	3486	byte_offset += 1;
	3487	} else {
	3488	assert(root_struct->token_ids[token] == TokenIdIdentifier);
	3489	}
	3490	assert(source[byte_offset] != '.'); // wrong token index
	3491
	3492	if (source[byte_offset] == '@') {
	3493	size_t bad_index;
	3494	Buf *str = buf_alloc();
	3495	if ((err = source_string_literal_buf(source + byte_offset + 1, str, &bad_index))) {
	3496	zig_panic("TODO handle string literal parse error");
	3497	}
	3498	return str;
	3499	} else {
	3500	size_t start = byte_offset;
	3501	for (;; byte_offset += 1) {
	3502	if (source[byte_offset] == 0) break;
	3503	if ((source[byte_offset] >= 'a' && source[byte_offset] <= 'z') \|\|
	3504	(source[byte_offset] >= 'A' && source[byte_offset] <= 'Z') \|\|
	3505	(source[byte_offset] >= '0' && source[byte_offset] <= '9') \|\|
	3506	source[byte_offset] == '_')
	3507	{
	3508	continue;
	3509	}
	3510	break;
	3511	}
	3512	return buf_create_from_mem(source + start, byte_offset - start);
	3513	}
	3514	}
	3515
	3516	Buf node_identifier_buf(AstNode node) {
	3517	assert(node->type == NodeTypeIdentifier);
	3518	RootStruct *root_struct = node->owner->data.structure.root_struct;
	3519	return token_identifier_buf(root_struct, node->main_token);
	3520	}
	3521
	3522	Buf node_string_literal_buf(AstNode node) {
	3523	assert(node->type == NodeTypeStringLiteral);
	3524	RootStruct *root_struct = node->owner->data.structure.root_struct;
	3525	return token_string_literal_buf(root_struct, node->main_token);
	3526	}
	3527
	3528	void token_number_literal_bigint(RootStruct root_struct, BigInt result, TokenIndex token) {
	3529	const char *source = buf_ptr(root_struct->source_code);
	3530	uint32_t byte_offset = root_struct->token_locs[token].offset;
	3531
	3532	bigint_init_unsigned(result, 0);
	3533	BigInt radix_bi;
	3534
	3535	if (source[byte_offset] == '0') {
	3536	byte_offset += 1;
	3537	switch (source[byte_offset]) {
	3538	case 'b':
	3539	byte_offset += 1;
	3540	bigint_init_unsigned(&radix_bi, 2);
	3541	break;
	3542	case 'o':
	3543	byte_offset += 1;
	3544	bigint_init_unsigned(&radix_bi, 8);
	3545	break;
	3546	case 'x':
	3547	byte_offset += 1;
	3548	bigint_init_unsigned(&radix_bi, 16);
	3549	break;
	3550	default:
	3551	bigint_init_unsigned(&radix_bi, 10);
	3552	break;
	3553	}
	3554	} else {
	3555	bigint_init_unsigned(&radix_bi, 10);
	3556	}
	3557
	3558	BigInt digit_value_bi = {};
	3559	BigInt multiplied = {};
	3560
	3561	for (;source[byte_offset] != 0; byte_offset += 1) {
	3562	uint8_t digit;
	3563	if (source[byte_offset] >= '0' && source[byte_offset] <= '9') {
	3564	digit = source[byte_offset] - '0';
	3565	} else if (source[byte_offset] >= 'a' && source[byte_offset] <= 'z') {
	3566	digit = source[byte_offset] - 'a' + 10;
	3567	} else if (source[byte_offset] >= 'A' && source[byte_offset] <= 'Z') {
	3568	digit = source[byte_offset] - 'A' + 10;
	3569	} else if (source[byte_offset] == '_') {
	3570	continue;
	3571	} else {
	3572	break;
	3573	}
	3574	bigint_deinit(&digit_value_bi);
	3575	bigint_init_unsigned(&digit_value_bi, digit);
	3576
	3577	bigint_deinit(&multiplied);
	3578	bigint_mul(&multiplied, result, &radix_bi);
	3579
	3580	bigint_add(result, &multiplied, &digit_value_bi);
	3581	}
	3582
	3583	bigint_deinit(&digit_value_bi);
	3584	bigint_deinit(&multiplied);
	3585	bigint_deinit(&radix_bi);
	3586	}
	3587

src/stage1/parser.hpp+12-5

...	...	@@ -12,14 +12,21 @@
12	12	#include "tokenizer.hpp"
13	13	#include "errmsg.hpp"
14	14
15		ATTRIBUTE_PRINTF(2, 3)
16		void ast_token_error(Token token, const char format, ...);
17
18
19		AstNode * ast_parse(Buf buf, ZigList<Token> tokens, ZigType *owner, ErrColor err_color);
	15	AstNode * ast_parse(Buf buf, ZigType owner, ErrColor err_color);
20	16
21	17	void ast_print(AstNode *node, int indent);
22	18
23	19	void ast_visit_node_children(AstNode node, void (visit)(AstNode *, void context), void *context);
24	20
	21	Buf node_identifier_buf(AstNode node);
	22	Buf node_string_literal_buf(AstNode node);
	23
	24	Buf token_identifier_buf(RootStruct root_struct, TokenIndex token);
	25	Buf token_string_literal_buf(RootStruct root_struct, TokenIndex token);
	26
	27	void token_number_literal_bigint(RootStruct root_struct, BigInt result, TokenIndex token);
	28
	29	Error source_string_literal_buf(const char source, Buf out, size_t *bad_index);
	30	Error source_char_literal(const char source, uint32_t out, size_t *bad_index);
	31
25	32	#endif

src/stage1/stage2.h+2

...	...	@@ -112,6 +112,8 @@ enum Error {
112	112	ErrorZigIsTheCCompiler,
113	113	ErrorFileBusy,
114	114	ErrorLocked,
	115	ErrorInvalidCharacter,
	116	ErrorUnicodePointTooLarge,
115	117	};
116	118
117	119	// ABI warning

src/stage1/tokenizer.cpp+875-984

...	...	@@ -30,18 +30,28 @@
30	30	case '7': \
31	31	case '8': \
32	32	case '9'
	33
33	34	#define DIGIT \
34	35	'0': \
35	36	case DIGIT_NON_ZERO
36	37
37		#define ALPHA \
	38	#define HEXDIGIT \
38	39	'a': \
39	40	case 'b': \
40	41	case 'c': \
41	42	case 'd': \
42	43	case 'e': \
43	44	case 'f': \
44		case 'g': \
	45	case 'A': \
	46	case 'B': \
	47	case 'C': \
	48	case 'D': \
	49	case 'E': \
	50	case 'F': \
	51	case DIGIT
	52
	53	#define ALPHA_EXCEPT_HEX_P_O_X \
	54	'g': \
45	55	case 'h': \
46	56	case 'i': \
47	57	case 'j': \
...	...	@@ -49,8 +59,6 @@
49	59	case 'l': \
50	60	case 'm': \
51	61	case 'n': \
52		case 'o': \
53		case 'p': \
54	62	case 'q': \
55	63	case 'r': \
56	64	case 's': \
...	...	@@ -58,15 +66,8 @@
58	66	case 'u': \
59	67	case 'v': \
60	68	case 'w': \
61		case 'x': \
62	69	case 'y': \
63	70	case 'z': \
64		case 'A': \
65		case 'B': \
66		case 'C': \
67		case 'D': \
68		case 'E': \
69		case 'F': \
70	71	case 'G': \
71	72	case 'H': \
72	73	case 'I': \
...	...	@@ -76,7 +77,6 @@
76	77	case 'M': \
77	78	case 'N': \
78	79	case 'O': \
79		case 'P': \
80	80	case 'Q': \
81	81	case 'R': \
82	82	case 'S': \
...	...	@@ -88,7 +88,46 @@
88	88	case 'Y': \
89	89	case 'Z'
90	90
91		#define SYMBOL_CHAR \
	91	#define ALPHA_EXCEPT_E_B_O_X \
	92	ALPHA_EXCEPT_HEX_P_O_X: \
	93	case 'a': \
	94	case 'c': \
	95	case 'd': \
	96	case 'f': \
	97	case 'A': \
	98	case 'B': \
	99	case 'C': \
	100	case 'D': \
	101	case 'F': \
	102	case 'p': \
	103	case 'P'
	104
	105	#define ALPHA_EXCEPT_HEX_AND_P \
	106	ALPHA_EXCEPT_HEX_P_O_X: \
	107	case 'o': \
	108	case 'x'
	109
	110	#define ALPHA_EXCEPT_E \
	111	ALPHA_EXCEPT_HEX_AND_P: \
	112	case 'a': \
	113	case 'b': \
	114	case 'c': \
	115	case 'd': \
	116	case 'f': \
	117	case 'A': \
	118	case 'B': \
	119	case 'C': \
	120	case 'D': \
	121	case 'F': \
	122	case 'p': \
	123	case 'P'
	124
	125	#define ALPHA \
	126	ALPHA_EXCEPT_E: \
	127	case 'e': \
	128	case 'E'
	129
	130	#define IDENTIFIER_CHAR \
92	131	ALPHA: \
93	132	case DIGIT: \
94	133	case '_'
...	...	@@ -157,101 +196,92 @@ static const struct ZigKeyword zig_keywords[] = {
157	196	{"while", TokenIdKeywordWhile},
158	197	};
159	198
160		bool is_zig_keyword(Buf *buf) {
	199	// Returns TokenIdIdentifier if it is not a keyword.
	200	static TokenId zig_keyword_token(const char *name_ptr, size_t name_len) {
161	201	for (size_t i = 0; i < array_length(zig_keywords); i += 1) {
162		if (buf_eql_str(buf, zig_keywords[i].text)) {
163		return true;
	202	if (mem_eql_str(name_ptr, name_len, zig_keywords[i].text)) {
	203	return zig_keywords[i].token_id;
164	204	}
165	205	}
166		return false;
167		}
168
169		static bool is_symbol_char(uint8_t c) {
170		switch (c) {
171		case SYMBOL_CHAR:
172		return true;
173		default:
174		return false;
175		}
	206	return TokenIdIdentifier;
176	207	}
177	208
178	209	enum TokenizeState {
179		TokenizeStateStart,
180		TokenizeStateSymbol,
181		TokenizeStateZero, // "0", which might lead to "0x"
182		TokenizeStateNumber, // "123", "0x123"
183		TokenizeStateNumberNoUnderscore, // "12_", "0x12_" next char must be digit
184		TokenizeStateNumberDot,
185		TokenizeStateFloatFraction, // "123.456", "0x123.456"
186		TokenizeStateFloatFractionNoUnderscore, // "123.45_", "0x123.45_"
187		TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"
188		TokenizeStateFloatExponentNumber, // "123.456e7", "123.456e+7", "123.456e-7"
189		TokenizeStateFloatExponentNumberNoUnderscore, // "123.456e7_", "123.456e+7_", "123.456e-7_"
190		TokenizeStateString,
191		TokenizeStateStringEscape,
192		TokenizeStateStringEscapeUnicodeStart,
193		TokenizeStateCharLiteral,
194		TokenizeStateCharLiteralEnd,
195		TokenizeStateCharLiteralUnicode,
196		TokenizeStateSawStar,
197		TokenizeStateSawStarPercent,
198		TokenizeStateSawSlash,
199		TokenizeStateSawSlash2,
200		TokenizeStateSawSlash3,
201		TokenizeStateSawSlashBang,
202		TokenizeStateSawBackslash,
203		TokenizeStateSawPercent,
204		TokenizeStateSawPlus,
205		TokenizeStateSawPlusPercent,
206		TokenizeStateSawDash,
207		TokenizeStateSawMinusPercent,
208		TokenizeStateSawAmpersand,
209		TokenizeStateSawCaret,
210		TokenizeStateSawBar,
211		TokenizeStateDocComment,
212		TokenizeStateContainerDocComment,
213		TokenizeStateLineComment,
214		TokenizeStateLineString,
215		TokenizeStateLineStringEnd,
216		TokenizeStateLineStringContinue,
217		TokenizeStateSawEq,
218		TokenizeStateSawBang,
219		TokenizeStateSawLessThan,
220		TokenizeStateSawLessThanLessThan,
221		TokenizeStateSawGreaterThan,
222		TokenizeStateSawGreaterThanGreaterThan,
223		TokenizeStateSawDot,
224		TokenizeStateSawDotDot,
225		TokenizeStateSawDotStar,
226		TokenizeStateSawAtSign,
227		TokenizeStateCharCode,
228		TokenizeStateError,
	210	TokenizeState_start,
	211	TokenizeState_identifier,
	212	TokenizeState_builtin,
	213	TokenizeState_string_literal,
	214	TokenizeState_string_literal_backslash,
	215	TokenizeState_multiline_string_literal_line,
	216	TokenizeState_char_literal,
	217	TokenizeState_char_literal_backslash,
	218	TokenizeState_char_literal_hex_escape,
	219	TokenizeState_char_literal_unicode_escape_saw_u,
	220	TokenizeState_char_literal_unicode_escape,
	221	TokenizeState_char_literal_unicode,
	222	TokenizeState_char_literal_end,
	223	TokenizeState_backslash,
	224	TokenizeState_equal,
	225	TokenizeState_bang,
	226	TokenizeState_pipe,
	227	TokenizeState_minus,
	228	TokenizeState_minus_percent,
	229	TokenizeState_asterisk,
	230	TokenizeState_asterisk_percent,
	231	TokenizeState_slash,
	232	TokenizeState_line_comment_start,
	233	TokenizeState_line_comment,
	234	TokenizeState_doc_comment_start,
	235	TokenizeState_doc_comment,
	236	TokenizeState_container_doc_comment,
	237	TokenizeState_zero,
	238	TokenizeState_int_literal_dec,
	239	TokenizeState_int_literal_dec_no_underscore,
	240	TokenizeState_int_literal_bin,
	241	TokenizeState_int_literal_bin_no_underscore,
	242	TokenizeState_int_literal_oct,
	243	TokenizeState_int_literal_oct_no_underscore,
	244	TokenizeState_int_literal_hex,
	245	TokenizeState_int_literal_hex_no_underscore,
	246	TokenizeState_num_dot_dec,
	247	TokenizeState_num_dot_hex,
	248	TokenizeState_float_fraction_dec,
	249	TokenizeState_float_fraction_dec_no_underscore,
	250	TokenizeState_float_fraction_hex,
	251	TokenizeState_float_fraction_hex_no_underscore,
	252	TokenizeState_float_exponent_unsigned,
	253	TokenizeState_float_exponent_num,
	254	TokenizeState_float_exponent_num_no_underscore,
	255	TokenizeState_ampersand,
	256	TokenizeState_caret,
	257	TokenizeState_percent,
	258	TokenizeState_plus,
	259	TokenizeState_plus_percent,
	260	TokenizeState_angle_bracket_left,
	261	TokenizeState_angle_bracket_angle_bracket_left,
	262	TokenizeState_angle_bracket_right,
	263	TokenizeState_angle_bracket_angle_bracket_right,
	264	TokenizeState_period,
	265	TokenizeState_period_2,
	266	TokenizeState_period_asterisk,
	267	TokenizeState_saw_at_sign,
	268	TokenizeState_error,
229	269	};
230	270
231	271
232	272	struct Tokenize {
233		Buf *buf;
	273	Tokenization *out;
234	274	size_t pos;
235	275	TokenizeState state;
236		ZigList<Token> *tokens;
237		int line;
238		int column;
239		Token *cur_tok;
240		Tokenization *out;
241		uint32_t radix;
242		bool is_trailing_underscore;
243		size_t char_code_index;
244		bool unicode;
245		uint32_t char_code;
246		size_t remaining_code_units;
	276	uint32_t line;
	277	uint32_t column;
247	278	};
248	279
249	280	ATTRIBUTE_PRINTF(2, 3)
250	281	static void tokenize_error(Tokenize t, const char format, ...) {
251		t->state = TokenizeStateError;
	282	t->state = TokenizeState_error;
252	283
253		t->out->err_line = t->line;
254		t->out->err_column = t->column;
	284	t->out->err_byte_offset = t->pos;
255	285
256	286	va_list ap;
257	287	va_start(ap, format);
...	...	@@ -259,98 +289,18 @@ static void tokenize_error(Tokenize t, const char format, ...) {
259	289	va_end(ap);
260	290	}
261	291
262		static void set_token_id(Tokenize t, Token token, TokenId id) {
263		token->id = id;
264
265		if (id == TokenIdIntLiteral) {
266		bigint_init_unsigned(&token->data.int_lit.bigint, 0);
267		} else if (id == TokenIdFloatLiteral) {
268		bigfloat_init_32(&token->data.float_lit.bigfloat, 0.0f);
269		token->data.float_lit.overflow = false;
270		} else if (id == TokenIdStringLiteral \|\| id == TokenIdMultilineStringLiteral \|\| id == TokenIdSymbol) {
271		memset(&token->data.str_lit.str, 0, sizeof(Buf));
272		buf_resize(&token->data.str_lit.str, 0);
273		}
274		}
275
276	292	static void begin_token(Tokenize *t, TokenId id) {
277		assert(!t->cur_tok);
278		t->tokens->add_one();
279		Token *token = &t->tokens->last();
280		token->start_line = t->line;
281		token->start_column = t->column;
282		token->start_pos = t->pos;
283
284		set_token_id(t, token, id);
285
286		t->cur_tok = token;
	293	t->out->ids.append(id);
	294	t->out->locs.append({
	295	.offset = (uint32_t) t->pos,
	296	.line = t->line,
	297	.column = t->column,
	298	});
287	299	}
288	300
289	301	static void cancel_token(Tokenize *t) {
290		t->tokens->pop();
291		t->cur_tok = nullptr;
292		}
293
294		static void end_float_token(Tokenize *t) {
295		uint8_t ptr_buf = (uint8_t)buf_ptr(t->buf) + t->cur_tok->start_pos;
296		size_t buf_len = t->cur_tok->end_pos - t->cur_tok->start_pos;
297		if (bigfloat_init_buf(&t->cur_tok->data.float_lit.bigfloat, ptr_buf, buf_len)) {
298		t->cur_tok->data.float_lit.overflow = true;
299		}
300		}
301
302		static void end_token(Tokenize *t) {
303		assert(t->cur_tok);
304		t->cur_tok->end_pos = t->pos + 1;
305
306		if (t->cur_tok->id == TokenIdFloatLiteral) {
307		end_float_token(t);
308		} else if (t->cur_tok->id == TokenIdSymbol) {
309		char *token_mem = buf_ptr(t->buf) + t->cur_tok->start_pos;
310		int token_len = (int)(t->cur_tok->end_pos - t->cur_tok->start_pos);
311
312		for (size_t i = 0; i < array_length(zig_keywords); i += 1) {
313		if (mem_eql_str(token_mem, token_len, zig_keywords[i].text)) {
314		t->cur_tok->id = zig_keywords[i].token_id;
315		break;
316		}
317		}
318		}
319
320		t->cur_tok = nullptr;
321		}
322
323		static bool is_exponent_signifier(uint8_t c, int radix) {
324		if (radix == 16) {
325		return c == 'p' \|\| c == 'P';
326		} else {
327		return c == 'e' \|\| c == 'E';
328		}
329		}
330
331		static uint32_t get_digit_value(uint8_t c) {
332		if ('0' <= c && c <= '9') {
333		return c - '0';
334		}
335		if ('A' <= c && c <= 'Z') {
336		return c - 'A' + 10;
337		}
338		if ('a' <= c && c <= 'z') {
339		return c - 'a' + 10;
340		}
341		return UINT32_MAX;
342		}
343
344		static void handle_string_escape(Tokenize *t, uint8_t c) {
345		if (t->cur_tok->id == TokenIdCharLiteral) {
346		t->cur_tok->data.char_lit.c = c;
347		t->state = TokenizeStateCharLiteralEnd;
348		} else if (t->cur_tok->id == TokenIdStringLiteral \|\| t->cur_tok->id == TokenIdSymbol) {
349		buf_append_char(&t->cur_tok->data.str_lit.str, c);
350		t->state = TokenizeStateString;
351		} else {
352		zig_unreachable();
353		}
	302	t->out->ids.pop();
	303	t->out->locs.pop();
354	304	}
355	305
356	306	static const char* get_escape_shorthand(uint8_t c) {
...	...	@@ -376,7 +326,15 @@ static const char* get_escape_shorthand(uint8_t c) {
376	326	}
377	327	}
378	328
	329	static void invalid_eof(Tokenize *t) {
	330	return tokenize_error(t, "unexpected End-Of-File");
	331	}
	332
379	333	static void invalid_char_error(Tokenize *t, uint8_t c) {
	334	if (c == 0) {
	335	return invalid_eof(t);
	336	}
	337
380	338	if (c == '\r') {
381	339	tokenize_error(t, "invalid carriage return, only '\\n' line endings are supported");
382	340	return;
...	...	@@ -396,1139 +354,1092 @@ static void invalid_char_error(Tokenize *t, uint8_t c) {
396	354	tokenize_error(t, "invalid character: '\\x%02x'", c);
397	355	}
398	356
399		void tokenize(Buf buf, Tokenization out) {
	357	void tokenize(const char source, Tokenization out) {
400	358	Tokenize t = {0};
401	359	t.out = out;
402		t.tokens = out->tokens = heap::c_allocator.create<ZigList<Token>>();
403		t.buf = buf;
404	360
405		out->line_offsets = heap::c_allocator.create<ZigList<size_t>>();
406		out->line_offsets->append(0);
	361	size_t remaining_code_units;
	362	size_t seen_escape_digits;
407	363
408		// Skip the UTF-8 BOM if present
409		if (buf_starts_with_mem(buf, "\xEF\xBB\xBF", 3)) {
	364	// Skip the UTF-8 BOM if present.
	365	if (source[0] == (char)0xef &&
	366	source[1] == (char)0xbb &&
	367	source[2] == (char)0xbf)
	368	{
410	369	t.pos += 3;
411	370	}
412	371
413		for (; t.pos < buf_len(t.buf); t.pos += 1) {
414		uint8_t c = buf_ptr(t.buf)[t.pos];
	372	// Invalid token takes up index 0 so that index 0 can mean "none".
	373	begin_token(&t, TokenIdCount);
	374
	375	for (;;) {
	376	uint8_t c = source[t.pos];
415	377	switch (t.state) {
416		case TokenizeStateError:
417		break;
418		case TokenizeStateStart:
	378	case TokenizeState_error:
	379	goto eof;
	380	case TokenizeState_start:
419	381	switch (c) {
	382	case 0:
	383	goto eof;
420	384	case WHITESPACE:
421	385	break;
	386	case '"':
	387	begin_token(&t, TokenIdStringLiteral);
	388	t.state = TokenizeState_string_literal;
	389	break;
	390	case '\'':
	391	begin_token(&t, TokenIdCharLiteral);
	392	t.state = TokenizeState_char_literal;
	393	break;
422	394	case ALPHA:
423	395	case '_':
424		t.state = TokenizeStateSymbol;
425		begin_token(&t, TokenIdSymbol);
426		buf_append_char(&t.cur_tok->data.str_lit.str, c);
	396	t.state = TokenizeState_identifier;
	397	begin_token(&t, TokenIdIdentifier);
427	398	break;
428		case '0':
429		t.state = TokenizeStateZero;
430		begin_token(&t, TokenIdIntLiteral);
431		t.is_trailing_underscore = false;
432		t.radix = 10;
433		bigint_init_unsigned(&t.cur_tok->data.int_lit.bigint, 0);
	399	case '@':
	400	begin_token(&t, TokenIdBuiltin);
	401	t.state = TokenizeState_saw_at_sign;
434	402	break;
435		case DIGIT_NON_ZERO:
436		t.state = TokenizeStateNumber;
437		begin_token(&t, TokenIdIntLiteral);
438		t.is_trailing_underscore = false;
439		t.radix = 10;
440		bigint_init_unsigned(&t.cur_tok->data.int_lit.bigint, get_digit_value(c));
	403	case '=':
	404	begin_token(&t, TokenIdEq);
	405	t.state = TokenizeState_equal;
441	406	break;
442		case '"':
443		begin_token(&t, TokenIdStringLiteral);
444		t.state = TokenizeStateString;
	407	case '!':
	408	begin_token(&t, TokenIdBang);
	409	t.state = TokenizeState_bang;
445	410	break;
446		case '\'':
447		begin_token(&t, TokenIdCharLiteral);
448		t.state = TokenizeStateCharLiteral;
	411	case '\|':
	412	begin_token(&t, TokenIdBinOr);
	413	t.state = TokenizeState_pipe;
449	414	break;
450	415	case '(':
451	416	begin_token(&t, TokenIdLParen);
452		end_token(&t);
453	417	break;
454	418	case ')':
455	419	begin_token(&t, TokenIdRParen);
456		end_token(&t);
457		break;
458		case ',':
459		begin_token(&t, TokenIdComma);
460		end_token(&t);
461		break;
462		case '?':
463		begin_token(&t, TokenIdQuestion);
464		end_token(&t);
465		break;
466		case '{':
467		begin_token(&t, TokenIdLBrace);
468		end_token(&t);
469		break;
470		case '}':
471		begin_token(&t, TokenIdRBrace);
472		end_token(&t);
473	420	break;
474	421	case '[':
475	422	begin_token(&t, TokenIdLBracket);
476		end_token(&t);
477	423	break;
478	424	case ']':
479	425	begin_token(&t, TokenIdRBracket);
480		end_token(&t);
481	426	break;
482	427	case ';':
483	428	begin_token(&t, TokenIdSemicolon);
484		end_token(&t);
	429	break;
	430	case ',':
	431	begin_token(&t, TokenIdComma);
	432	break;
	433	case '?':
	434	begin_token(&t, TokenIdQuestion);
485	435	break;
486	436	case ':':
487	437	begin_token(&t, TokenIdColon);
488		end_token(&t);
489	438	break;
490		case '#':
491		begin_token(&t, TokenIdNumberSign);
492		end_token(&t);
	439	case '%':
	440	begin_token(&t, TokenIdPercent);
	441	t.state = TokenizeState_percent;
493	442	break;
494	443	case '*':
495	444	begin_token(&t, TokenIdStar);
496		t.state = TokenizeStateSawStar;
	445	t.state = TokenizeState_asterisk;
497	446	break;
498		case '/':
499		begin_token(&t, TokenIdSlash);
500		t.state = TokenizeStateSawSlash;
	447	case '+':
	448	begin_token(&t, TokenIdPlus);
	449	t.state = TokenizeState_plus;
	450	break;
	451	case '<':
	452	begin_token(&t, TokenIdCmpLessThan);
	453	t.state = TokenizeState_angle_bracket_left;
	454	break;
	455	case '>':
	456	begin_token(&t, TokenIdCmpGreaterThan);
	457	t.state = TokenizeState_angle_bracket_right;
	458	break;
	459	case '^':
	460	begin_token(&t, TokenIdBinXor);
	461	t.state = TokenizeState_caret;
501	462	break;
502	463	case '\\':
503		begin_token(&t, TokenIdMultilineStringLiteral);
504		t.state = TokenizeStateSawBackslash;
	464	begin_token(&t, TokenIdMultilineStringLiteralLine);
	465	t.state = TokenizeState_backslash;
505	466	break;
506		case '%':
507		begin_token(&t, TokenIdPercent);
508		t.state = TokenizeStateSawPercent;
	467	case '{':
	468	begin_token(&t, TokenIdLBrace);
509	469	break;
510		case '+':
511		begin_token(&t, TokenIdPlus);
512		t.state = TokenizeStateSawPlus;
	470	case '}':
	471	begin_token(&t, TokenIdRBrace);
513	472	break;
514	473	case '~':
515	474	begin_token(&t, TokenIdTilde);
516		end_token(&t);
517	475	break;
518		case '@':
519		begin_token(&t, TokenIdAtSign);
520		t.state = TokenizeStateSawAtSign;
	476	case '.':
	477	begin_token(&t, TokenIdDot);
	478	t.state = TokenizeState_period;
521	479	break;
522	480	case '-':
523	481	begin_token(&t, TokenIdDash);
524		t.state = TokenizeStateSawDash;
	482	t.state = TokenizeState_minus;
	483	break;
	484	case '/':
	485	begin_token(&t, TokenIdSlash);
	486	t.state = TokenizeState_slash;
525	487	break;
526	488	case '&':
527	489	begin_token(&t, TokenIdAmpersand);
528		t.state = TokenizeStateSawAmpersand;
	490	t.state = TokenizeState_ampersand;
529	491	break;
530		case '^':
531		begin_token(&t, TokenIdBinXor);
532		t.state = TokenizeStateSawCaret;
533		break;
534		case '\|':
535		begin_token(&t, TokenIdBinOr);
536		t.state = TokenizeStateSawBar;
	492	case '0':
	493	t.state = TokenizeState_zero;
	494	begin_token(&t, TokenIdIntLiteral);
537	495	break;
538		case '=':
539		begin_token(&t, TokenIdEq);
540		t.state = TokenizeStateSawEq;
	496	case DIGIT_NON_ZERO:
	497	t.state = TokenizeState_int_literal_dec;
	498	begin_token(&t, TokenIdIntLiteral);
541	499	break;
542		case '!':
543		begin_token(&t, TokenIdBang);
544		t.state = TokenizeStateSawBang;
	500	default:
	501	invalid_char_error(&t, c);
	502	}
	503	break;
	504	case TokenizeState_saw_at_sign:
	505	switch (c) {
	506	case 0:
	507	invalid_eof(&t);
	508	goto eof;
	509	case '"':
	510	t.out->ids.last() = TokenIdIdentifier;
	511	t.state = TokenizeState_string_literal;
545	512	break;
546		case '<':
547		begin_token(&t, TokenIdCmpLessThan);
548		t.state = TokenizeStateSawLessThan;
	513	case IDENTIFIER_CHAR:
	514	t.state = TokenizeState_builtin;
549	515	break;
550		case '>':
551		begin_token(&t, TokenIdCmpGreaterThan);
552		t.state = TokenizeStateSawGreaterThan;
	516	default:
	517	invalid_char_error(&t, c);
	518	}
	519	break;
	520	case TokenizeState_ampersand:
	521	switch (c) {
	522	case 0:
	523	goto eof;
	524	case '&':
	525	tokenize_error(&t, "`&&` is invalid. Note that `and` is boolean AND");
553	526	break;
554		case '.':
555		begin_token(&t, TokenIdDot);
556		t.state = TokenizeStateSawDot;
	527	case '=':
	528	t.out->ids.last() = TokenIdBitAndEq;
	529	t.state = TokenizeState_start;
557	530	break;
558	531	default:
559		invalid_char_error(&t, c);
	532	t.state = TokenizeState_start;
	533	continue;
560	534	}
561	535	break;
562		case TokenizeStateSawDot:
	536	case TokenizeState_asterisk:
563	537	switch (c) {
564		case '.':
565		t.state = TokenizeStateSawDotDot;
566		set_token_id(&t, t.cur_tok, TokenIdEllipsis2);
	538	case 0:
	539	goto eof;
	540	case '=':
	541	t.out->ids.last() = TokenIdTimesEq;
	542	t.state = TokenizeState_start;
567	543	break;
568	544	case '*':
569		t.state = TokenizeStateSawDotStar;
570		set_token_id(&t, t.cur_tok, TokenIdDotStar);
	545	t.out->ids.last() = TokenIdStarStar;
	546	t.state = TokenizeState_start;
	547	break;
	548	case '%':
	549	t.state = TokenizeState_asterisk_percent;
571	550	break;
572	551	default:
573		t.pos -= 1;
574		end_token(&t);
575		t.state = TokenizeStateStart;
	552	t.state = TokenizeState_start;
576	553	continue;
577	554	}
578	555	break;
579		case TokenizeStateSawDotDot:
	556	case TokenizeState_asterisk_percent:
580	557	switch (c) {
581		case '.':
582		t.state = TokenizeStateStart;
583		set_token_id(&t, t.cur_tok, TokenIdEllipsis3);
584		end_token(&t);
	558	case 0:
	559	t.out->ids.last() = TokenIdTimesPercent;
	560	goto eof;
	561	case '=':
	562	t.out->ids.last() = TokenIdTimesPercentEq;
	563	t.state = TokenizeState_start;
585	564	break;
586	565	default:
587		t.pos -= 1;
588		end_token(&t);
589		t.state = TokenizeStateStart;
	566	t.out->ids.last() = TokenIdTimesPercent;
	567	t.state = TokenizeState_start;
590	568	continue;
591	569	}
592	570	break;
593		case TokenizeStateSawDotStar:
	571	case TokenizeState_percent:
594	572	switch (c) {
595		case '*':
596		tokenize_error(&t, "`.` can't be followed by ``. Are you missing a space?");
	573	case 0:
	574	goto eof;
	575	case '=':
	576	t.out->ids.last() = TokenIdModEq;
	577	t.state = TokenizeState_start;
597	578	break;
598	579	default:
599		t.pos -= 1;
600		end_token(&t);
601		t.state = TokenizeStateStart;
	580	t.state = TokenizeState_start;
602	581	continue;
603	582	}
604	583	break;
605		case TokenizeStateSawGreaterThan:
	584	case TokenizeState_plus:
606	585	switch (c) {
	586	case 0:
	587	goto eof;
607	588	case '=':
608		set_token_id(&t, t.cur_tok, TokenIdCmpGreaterOrEq);
609		end_token(&t);
610		t.state = TokenizeStateStart;
	589	t.out->ids.last() = TokenIdPlusEq;
	590	t.state = TokenizeState_start;
611	591	break;
612		case '>':
613		set_token_id(&t, t.cur_tok, TokenIdBitShiftRight);
614		t.state = TokenizeStateSawGreaterThanGreaterThan;
	592	case '+':
	593	t.out->ids.last() = TokenIdPlusPlus;
	594	t.state = TokenizeState_start;
	595	break;
	596	case '%':
	597	t.state = TokenizeState_plus_percent;
615	598	break;
616	599	default:
617		t.pos -= 1;
618		end_token(&t);
619		t.state = TokenizeStateStart;
	600	t.state = TokenizeState_start;
620	601	continue;
621	602	}
622	603	break;
623		case TokenizeStateSawGreaterThanGreaterThan:
	604	case TokenizeState_plus_percent:
624	605	switch (c) {
	606	case 0:
	607	t.out->ids.last() = TokenIdPlusPercent;
	608	goto eof;
625	609	case '=':
626		set_token_id(&t, t.cur_tok, TokenIdBitShiftRightEq);
627		end_token(&t);
628		t.state = TokenizeStateStart;
	610	t.out->ids.last() = TokenIdPlusPercentEq;
	611	t.state = TokenizeState_start;
629	612	break;
630	613	default:
631		t.pos -= 1;
632		end_token(&t);
633		t.state = TokenizeStateStart;
	614	t.out->ids.last() = TokenIdPlusPercent;
	615	t.state = TokenizeState_start;
634	616	continue;
635	617	}
636	618	break;
637		case TokenizeStateSawLessThan:
	619	case TokenizeState_caret:
638	620	switch (c) {
	621	case 0:
	622	goto eof;
639	623	case '=':
640		set_token_id(&t, t.cur_tok, TokenIdCmpLessOrEq);
641		end_token(&t);
642		t.state = TokenizeStateStart;
	624	t.out->ids.last() = TokenIdBitXorEq;
	625	t.state = TokenizeState_start;
643	626	break;
644		case '<':
645		set_token_id(&t, t.cur_tok, TokenIdBitShiftLeft);
646		t.state = TokenizeStateSawLessThanLessThan;
	627	default:
	628	t.state = TokenizeState_start;
	629	continue;
	630	}
	631	break;
	632	case TokenizeState_identifier:
	633	switch (c) {
	634	case 0: {
	635	uint32_t start_pos = t.out->locs.last().offset;
	636	t.out->ids.last() = zig_keyword_token(
	637	source + start_pos, t.pos - start_pos);
	638	goto eof;
	639	}
	640	case IDENTIFIER_CHAR:
	641	break;
	642	default: {
	643	uint32_t start_pos = t.out->locs.last().offset;
	644	t.out->ids.last() = zig_keyword_token(
	645	source + start_pos, t.pos - start_pos);
	646
	647	t.state = TokenizeState_start;
	648	continue;
	649	}
	650	}
	651	break;
	652	case TokenizeState_builtin:
	653	switch (c) {
	654	case 0:
	655	goto eof;
	656	case IDENTIFIER_CHAR:
647	657	break;
648	658	default:
649		t.pos -= 1;
650		end_token(&t);
651		t.state = TokenizeStateStart;
	659	t.state = TokenizeState_start;
652	660	continue;
653	661	}
654	662	break;
655		case TokenizeStateSawLessThanLessThan:
	663	case TokenizeState_backslash:
	664	switch (c) {
	665	case '\\':
	666	t.state = TokenizeState_multiline_string_literal_line;
	667	break;
	668	default:
	669	invalid_char_error(&t, c);
	670	break;
	671	}
	672	break;
	673	case TokenizeState_string_literal:
	674	switch (c) {
	675	case 0:
	676	invalid_eof(&t);
	677	goto eof;
	678	case '\\':
	679	t.state = TokenizeState_string_literal_backslash;
	680	break;
	681	case '"':
	682	t.state = TokenizeState_start;
	683	break;
	684	case '\n':
	685	case '\r':
	686	tokenize_error(&t, "newline not allowed in string literal");
	687	break;
	688	default:
	689	break;
	690	}
	691	break;
	692	case TokenizeState_string_literal_backslash:
	693	switch (c) {
	694	case 0:
	695	invalid_eof(&t);
	696	goto eof;
	697	case '\n':
	698	case '\r':
	699	tokenize_error(&t, "newline not allowed in string literal");
	700	break;
	701	default:
	702	t.state = TokenizeState_string_literal;
	703	break;
	704	}
	705	break;
	706	case TokenizeState_char_literal:
	707	if (c == 0) {
	708	invalid_eof(&t);
	709	goto eof;
	710	} else if (c == '\\') {
	711	t.state = TokenizeState_char_literal_backslash;
	712	} else if (c == '\'') {
	713	tokenize_error(&t, "expected character");
	714	} else if ((c >= 0x80 && c <= 0xbf) \|\| c >= 0xf8) {
	715	// 10xxxxxx
	716	// 11111xxx
	717	invalid_char_error(&t, c);
	718	} else if (c >= 0xc0 && c <= 0xdf) {
	719	// 110xxxxx
	720	remaining_code_units = 1;
	721	t.state = TokenizeState_char_literal_unicode;
	722	} else if (c >= 0xe0 && c <= 0xef) {
	723	// 1110xxxx
	724	remaining_code_units = 2;
	725	t.state = TokenizeState_char_literal_unicode;
	726	} else if (c >= 0xf0 && c <= 0xf7) {
	727	// 11110xxx
	728	remaining_code_units = 3;
	729	t.state = TokenizeState_char_literal_unicode;
	730	} else {
	731	t.state = TokenizeState_char_literal_end;
	732	}
	733	break;
	734	case TokenizeState_char_literal_backslash:
	735	switch (c) {
	736	case 0:
	737	invalid_eof(&t);
	738	goto eof;
	739	case '\n':
	740	case '\r':
	741	tokenize_error(&t, "newline not allowed in character literal");
	742	break;
	743	case 'x':
	744	t.state = TokenizeState_char_literal_hex_escape;
	745	seen_escape_digits = 0;
	746	break;
	747	case 'u':
	748	t.state = TokenizeState_char_literal_unicode_escape_saw_u;
	749	break;
	750	default:
	751	t.state = TokenizeState_char_literal_end;
	752	break;
	753	}
	754	break;
	755	case TokenizeState_char_literal_hex_escape:
	756	switch (c) {
	757	case ALPHA:
	758	case DIGIT:
	759	seen_escape_digits += 1;
	760	if (seen_escape_digits == 2) {
	761	t.state = TokenizeState_char_literal_end;
	762	}
	763	break;
	764	default:
	765	tokenize_error(&t, "expected hex digit");
	766	break;
	767	}
	768	break;
	769	case TokenizeState_char_literal_unicode_escape_saw_u:
	770	switch (c) {
	771	case '{':
	772	t.state = TokenizeState_char_literal_unicode_escape;
	773	seen_escape_digits = 0;
	774	break;
	775	default:
	776	tokenize_error(&t, "expected '{' to begin unicode escape sequence");
	777	break;
	778	}
	779	break;
	780	case TokenizeState_char_literal_unicode_escape:
	781	switch (c) {
	782	case ALPHA:
	783	case DIGIT:
	784	seen_escape_digits += 1;
	785	break;
	786	case '}':
	787	if (seen_escape_digits == 0) {
	788	tokenize_error(&t, "missing unicode escape sequence");
	789	break;
	790	}
	791	t.state = TokenizeState_char_literal_end;
	792	break;
	793	default:
	794	tokenize_error(&t, "expected hex digit");
	795	break;
	796	}
	797	break;
	798	case TokenizeState_char_literal_end:
	799	switch (c) {
	800	case '\'':
	801	t.state = TokenizeState_start;
	802	break;
	803	default:
	804	invalid_char_error(&t, c);
	805	break;
	806	}
	807	break;
	808	case TokenizeState_char_literal_unicode:
	809	if (c >= 0x80 && c <= 0xbf) {
	810	remaining_code_units -= 1;
	811	if (remaining_code_units == 0) {
	812	t.state = TokenizeState_char_literal_end;
	813	}
	814	} else {
	815	invalid_char_error(&t, c);
	816	}
	817	break;
	818	case TokenizeState_multiline_string_literal_line:
	819	switch (c) {
	820	case 0:
	821	goto eof;
	822	case '\n':
	823	t.state = TokenizeState_start;
	824	break;
	825	default:
	826	break;
	827	}
	828	break;
	829	case TokenizeState_bang:
656	830	switch (c) {
	831	case 0:
	832	goto eof;
657	833	case '=':
658		set_token_id(&t, t.cur_tok, TokenIdBitShiftLeftEq);
659		end_token(&t);
660		t.state = TokenizeStateStart;
	834	t.out->ids.last() = TokenIdCmpNotEq;
	835	t.state = TokenizeState_start;
661	836	break;
662	837	default:
663		t.pos -= 1;
664		end_token(&t);
665		t.state = TokenizeStateStart;
	838	t.state = TokenizeState_start;
666	839	continue;
667	840	}
668	841	break;
669		case TokenizeStateSawBang:
	842	case TokenizeState_pipe:
670	843	switch (c) {
	844	case 0:
	845	goto eof;
671	846	case '=':
672		set_token_id(&t, t.cur_tok, TokenIdCmpNotEq);
673		end_token(&t);
674		t.state = TokenizeStateStart;
	847	t.out->ids.last() = TokenIdBitOrEq;
	848	t.state = TokenizeState_start;
	849	break;
	850	case '\|':
	851	t.out->ids.last() = TokenIdBarBar;
	852	t.state = TokenizeState_start;
675	853	break;
676	854	default:
677		t.pos -= 1;
678		end_token(&t);
679		t.state = TokenizeStateStart;
	855	t.state = TokenizeState_start;
680	856	continue;
681	857	}
682	858	break;
683		case TokenizeStateSawEq:
	859	case TokenizeState_equal:
684	860	switch (c) {
	861	case 0:
	862	goto eof;
685	863	case '=':
686		set_token_id(&t, t.cur_tok, TokenIdCmpEq);
687		end_token(&t);
688		t.state = TokenizeStateStart;
	864	t.out->ids.last() = TokenIdCmpEq;
	865	t.state = TokenizeState_start;
689	866	break;
690	867	case '>':
691		set_token_id(&t, t.cur_tok, TokenIdFatArrow);
692		end_token(&t);
693		t.state = TokenizeStateStart;
	868	t.out->ids.last() = TokenIdFatArrow;
	869	t.state = TokenizeState_start;
	870	break;
	871	default:
	872	t.state = TokenizeState_start;
	873	continue;
	874	}
	875	break;
	876	case TokenizeState_minus:
	877	switch (c) {
	878	case 0:
	879	goto eof;
	880	case '>':
	881	t.out->ids.last() = TokenIdArrow;
	882	t.state = TokenizeState_start;
	883	break;
	884	case '=':
	885	t.out->ids.last() = TokenIdMinusEq;
	886	t.state = TokenizeState_start;
	887	break;
	888	case '%':
	889	t.state = TokenizeState_minus_percent;
694	890	break;
695	891	default:
696		t.pos -= 1;
697		end_token(&t);
698		t.state = TokenizeStateStart;
	892	t.state = TokenizeState_start;
699	893	continue;
700	894	}
701	895	break;
702		case TokenizeStateSawStar:
	896	case TokenizeState_minus_percent:
703	897	switch (c) {
	898	case 0:
	899	t.out->ids.last() = TokenIdMinusPercent;
	900	goto eof;
704	901	case '=':
705		set_token_id(&t, t.cur_tok, TokenIdTimesEq);
706		end_token(&t);
707		t.state = TokenizeStateStart;
708		break;
709		case '*':
710		set_token_id(&t, t.cur_tok, TokenIdStarStar);
711		end_token(&t);
712		t.state = TokenizeStateStart;
713		break;
714		case '%':
715		set_token_id(&t, t.cur_tok, TokenIdTimesPercent);
716		t.state = TokenizeStateSawStarPercent;
	902	t.out->ids.last() = TokenIdMinusPercentEq;
	903	t.state = TokenizeState_start;
717	904	break;
718	905	default:
719		t.pos -= 1;
720		end_token(&t);
721		t.state = TokenizeStateStart;
	906	t.out->ids.last() = TokenIdMinusPercent;
	907	t.state = TokenizeState_start;
722	908	continue;
723	909	}
724	910	break;
725		case TokenizeStateSawStarPercent:
	911	case TokenizeState_angle_bracket_left:
726	912	switch (c) {
	913	case 0:
	914	goto eof;
727	915	case '=':
728		set_token_id(&t, t.cur_tok, TokenIdTimesPercentEq);
729		end_token(&t);
730		t.state = TokenizeStateStart;
	916	t.out->ids.last() = TokenIdCmpLessOrEq;
	917	t.state = TokenizeState_start;
	918	break;
	919	case '<':
	920	t.state = TokenizeState_angle_bracket_angle_bracket_left;
731	921	break;
732	922	default:
733		t.pos -= 1;
734		end_token(&t);
735		t.state = TokenizeStateStart;
	923	t.state = TokenizeState_start;
736	924	continue;
737	925	}
738	926	break;
739		case TokenizeStateSawPercent:
	927	case TokenizeState_angle_bracket_angle_bracket_left:
740	928	switch (c) {
	929	case 0:
	930	t.out->ids.last() = TokenIdBitShiftLeft;
	931	goto eof;
741	932	case '=':
742		set_token_id(&t, t.cur_tok, TokenIdModEq);
743		end_token(&t);
744		t.state = TokenizeStateStart;
745		break;
746		case '.':
747		set_token_id(&t, t.cur_tok, TokenIdPercentDot);
748		end_token(&t);
749		t.state = TokenizeStateStart;
	933	t.out->ids.last() = TokenIdBitShiftLeftEq;
	934	t.state = TokenizeState_start;
750	935	break;
751	936	default:
752		t.pos -= 1;
753		end_token(&t);
754		t.state = TokenizeStateStart;
	937	t.out->ids.last() = TokenIdBitShiftLeft;
	938	t.state = TokenizeState_start;
755	939	continue;
756	940	}
757	941	break;
758		case TokenizeStateSawPlus:
	942	case TokenizeState_angle_bracket_right:
759	943	switch (c) {
	944	case 0:
	945	goto eof;
760	946	case '=':
761		set_token_id(&t, t.cur_tok, TokenIdPlusEq);
762		end_token(&t);
763		t.state = TokenizeStateStart;
	947	t.out->ids.last() = TokenIdCmpGreaterOrEq;
	948	t.state = TokenizeState_start;
764	949	break;
765		case '+':
766		set_token_id(&t, t.cur_tok, TokenIdPlusPlus);
767		end_token(&t);
768		t.state = TokenizeStateStart;
769		break;
770		case '%':
771		set_token_id(&t, t.cur_tok, TokenIdPlusPercent);
772		t.state = TokenizeStateSawPlusPercent;
	950	case '>':
	951	t.state = TokenizeState_angle_bracket_angle_bracket_right;
773	952	break;
774	953	default:
775		t.pos -= 1;
776		end_token(&t);
777		t.state = TokenizeStateStart;
	954	t.state = TokenizeState_start;
778	955	continue;
779	956	}
780	957	break;
781		case TokenizeStateSawPlusPercent:
	958	case TokenizeState_angle_bracket_angle_bracket_right:
782	959	switch (c) {
	960	case 0:
	961	t.out->ids.last() = TokenIdBitShiftRight;
	962	goto eof;
783	963	case '=':
784		set_token_id(&t, t.cur_tok, TokenIdPlusPercentEq);
785		end_token(&t);
786		t.state = TokenizeStateStart;
	964	t.out->ids.last() = TokenIdBitShiftRightEq;
	965	t.state = TokenizeState_start;
787	966	break;
788	967	default:
789		t.pos -= 1;
790		end_token(&t);
791		t.state = TokenizeStateStart;
	968	t.out->ids.last() = TokenIdBitShiftRight;
	969	t.state = TokenizeState_start;
792	970	continue;
793	971	}
794	972	break;
795		case TokenizeStateSawAmpersand:
	973	case TokenizeState_period:
796	974	switch (c) {
797		case '&':
798		tokenize_error(&t, "`&&` is invalid. Note that `and` is boolean AND");
	975	case 0:
	976	goto eof;
	977	case '.':
	978	t.state = TokenizeState_period_2;
799	979	break;
800		case '=':
801		set_token_id(&t, t.cur_tok, TokenIdBitAndEq);
802		end_token(&t);
803		t.state = TokenizeStateStart;
	980	case '*':
	981	t.state = TokenizeState_period_asterisk;
804	982	break;
805	983	default:
806		t.pos -= 1;
807		end_token(&t);
808		t.state = TokenizeStateStart;
	984	t.state = TokenizeState_start;
809	985	continue;
810	986	}
811	987	break;
812		case TokenizeStateSawCaret:
	988	case TokenizeState_period_2:
813	989	switch (c) {
814		case '=':
815		set_token_id(&t, t.cur_tok, TokenIdBitXorEq);
816		end_token(&t);
817		t.state = TokenizeStateStart;
	990	case 0:
	991	t.out->ids.last() = TokenIdEllipsis2;
	992	goto eof;
	993	case '.':
	994	t.out->ids.last() = TokenIdEllipsis3;
	995	t.state = TokenizeState_start;
818	996	break;
819	997	default:
820		t.pos -= 1;
821		end_token(&t);
822		t.state = TokenizeStateStart;
	998	t.out->ids.last() = TokenIdEllipsis2;
	999	t.state = TokenizeState_start;
823	1000	continue;
824	1001	}
825	1002	break;
826		case TokenizeStateSawBar:
	1003	case TokenizeState_period_asterisk:
827	1004	switch (c) {
828		case '=':
829		set_token_id(&t, t.cur_tok, TokenIdBitOrEq);
830		end_token(&t);
831		t.state = TokenizeStateStart;
832		break;
833		case '\|':
834		set_token_id(&t, t.cur_tok, TokenIdBarBar);
835		end_token(&t);
836		t.state = TokenizeStateStart;
	1005	case 0:
	1006	t.out->ids.last() = TokenIdDotStar;
	1007	goto eof;
	1008	case '*':
	1009	tokenize_error(&t, "`.` cannot be followed by ``. Are you missing a space?");
837	1010	break;
838	1011	default:
839		t.pos -= 1;
840		end_token(&t);
841		t.state = TokenizeStateStart;
	1012	t.out->ids.last() = TokenIdDotStar;
	1013	t.state = TokenizeState_start;
842	1014	continue;
843	1015	}
844	1016	break;
845		case TokenizeStateSawSlash:
	1017	case TokenizeState_slash:
846	1018	switch (c) {
	1019	case 0:
	1020	goto eof;
847	1021	case '/':
848		t.state = TokenizeStateSawSlash2;
	1022	t.state = TokenizeState_line_comment_start;
849	1023	break;
850	1024	case '=':
851		set_token_id(&t, t.cur_tok, TokenIdDivEq);
852		end_token(&t);
853		t.state = TokenizeStateStart;
	1025	t.out->ids.last() = TokenIdDivEq;
	1026	t.state = TokenizeState_start;
854	1027	break;
855	1028	default:
856		t.pos -= 1;
857		end_token(&t);
858		t.state = TokenizeStateStart;
	1029	t.state = TokenizeState_start;
859	1030	continue;
860	1031	}
861	1032	break;
862		case TokenizeStateSawSlash2:
	1033	case TokenizeState_line_comment_start:
863	1034	switch (c) {
	1035	case 0:
	1036	goto eof;
864	1037	case '/':
865		t.state = TokenizeStateSawSlash3;
	1038	t.state = TokenizeState_doc_comment_start;
866	1039	break;
867	1040	case '!':
868		t.state = TokenizeStateSawSlashBang;
	1041	t.out->ids.last() = TokenIdContainerDocComment;
	1042	t.state = TokenizeState_container_doc_comment;
869	1043	break;
870	1044	case '\n':
871	1045	cancel_token(&t);
872		t.state = TokenizeStateStart;
	1046	t.state = TokenizeState_start;
873	1047	break;
874	1048	default:
875	1049	cancel_token(&t);
876		t.state = TokenizeStateLineComment;
	1050	t.state = TokenizeState_line_comment;
877	1051	break;
878	1052	}
879	1053	break;
880		case TokenizeStateSawSlash3:
	1054	case TokenizeState_doc_comment_start:
881	1055	switch (c) {
	1056	case 0:
	1057	t.out->ids.last() = TokenIdDocComment;
	1058	goto eof;
882	1059	case '/':
883	1060	cancel_token(&t);
884		t.state = TokenizeStateLineComment;
	1061	t.state = TokenizeState_line_comment;
885	1062	break;
886	1063	case '\n':
887		set_token_id(&t, t.cur_tok, TokenIdDocComment);
888		end_token(&t);
889		t.state = TokenizeStateStart;
	1064	t.out->ids.last() = TokenIdDocComment;
	1065	t.state = TokenizeState_start;
890	1066	break;
891	1067	default:
892		set_token_id(&t, t.cur_tok, TokenIdDocComment);
893		t.state = TokenizeStateDocComment;
	1068	t.out->ids.last() = TokenIdDocComment;
	1069	t.state = TokenizeState_doc_comment;
894	1070	break;
895	1071	}
896	1072	break;
897		case TokenizeStateSawSlashBang:
	1073	case TokenizeState_line_comment:
898	1074	switch (c) {
	1075	case 0:
	1076	goto eof;
899	1077	case '\n':
900		set_token_id(&t, t.cur_tok, TokenIdContainerDocComment);
901		end_token(&t);
902		t.state = TokenizeStateStart;
	1078	t.state = TokenizeState_start;
903	1079	break;
904	1080	default:
905		set_token_id(&t, t.cur_tok, TokenIdContainerDocComment);
906		t.state = TokenizeStateContainerDocComment;
907	1081	break;
908	1082	}
909	1083	break;
910		case TokenizeStateSawBackslash:
	1084	case TokenizeState_doc_comment:
	1085	case TokenizeState_container_doc_comment:
911	1086	switch (c) {
912		case '\\':
913		t.state = TokenizeStateLineString;
	1087	case 0:
	1088	goto eof;
	1089	case '\n':
	1090	t.state = TokenizeState_start;
914	1091	break;
915	1092	default:
916		invalid_char_error(&t, c);
	1093	// do nothing
917	1094	break;
918	1095	}
919	1096	break;
920		case TokenizeStateLineString:
	1097	case TokenizeState_zero:
921	1098	switch (c) {
922		case '\n':
923		t.state = TokenizeStateLineStringEnd;
	1099	case 0:
	1100	goto eof;
	1101	case 'b':
	1102	t.state = TokenizeState_int_literal_bin_no_underscore;
924	1103	break;
925		default:
926		buf_append_char(&t.cur_tok->data.str_lit.str, c);
	1104	case 'o':
	1105	t.state = TokenizeState_int_literal_oct_no_underscore;
927	1106	break;
928		}
929		break;
930		case TokenizeStateLineStringEnd:
931		switch (c) {
932		case WHITESPACE:
	1107	case 'x':
	1108	t.state = TokenizeState_int_literal_hex_no_underscore;
933	1109	break;
934		case '\\':
935		t.state = TokenizeStateLineStringContinue;
	1110	case DIGIT:
	1111	case '_':
	1112	case '.':
	1113	case 'e':
	1114	case 'E':
	1115	// Reinterpret as a decimal number.
	1116	t.state = TokenizeState_int_literal_dec;
	1117	continue;
	1118	case ALPHA_EXCEPT_E_B_O_X:
	1119	invalid_char_error(&t, c);
936	1120	break;
937	1121	default:
938		t.pos -= 1;
939		end_token(&t);
940		t.state = TokenizeStateStart;
	1122	t.state = TokenizeState_start;
941	1123	continue;
942	1124	}
943	1125	break;
944		case TokenizeStateLineStringContinue:
	1126	case TokenizeState_int_literal_bin_no_underscore:
945	1127	switch (c) {
946		case '\\':
947		t.state = TokenizeStateLineString;
948		buf_append_char(&t.cur_tok->data.str_lit.str, '\n');
	1128	case '0':
	1129	case '1':
	1130	t.state = TokenizeState_int_literal_bin;
949	1131	break;
950	1132	default:
951	1133	invalid_char_error(&t, c);
952		break;
953	1134	}
954	1135	break;
955		case TokenizeStateLineComment:
	1136	case TokenizeState_int_literal_bin:
956	1137	switch (c) {
957		case '\n':
958		t.state = TokenizeStateStart;
	1138	case 0:
	1139	goto eof;
	1140	case '_':
	1141	t.state = TokenizeState_int_literal_bin_no_underscore;
959	1142	break;
960		default:
961		// do nothing
	1143	case '0':
	1144	case '1':
	1145	break;
	1146	case '2':
	1147	case '3':
	1148	case '4':
	1149	case '5':
	1150	case '6':
	1151	case '7':
	1152	case '8':
	1153	case '9':
	1154	case ALPHA:
	1155	invalid_char_error(&t, c);
962	1156	break;
	1157	default:
	1158	t.state = TokenizeState_start;
	1159	continue;
963	1160	}
964	1161	break;
965		case TokenizeStateDocComment:
	1162	case TokenizeState_int_literal_oct_no_underscore:
966	1163	switch (c) {
967		case '\n':
968		end_token(&t);
969		t.state = TokenizeStateStart;
	1164	case '0':
	1165	case '1':
	1166	case '2':
	1167	case '3':
	1168	case '4':
	1169	case '5':
	1170	case '6':
	1171	case '7':
	1172	t.state = TokenizeState_int_literal_oct;
970	1173	break;
971	1174	default:
972		// do nothing
	1175	invalid_char_error(&t, c);
973	1176	break;
974	1177	}
975	1178	break;
976		case TokenizeStateContainerDocComment:
	1179	case TokenizeState_int_literal_oct:
977	1180	switch (c) {
978		case '\n':
979		end_token(&t);
980		t.state = TokenizeStateStart;
	1181	case 0:
	1182	goto eof;
	1183	case '_':
	1184	t.state = TokenizeState_int_literal_oct_no_underscore;
981	1185	break;
982		default:
983		// do nothing
	1186	case '0':
	1187	case '1':
	1188	case '2':
	1189	case '3':
	1190	case '4':
	1191	case '5':
	1192	case '6':
	1193	case '7':
984	1194	break;
985		}
986		break;
987		case TokenizeStateSawAtSign:
988		switch (c) {
989		case '"':
990		set_token_id(&t, t.cur_tok, TokenIdSymbol);
991		t.state = TokenizeStateString;
	1195	case ALPHA:
	1196	case '8':
	1197	case '9':
	1198	invalid_char_error(&t, c);
992	1199	break;
993	1200	default:
994		t.pos -= 1;
995		end_token(&t);
996		t.state = TokenizeStateStart;
	1201	t.state = TokenizeState_start;
997	1202	continue;
998	1203	}
999	1204	break;
1000		case TokenizeStateSymbol:
	1205	case TokenizeState_int_literal_dec_no_underscore:
1001	1206	switch (c) {
1002		case SYMBOL_CHAR:
1003		buf_append_char(&t.cur_tok->data.str_lit.str, c);
	1207	case DIGIT:
	1208	t.state = TokenizeState_int_literal_dec;
1004	1209	break;
1005	1210	default:
1006		t.pos -= 1;
1007		end_token(&t);
1008		t.state = TokenizeStateStart;
1009		continue;
	1211	invalid_char_error(&t, c);
	1212	break;
1010	1213	}
1011	1214	break;
1012		case TokenizeStateString:
	1215	case TokenizeState_int_literal_dec:
1013	1216	switch (c) {
1014		case '"':
1015		end_token(&t);
1016		t.state = TokenizeStateStart;
	1217	case 0:
	1218	goto eof;
	1219	case '_':
	1220	t.state = TokenizeState_int_literal_dec_no_underscore;
1017	1221	break;
1018		case '\n':
1019		tokenize_error(&t, "newline not allowed in string literal");
	1222	case '.':
	1223	t.state = TokenizeState_num_dot_dec;
	1224	t.out->ids.last() = TokenIdFloatLiteral;
1020	1225	break;
1021		case '\\':
1022		t.state = TokenizeStateStringEscape;
	1226	case 'e':
	1227	case 'E':
	1228	t.state = TokenizeState_float_exponent_unsigned;
	1229	t.out->ids.last() = TokenIdFloatLiteral;
1023	1230	break;
1024		default:
1025		buf_append_char(&t.cur_tok->data.str_lit.str, c);
	1231	case DIGIT:
	1232	break;
	1233	case ALPHA_EXCEPT_E:
	1234	invalid_char_error(&t, c);
1026	1235	break;
	1236	default:
	1237	t.state = TokenizeState_start;
	1238	continue;
1027	1239	}
1028	1240	break;
1029		case TokenizeStateStringEscape:
	1241	case TokenizeState_int_literal_hex_no_underscore:
1030	1242	switch (c) {
1031		case 'x':
1032		t.state = TokenizeStateCharCode;
1033		t.radix = 16;
1034		t.char_code = 0;
1035		t.char_code_index = 0;
1036		t.unicode = false;
	1243	case HEXDIGIT:
	1244	t.state = TokenizeState_int_literal_hex;
1037	1245	break;
1038		case 'u':
1039		t.state = TokenizeStateStringEscapeUnicodeStart;
1040		break;
1041		case 'n':
1042		handle_string_escape(&t, '\n');
1043		break;
1044		case 'r':
1045		handle_string_escape(&t, '\r');
	1246	default:
	1247	invalid_char_error(&t, c);
	1248	}
	1249	break;
	1250	case TokenizeState_int_literal_hex:
	1251	switch (c) {
	1252	case 0:
	1253	goto eof;
	1254	case '_':
	1255	t.state = TokenizeState_int_literal_hex_no_underscore;
1046	1256	break;
1047		case '\\':
1048		handle_string_escape(&t, '\\');
	1257	case '.':
	1258	t.state = TokenizeState_num_dot_hex;
	1259	t.out->ids.last() = TokenIdFloatLiteral;
1049	1260	break;
1050		case 't':
1051		handle_string_escape(&t, '\t');
	1261	case 'p':
	1262	case 'P':
	1263	t.state = TokenizeState_float_exponent_unsigned;
	1264	t.out->ids.last() = TokenIdFloatLiteral;
1052	1265	break;
1053		case '\'':
1054		handle_string_escape(&t, '\'');
	1266	case HEXDIGIT:
1055	1267	break;
1056		case '"':
1057		handle_string_escape(&t, '\"');
	1268	case ALPHA_EXCEPT_HEX_AND_P:
	1269	invalid_char_error(&t, c);
1058	1270	break;
1059	1271	default:
1060		invalid_char_error(&t, c);
	1272	t.state = TokenizeState_start;
	1273	continue;
1061	1274	}
1062	1275	break;
1063		case TokenizeStateStringEscapeUnicodeStart:
	1276	case TokenizeState_num_dot_dec:
1064	1277	switch (c) {
1065		case '{':
1066		t.state = TokenizeStateCharCode;
1067		t.radix = 16;
1068		t.char_code = 0;
1069		t.char_code_index = 0;
1070		t.unicode = true;
	1278	case 0:
	1279	goto eof;
	1280	case '.':
	1281	t.out->ids.last() = TokenIdIntLiteral;
	1282	t.pos -= 1;
	1283	t.column -= 1;
	1284	t.state = TokenizeState_start;
	1285	continue;
	1286	case 'e':
	1287	case 'E':
	1288	t.state = TokenizeState_float_exponent_unsigned;
1071	1289	break;
1072		default:
	1290	case DIGIT:
	1291	t.state = TokenizeState_float_fraction_dec;
	1292	break;
	1293	case ALPHA_EXCEPT_E:
1073	1294	invalid_char_error(&t, c);
	1295	break;
	1296	default:
	1297	t.state = TokenizeState_start;
	1298	continue;
1074	1299	}
1075	1300	break;
1076		case TokenizeStateCharCode:
1077		{
1078		if (t.unicode && c == '}') {
1079		if (t.char_code_index == 0) {
1080		tokenize_error(&t, "empty unicode escape sequence");
1081		break;
1082		}
1083		if (t.char_code > 0x10ffff) {
1084		tokenize_error(&t, "unicode value out of range: %x", t.char_code);
1085		break;
1086		}
1087		if (t.cur_tok->id == TokenIdCharLiteral) {
1088		t.cur_tok->data.char_lit.c = t.char_code;
1089		t.state = TokenizeStateCharLiteralEnd;
1090		} else if (t.char_code <= 0x7f) {
1091		// 00000000 00000000 00000000 0xxxxxxx
1092		handle_string_escape(&t, (uint8_t)t.char_code);
1093		} else if (t.char_code <= 0x7ff) {
1094		// 00000000 00000000 00000xxx xx000000
1095		handle_string_escape(&t, (uint8_t)(0xc0 \| (t.char_code >> 6)));
1096		// 00000000 00000000 00000000 00xxxxxx
1097		handle_string_escape(&t, (uint8_t)(0x80 \| (t.char_code & 0x3f)));
1098		} else if (t.char_code <= 0xffff) {
1099		// 00000000 00000000 xxxx0000 00000000
1100		handle_string_escape(&t, (uint8_t)(0xe0 \| (t.char_code >> 12)));
1101		// 00000000 00000000 0000xxxx xx000000
1102		handle_string_escape(&t, (uint8_t)(0x80 \| ((t.char_code >> 6) & 0x3f)));
1103		// 00000000 00000000 00000000 00xxxxxx
1104		handle_string_escape(&t, (uint8_t)(0x80 \| (t.char_code & 0x3f)));
1105		} else if (t.char_code <= 0x10ffff) {
1106		// 00000000 000xxx00 00000000 00000000
1107		handle_string_escape(&t, (uint8_t)(0xf0 \| (t.char_code >> 18)));
1108		// 00000000 000000xx xxxx0000 00000000
1109		handle_string_escape(&t, (uint8_t)(0x80 \| ((t.char_code >> 12) & 0x3f)));
1110		// 00000000 00000000 0000xxxx xx000000
1111		handle_string_escape(&t, (uint8_t)(0x80 \| ((t.char_code >> 6) & 0x3f)));
1112		// 00000000 00000000 00000000 00xxxxxx
1113		handle_string_escape(&t, (uint8_t)(0x80 \| (t.char_code & 0x3f)));
1114		} else {
1115		zig_unreachable();
1116		}
	1301	case TokenizeState_num_dot_hex:
	1302	switch (c) {
	1303	case 0:
	1304	goto eof;
	1305	case '.':
	1306	t.out->ids.last() = TokenIdIntLiteral;
	1307	t.pos -= 1;
	1308	t.column -= 1;
	1309	t.state = TokenizeState_start;
	1310	continue;
	1311	case 'p':
	1312	case 'P':
	1313	t.state = TokenizeState_float_exponent_unsigned;
1117	1314	break;
1118		}
1119
1120		uint32_t digit_value = get_digit_value(c);
1121		if (digit_value >= t.radix) {
1122		tokenize_error(&t, "invalid digit: '%c'", c);
	1315	case HEXDIGIT:
	1316	t.out->ids.last() = TokenIdFloatLiteral;
	1317	t.state = TokenizeState_float_fraction_hex;
1123	1318	break;
1124		}
1125		t.char_code *= t.radix;
1126		t.char_code += digit_value;
1127		t.char_code_index += 1;
1128
1129		if (!t.unicode && t.char_code_index >= 2) {
1130		assert(t.char_code <= 255);
1131		handle_string_escape(&t, (uint8_t)t.char_code);
1132		}
1133		}
1134		break;
1135		case TokenizeStateCharLiteral:
1136		if (c == '\'') {
1137		tokenize_error(&t, "expected character");
1138		} else if (c == '\\') {
1139		t.state = TokenizeStateStringEscape;
1140		} else if ((c >= 0x80 && c <= 0xbf) \|\| c >= 0xf8) {
1141		// 10xxxxxx
1142		// 11111xxx
1143		invalid_char_error(&t, c);
1144		} else if (c >= 0xc0 && c <= 0xdf) {
1145		// 110xxxxx
1146		t.cur_tok->data.char_lit.c = c & 0x1f;
1147		t.remaining_code_units = 1;
1148		t.state = TokenizeStateCharLiteralUnicode;
1149		} else if (c >= 0xe0 && c <= 0xef) {
1150		// 1110xxxx
1151		t.cur_tok->data.char_lit.c = c & 0x0f;
1152		t.remaining_code_units = 2;
1153		t.state = TokenizeStateCharLiteralUnicode;
1154		} else if (c >= 0xf0 && c <= 0xf7) {
1155		// 11110xxx
1156		t.cur_tok->data.char_lit.c = c & 0x07;
1157		t.remaining_code_units = 3;
1158		t.state = TokenizeStateCharLiteralUnicode;
1159		} else {
1160		t.cur_tok->data.char_lit.c = c;
1161		t.state = TokenizeStateCharLiteralEnd;
	1319	case ALPHA_EXCEPT_HEX_AND_P:
	1320	invalid_char_error(&t, c);
	1321	break;
	1322	default:
	1323	t.state = TokenizeState_start;
	1324	continue;
1162	1325	}
1163	1326	break;
1164		case TokenizeStateCharLiteralEnd:
	1327	case TokenizeState_float_fraction_dec_no_underscore:
1165	1328	switch (c) {
1166		case '\'':
1167		end_token(&t);
1168		t.state = TokenizeStateStart;
	1329	case DIGIT:
	1330	t.state = TokenizeState_float_fraction_dec;
1169	1331	break;
1170	1332	default:
1171	1333	invalid_char_error(&t, c);
1172	1334	}
1173	1335	break;
1174		case TokenizeStateCharLiteralUnicode:
1175		if (c <= 0x7f \|\| c >= 0xc0) {
1176		invalid_char_error(&t, c);
1177		}
1178		t.cur_tok->data.char_lit.c <<= 6;
1179		t.cur_tok->data.char_lit.c += c & 0x3f;
1180		t.remaining_code_units--;
1181		if (t.remaining_code_units == 0) {
1182		t.state = TokenizeStateCharLiteralEnd;
1183		}
1184		break;
1185		case TokenizeStateZero:
	1336	case TokenizeState_float_fraction_dec:
1186	1337	switch (c) {
1187		case 'b':
1188		t.radix = 2;
1189		t.state = TokenizeStateNumberNoUnderscore;
	1338	case 0:
	1339	goto eof;
	1340	case '_':
	1341	t.state = TokenizeState_float_fraction_dec_no_underscore;
1190	1342	break;
1191		case 'o':
1192		t.radix = 8;
1193		t.state = TokenizeStateNumberNoUnderscore;
	1343	case 'e':
	1344	case 'E':
	1345	t.state = TokenizeState_float_exponent_unsigned;
1194	1346	break;
1195		case 'x':
1196		t.radix = 16;
1197		t.state = TokenizeStateNumberNoUnderscore;
	1347	case DIGIT:
	1348	break;
	1349	case ALPHA_EXCEPT_E:
	1350	invalid_char_error(&t, c);
1198	1351	break;
1199	1352	default:
1200		// reinterpret as normal number
1201		t.pos -= 1;
1202		t.state = TokenizeStateNumber;
	1353	t.state = TokenizeState_start;
1203	1354	continue;
1204	1355	}
1205	1356	break;
1206		case TokenizeStateNumberNoUnderscore:
1207		if (c == '_') {
1208		invalid_char_error(&t, c);
1209		break;
1210		} else if (get_digit_value(c) < t.radix) {
1211		t.is_trailing_underscore = false;
1212		t.state = TokenizeStateNumber;
	1357	case TokenizeState_float_fraction_hex_no_underscore:
	1358	switch (c) {
	1359	case HEXDIGIT:
	1360	t.state = TokenizeState_float_fraction_hex;
	1361	break;
	1362	default:
	1363	invalid_char_error(&t, c);
1213	1364	}
1214		ZIG_FALLTHROUGH;
1215		case TokenizeStateNumber:
1216		{
1217		if (c == '_') {
1218		t.is_trailing_underscore = true;
1219		t.state = TokenizeStateNumberNoUnderscore;
	1365	break;
	1366	case TokenizeState_float_fraction_hex:
	1367	switch (c) {
	1368	case 0:
	1369	goto eof;
	1370	case '_':
	1371	t.state = TokenizeState_float_fraction_hex_no_underscore;
1220	1372	break;
1221		}
1222		if (c == '.') {
1223		if (t.is_trailing_underscore) {
1224		invalid_char_error(&t, c);
1225		break;
1226		}
1227		t.state = TokenizeStateNumberDot;
	1373	case 'p':
	1374	case 'P':
	1375	t.state = TokenizeState_float_exponent_unsigned;
1228	1376	break;
1229		}
1230		if (is_exponent_signifier(c, t.radix)) {
1231		if (t.is_trailing_underscore) {
1232		invalid_char_error(&t, c);
1233		break;
1234		}
1235		if (t.radix != 16 && t.radix != 10) {
1236		invalid_char_error(&t, c);
1237		}
1238		t.state = TokenizeStateFloatExponentUnsigned;
1239		t.radix = 10; // exponent is always base 10
1240		assert(t.cur_tok->id == TokenIdIntLiteral);
1241		set_token_id(&t, t.cur_tok, TokenIdFloatLiteral);
	1377	case HEXDIGIT:
1242	1378	break;
1243		}
1244		uint32_t digit_value = get_digit_value(c);
1245		if (digit_value >= t.radix) {
1246		if (t.is_trailing_underscore) {
1247		invalid_char_error(&t, c);
1248		break;
1249		}
1250
1251		if (is_symbol_char(c)) {
1252		invalid_char_error(&t, c);
1253		}
1254		// not my char
1255		t.pos -= 1;
1256		end_token(&t);
1257		t.state = TokenizeStateStart;
1258		continue;
1259		}
1260		BigInt digit_value_bi;
1261		bigint_init_unsigned(&digit_value_bi, digit_value);
1262
1263		BigInt radix_bi;
1264		bigint_init_unsigned(&radix_bi, t.radix);
1265
1266		BigInt multiplied;
1267		bigint_mul(&multiplied, &t.cur_tok->data.int_lit.bigint, &radix_bi);
1268
1269		bigint_add(&t.cur_tok->data.int_lit.bigint, &multiplied, &digit_value_bi);
1270		break;
1271		}
1272		case TokenizeStateNumberDot:
1273		{
1274		if (c == '.') {
1275		t.pos -= 2;
1276		end_token(&t);
1277		t.state = TokenizeStateStart;
1278		continue;
1279		}
1280		if (t.radix != 16 && t.radix != 10) {
	1379	case ALPHA_EXCEPT_HEX_AND_P:
1281	1380	invalid_char_error(&t, c);
1282		}
1283		t.pos -= 1;
1284		t.state = TokenizeStateFloatFractionNoUnderscore;
1285		assert(t.cur_tok->id == TokenIdIntLiteral);
1286		set_token_id(&t, t.cur_tok, TokenIdFloatLiteral);
1287		continue;
1288		}
1289		case TokenizeStateFloatFractionNoUnderscore:
1290		if (c == '_') {
1291		invalid_char_error(&t, c);
1292		} else if (get_digit_value(c) < t.radix) {
1293		t.is_trailing_underscore = false;
1294		t.state = TokenizeStateFloatFraction;
1295		}
1296		ZIG_FALLTHROUGH;
1297		case TokenizeStateFloatFraction:
1298		{
1299		if (c == '_') {
1300		t.is_trailing_underscore = true;
1301		t.state = TokenizeStateFloatFractionNoUnderscore;
1302		break;
1303		}
1304		if (is_exponent_signifier(c, t.radix)) {
1305		if (t.is_trailing_underscore) {
1306		invalid_char_error(&t, c);
1307		break;
1308		}
1309		t.state = TokenizeStateFloatExponentUnsigned;
1310		t.radix = 10; // exponent is always base 10
1311	1381	break;
1312		}
1313		uint32_t digit_value = get_digit_value(c);
1314		if (digit_value >= t.radix) {
1315		if (t.is_trailing_underscore) {
1316		invalid_char_error(&t, c);
1317		break;
1318		}
1319		if (is_symbol_char(c)) {
1320		invalid_char_error(&t, c);
1321		}
1322		// not my char
1323		t.pos -= 1;
1324		end_token(&t);
1325		t.state = TokenizeStateStart;
	1382	default:
	1383	t.state = TokenizeState_start;
1326	1384	continue;
1327		}
1328
1329		// we use parse_f128 to generate the float literal, so just
1330		// need to get to the end of the token
1331	1385	}
1332	1386	break;
1333		case TokenizeStateFloatExponentUnsigned:
	1387	case TokenizeState_float_exponent_unsigned:
1334	1388	switch (c) {
1335	1389	case '+':
1336		t.state = TokenizeStateFloatExponentNumberNoUnderscore;
1337		break;
1338	1390	case '-':
1339		t.state = TokenizeStateFloatExponentNumberNoUnderscore;
	1391	t.state = TokenizeState_float_exponent_num_no_underscore;
1340	1392	break;
1341	1393	default:
1342		// reinterpret as normal exponent number
1343		t.pos -= 1;
1344		t.state = TokenizeStateFloatExponentNumberNoUnderscore;
1345		continue;
1346		}
1347		break;
1348		case TokenizeStateFloatExponentNumberNoUnderscore:
1349		if (c == '_') {
1350		invalid_char_error(&t, c);
1351		} else if (get_digit_value(c) < t.radix) {
1352		t.is_trailing_underscore = false;
1353		t.state = TokenizeStateFloatExponentNumber;
1354		}
1355		ZIG_FALLTHROUGH;
1356		case TokenizeStateFloatExponentNumber:
1357		{
1358		if (c == '_') {
1359		t.is_trailing_underscore = true;
1360		t.state = TokenizeStateFloatExponentNumberNoUnderscore;
1361		break;
1362		}
1363		uint32_t digit_value = get_digit_value(c);
1364		if (digit_value >= t.radix) {
1365		if (t.is_trailing_underscore) {
1366		invalid_char_error(&t, c);
1367		break;
1368		}
1369		if (is_symbol_char(c)) {
1370		invalid_char_error(&t, c);
1371		}
1372		// not my char
1373		t.pos -= 1;
1374		end_token(&t);
1375		t.state = TokenizeStateStart;
	1394	// Reinterpret as a normal exponent number.
	1395	t.state = TokenizeState_float_exponent_num_no_underscore;
1376	1396	continue;
1377		}
1378
1379		// we use parse_f128 to generate the float literal, so just
1380		// need to get to the end of the token
1381	1397	}
1382	1398	break;
1383		case TokenizeStateSawDash:
	1399	case TokenizeState_float_exponent_num_no_underscore:
1384	1400	switch (c) {
1385		case '>':
1386		set_token_id(&t, t.cur_tok, TokenIdArrow);
1387		end_token(&t);
1388		t.state = TokenizeStateStart;
1389		break;
1390		case '=':
1391		set_token_id(&t, t.cur_tok, TokenIdMinusEq);
1392		end_token(&t);
1393		t.state = TokenizeStateStart;
1394		break;
1395		case '%':
1396		set_token_id(&t, t.cur_tok, TokenIdMinusPercent);
1397		t.state = TokenizeStateSawMinusPercent;
	1401	case DIGIT:
	1402	t.state = TokenizeState_float_exponent_num;
1398	1403	break;
1399	1404	default:
1400		t.pos -= 1;
1401		end_token(&t);
1402		t.state = TokenizeStateStart;
1403		continue;
	1405	invalid_char_error(&t, c);
1404	1406	}
1405	1407	break;
1406		case TokenizeStateSawMinusPercent:
	1408	case TokenizeState_float_exponent_num:
1407	1409	switch (c) {
1408		case '=':
1409		set_token_id(&t, t.cur_tok, TokenIdMinusPercentEq);
1410		end_token(&t);
1411		t.state = TokenizeStateStart;
	1410	case 0:
	1411	goto eof;
	1412	case '_':
	1413	t.state = TokenizeState_float_exponent_num_no_underscore;
	1414	break;
	1415	case DIGIT:
	1416	break;
	1417	case ALPHA:
	1418	invalid_char_error(&t, c);
1412	1419	break;
1413	1420	default:
1414		t.pos -= 1;
1415		end_token(&t);
1416		t.state = TokenizeStateStart;
	1421	t.state = TokenizeState_start;
1417	1422	continue;
1418	1423	}
1419	1424	break;
1420	1425	}
	1426	t.pos += 1;
1421	1427	if (c == '\n') {
1422		out->line_offsets->append(t.pos + 1);
1423	1428	t.line += 1;
1424	1429	t.column = 0;
1425	1430	} else {
1426	1431	t.column += 1;
1427	1432	}
1428	1433	}
1429		// EOF
1430		switch (t.state) {
1431		case TokenizeStateStart:
1432		case TokenizeStateError:
1433		break;
1434		case TokenizeStateNumberNoUnderscore:
1435		case TokenizeStateFloatFractionNoUnderscore:
1436		case TokenizeStateFloatExponentNumberNoUnderscore:
1437		case TokenizeStateNumberDot:
1438		tokenize_error(&t, "unterminated number literal");
1439		break;
1440		case TokenizeStateString:
1441		tokenize_error(&t, "unterminated string");
1442		break;
1443		case TokenizeStateStringEscape:
1444		case TokenizeStateStringEscapeUnicodeStart:
1445		case TokenizeStateCharCode:
1446		if (t.cur_tok->id == TokenIdStringLiteral) {
1447		tokenize_error(&t, "unterminated string");
1448		break;
1449		} else if (t.cur_tok->id == TokenIdCharLiteral) {
1450		tokenize_error(&t, "unterminated Unicode code point literal");
1451		break;
1452		} else {
1453		zig_unreachable();
1454		}
1455		break;
1456		case TokenizeStateCharLiteral:
1457		case TokenizeStateCharLiteralEnd:
1458		case TokenizeStateCharLiteralUnicode:
1459		tokenize_error(&t, "unterminated Unicode code point literal");
1460		break;
1461		case TokenizeStateSymbol:
1462		case TokenizeStateZero:
1463		case TokenizeStateNumber:
1464		case TokenizeStateFloatFraction:
1465		case TokenizeStateFloatExponentUnsigned:
1466		case TokenizeStateFloatExponentNumber:
1467		case TokenizeStateSawStar:
1468		case TokenizeStateSawSlash:
1469		case TokenizeStateSawPercent:
1470		case TokenizeStateSawPlus:
1471		case TokenizeStateSawDash:
1472		case TokenizeStateSawAmpersand:
1473		case TokenizeStateSawCaret:
1474		case TokenizeStateSawBar:
1475		case TokenizeStateSawEq:
1476		case TokenizeStateSawBang:
1477		case TokenizeStateSawLessThan:
1478		case TokenizeStateSawLessThanLessThan:
1479		case TokenizeStateSawGreaterThan:
1480		case TokenizeStateSawGreaterThanGreaterThan:
1481		case TokenizeStateSawDot:
1482		case TokenizeStateSawDotStar:
1483		case TokenizeStateSawAtSign:
1484		case TokenizeStateSawStarPercent:
1485		case TokenizeStateSawPlusPercent:
1486		case TokenizeStateSawMinusPercent:
1487		case TokenizeStateLineString:
1488		case TokenizeStateLineStringEnd:
1489		case TokenizeStateDocComment:
1490		case TokenizeStateContainerDocComment:
1491		end_token(&t);
1492		break;
1493		case TokenizeStateSawDotDot:
1494		case TokenizeStateSawBackslash:
1495		case TokenizeStateLineStringContinue:
1496		tokenize_error(&t, "unexpected EOF");
1497		break;
1498		case TokenizeStateLineComment:
1499		break;
1500		case TokenizeStateSawSlash2:
1501		cancel_token(&t);
1502		break;
1503		case TokenizeStateSawSlash3:
1504		set_token_id(&t, t.cur_tok, TokenIdDocComment);
1505		end_token(&t);
1506		break;
1507		case TokenizeStateSawSlashBang:
1508		set_token_id(&t, t.cur_tok, TokenIdContainerDocComment);
1509		end_token(&t);
1510		break;
1511		}
1512		if (t.state != TokenizeStateError) {
1513		if (t.tokens->length > 0) {
1514		Token *last_token = &t.tokens->last();
1515		t.line = (int)last_token->start_line;
1516		t.column = (int)last_token->start_column;
1517		t.pos = last_token->start_pos;
1518		} else {
1519		t.pos = 0;
1520		}
1521		begin_token(&t, TokenIdEof);
1522		end_token(&t);
1523		assert(!t.cur_tok);
1524		}
	1434	eof:;
	1435
	1436	begin_token(&t, TokenIdEof);
1525	1437	}
1526	1438
1527	1439	const char * token_name(TokenId id) {
1528	1440	switch (id) {
1529	1441	case TokenIdAmpersand: return "&";
1530	1442	case TokenIdArrow: return "->";
1531		case TokenIdAtSign: return "@";
1532	1443	case TokenIdBang: return "!";
1533	1444	case TokenIdBarBar: return "\|\|";
1534	1445	case TokenIdBinOr: return "\|";
...	...	@@ -1622,9 +1533,7 @@ const char * token_name(TokenId id) {
1622	1533	case TokenIdMinusPercent: return "-%";
1623	1534	case TokenIdMinusPercentEq: return "-%=";
1624	1535	case TokenIdModEq: return "%=";
1625		case TokenIdNumberSign: return "#";
1626	1536	case TokenIdPercent: return "%";
1627		case TokenIdPercentDot: return "%.";
1628	1537	case TokenIdPlus: return "+";
1629	1538	case TokenIdPlusEq: return "+=";
1630	1539	case TokenIdPlusPercent: return "+%";
...	...	@@ -1638,33 +1547,15 @@ const char * token_name(TokenId id) {
1638	1547	case TokenIdStar: return "*";
1639	1548	case TokenIdStarStar: return "**";
1640	1549	case TokenIdStringLiteral: return "StringLiteral";
1641		case TokenIdMultilineStringLiteral: return "MultilineStringLiteral";
1642		case TokenIdSymbol: return "Symbol";
	1550	case TokenIdMultilineStringLiteralLine: return "MultilineStringLiteralLine";
	1551	case TokenIdIdentifier: return "Identifier";
1643	1552	case TokenIdTilde: return "~";
1644	1553	case TokenIdTimesEq: return "*=";
1645	1554	case TokenIdTimesPercent: return "*%";
1646	1555	case TokenIdTimesPercentEq: return "*%=";
	1556	case TokenIdBuiltin: return "Builtin";
1647	1557	case TokenIdCount:
1648	1558	zig_unreachable();
1649	1559	}
1650	1560	return "(invalid token)";
1651	1561	}
1652
1653		void print_tokens(Buf buf, ZigList<Token> tokens) {
1654		for (size_t i = 0; i < tokens->length; i += 1) {
1655		Token *token = &tokens->at(i);
1656		fprintf(stderr, "%s ", token_name(token->id));
1657		if (token->start_pos != SIZE_MAX) {
1658		fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stderr);
1659		}
1660		fprintf(stderr, "\n");
1661		}
1662		}
1663
1664		bool valid_symbol_starter(uint8_t c) {
1665		switch (c) {
1666		case SYMBOL_START:
1667		return true;
1668		}
1669		return false;
1670		}

src/stage1/tokenizer.hpp+14-56

...	...	@@ -12,10 +12,9 @@
12	12	#include "bigint.hpp"
13	13	#include "bigfloat.hpp"
14	14
15		enum TokenId {
	15	enum TokenId : uint8_t {
16	16	TokenIdAmpersand,
17	17	TokenIdArrow,
18		TokenIdAtSign,
19	18	TokenIdBang,
20	19	TokenIdBarBar,
21	20	TokenIdBinOr,
...	...	@@ -27,6 +26,7 @@ enum TokenId {
27	26	TokenIdBitShiftRight,
28	27	TokenIdBitShiftRightEq,
29	28	TokenIdBitXorEq,
	29	TokenIdBuiltin,
30	30	TokenIdCharLiteral,
31	31	TokenIdCmpEq,
32	32	TokenIdCmpGreaterOrEq,
...	...	@@ -109,9 +109,7 @@ enum TokenId {
109	109	TokenIdMinusPercent,
110	110	TokenIdMinusPercentEq,
111	111	TokenIdModEq,
112		TokenIdNumberSign,
113	112	TokenIdPercent,
114		TokenIdPercentDot,
115	113	TokenIdPlus,
116	114	TokenIdPlusEq,
117	115	TokenIdPlusPercent,
...	...	@@ -125,75 +123,35 @@ enum TokenId {
125	123	TokenIdStar,
126	124	TokenIdStarStar,
127	125	TokenIdStringLiteral,
128		TokenIdMultilineStringLiteral,
129		TokenIdSymbol,
	126	TokenIdMultilineStringLiteralLine,
	127	TokenIdIdentifier,
130	128	TokenIdTilde,
131	129	TokenIdTimesEq,
132	130	TokenIdTimesPercent,
133	131	TokenIdTimesPercentEq,
134		TokenIdCount,
135		};
136
137		struct TokenFloatLit {
138		BigFloat bigfloat;
139		// overflow is true if when parsing the number, we discovered it would not fit
140		// without losing data
141		bool overflow;
142		};
143
144		struct TokenIntLit {
145		BigInt bigint;
146		};
147
148		struct TokenStrLit {
149		Buf str;
150		};
151	132
152		struct TokenCharLit {
153		uint32_t c;
	133	TokenIdCount,
154	134	};
155	135
156		struct Token {
157		TokenId id;
158		size_t start_pos;
159		size_t end_pos;
160		size_t start_line;
161		size_t start_column;
162
163		union {
164		// TokenIdIntLiteral
165		TokenIntLit int_lit;
166
167		// TokenIdFloatLiteral
168		TokenFloatLit float_lit;
	136	typedef uint32_t TokenIndex;
169	137
170		// TokenIdStringLiteral, TokenIdMultilineStringLiteral or TokenIdSymbol
171		TokenStrLit str_lit;
172
173		// TokenIdCharLiteral
174		TokenCharLit char_lit;
175		} data;
	138	struct TokenLoc {
	139	uint32_t offset;
	140	uint32_t line;
	141	uint32_t column;
176	142	};
177		// work around conflicting name Token which is also found in libclang
178		typedef Token ZigToken;
179	143
180	144	struct Tokenization {
181		ZigList<Token> *tokens;
182		ZigList<size_t> *line_offsets;
	145	ZigList<TokenId> ids;
	146	ZigList<TokenLoc> locs;
183	147
184	148	// if an error occurred
185	149	Buf *err;
186		size_t err_line;
187		size_t err_column;
	150	uint32_t err_byte_offset;
188	151	};
189	152
190		void tokenize(Buf buf, Tokenization out_tokenization);
191
192		void print_tokens(Buf buf, ZigList<Token> tokens);
	153	void tokenize(const char source, Tokenization out_tokenization);
193	154
194	155	const char * token_name(TokenId id);
195	156
196		bool valid_symbol_starter(uint8_t c);
197		bool is_zig_keyword(Buf *buf);
198
199	157	#endif

test/behavior/misc.zig+8-7

...	...	@@ -1,6 +1,7 @@
1	1	const std = @import("std");
2	2	const expect = std.testing.expect;
3	3	const expectEqualSlices = std.testing.expectEqualSlices;
	4	const expectEqualStrings = std.testing.expectEqualStrings;
4	5	const mem = std.mem;
5	6	const builtin = @import("builtin");
6	7
...	...	@@ -147,13 +148,13 @@ test "array mult operator" {
147	148	}
148	149
149	150	test "string escapes" {
150		try expect(mem.eql(u8, "\"", "\x22"));
151		try expect(mem.eql(u8, "\'", "\x27"));
152		try expect(mem.eql(u8, "\n", "\x0a"));
153		try expect(mem.eql(u8, "\r", "\x0d"));
154		try expect(mem.eql(u8, "\t", "\x09"));
155		try expect(mem.eql(u8, "\\", "\x5c"));
156		try expect(mem.eql(u8, "\u{1234}\u{069}\u{1}", "\xe1\x88\xb4\x69\x01"));
	151	try expectEqualStrings("\"", "\x22");
	152	try expectEqualStrings("\'", "\x27");
	153	try expectEqualStrings("\n", "\x0a");
	154	try expectEqualStrings("\r", "\x0d");
	155	try expectEqualStrings("\t", "\x09");
	156	try expectEqualStrings("\\", "\x5c");
	157	try expectEqualStrings("\u{1234}\u{069}\u{1}", "\xe1\x88\xb4\x69\x01");
157	158	}
158	159
159	160	test "multiline string" {