Commit 457c94d353b77b08786aa8794e1afc6a62d5c34a | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2024-08-22 22:27:46+01:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2024-08-27 00:41:49+01:00
log	457c94d353b77b08786aa8794e1afc6a62d5c34a
tree	ef1463f066a1ed7526aadcb1b6b4e488845b34f1
parent	72e00805a61719174b1668d5ce3cbef8338e4690
signature	Commit is signed but in an unrecognized format.

compiler: implement `@branchHint`, replacing `@setCold`

Implements the accepted proposal to introduce `@branchHint`. This
builtin is permitted as the first statement of a block if that block is
the direct body of any of the following:

* a function (*not* a `test`)
* either branch of an `if`
* the RHS of a `catch` or `orelse`
* a `switch` prong
* an `or` or `and` expression

It lowers to the ZIR instruction `extended(branch_hint(...))`. When Sema
encounters this instruction, it sets `sema.branch_hint` appropriately,
and `zirCondBr` etc are expected to reset this value as necessary. The
state is on `Sema` rather than `Block` to make it automatically
propagate up non-conditional blocks without special handling. If
`@panic` is reached, the branch hint is set to `.cold` if none was
already set; similarly, error branches get a hint of `.unlikely` if no
hint is explicitly provided. If a condition is comptime-known, `cold`
hints from the taken branch are allowed to propagate up, but other hints
are discarded. This is because a `likely`/`unlikely` hint just indicates
the direction this branch is likely to go, which is redundant
information when the branch is known at comptime; but `cold` hints
indicate that control flow is unlikely to ever reach this branch,
meaning if the branch is always taken from its parent, then the parent
is also unlikely to ever be reached.

This branch information is stored in AIR `cond_br` and `switch_br`. In
addition, `try` and `try_ptr` instructions have variants `try_cold` and
`try_ptr_cold` which indicate that the error case is cold (rather than
just unlikely); this is reachable through e.g. `errdefer unreachable` or
`errdefer @panic("")`.

A new API `unwrapSwitch` is introduced to `Air` to make it more
convenient to access `switch_br` instructions. In time, I plan to update
all AIR instructions to be accessed via an `unwrap` method which returns
a convenient tagged union a la `InternPool.indexToKey`.

The LLVM backend lowers branch hints for conditional branches and
switches as follows:

* If any branch is marked `unpredictable`, the instruction is marked
  `!unpredictable`.
* Any branch which is marked as `cold` gets a
  `llvm.assume(i1 true) [ "cold"() ]` call to mark the code path cold.
* If any branch is marked `likely` or `unlikely`, branch weight metadata
  is attached with `!prof`. Likely branches get a weight of 2000, and
  unlikely branches a weight of 1. In `switch` statements, un-annotated
  branches get a weight of 1000 as a "middle ground" hint, since there
  could be likely *and* unlikely *and* un-annotated branches.

For functions, a `cold` hint corresponds to the `cold` function
attribute, and other hints are currently ignored -- as far as I can tell
LLVM doesn't really have a way to lower them. (Ideally, we would want
the branch hint given in the function to propagate to call sites.)

The compiler and standard library do not yet use this new builtin.

Resolves: #21148

25 files changed, 1127 insertions(+), 563 deletions(-)

lib
- std
  - builtin.zig
  - zig
    AstGen.zig
    AstRlAnnotate.zig
    BuiltinFn.zig
    Zir.zig
src
- Air.zig
- Air
  - types_resolved.zig
- InternPool.zig
- Liveness.zig
- Liveness
  - Verify.zig
- Sema.zig
- Zcu
  - PerThread.zig
- arch
  - aarch64
    CodeGen.zig
  - arm
    CodeGen.zig
  - riscv64
    CodeGen.zig
  - sparc64
    CodeGen.zig
  - wasm
    CodeGen.zig
  - x86_64
    CodeGen.zig
- codegen
  - c.zig
  - llvm.zig
  - llvm
    Builder.zig
    ir.zig
  - spirv.zig
- print_air.zig
- print_zir.zig

lib/std/builtin.zig+19

...	@@ -675,6 +675,25 @@ pub const ExternOptions = struct {	...	@@ -675,6 +675,25 @@ pub const ExternOptions = struct {
675	is_thread_local: bool = false,	675	is_thread_local: bool = false,
676	};	676	};
677		677
		678	/// This data structure is used by the Zig language code generation and
		679	/// therefore must be kept in sync with the compiler implementation.
		680	pub const BranchHint = enum(u3) {
		681	/// Equivalent to no hint given.
		682	none,
		683	/// This branch of control flow is more likely to be reached than its peers.
		684	/// The optimizer should optimize for reaching it.
		685	likely,
		686	/// This branch of control flow is less likely to be reached than its peers.
		687	/// The optimizer should optimize for not reaching it.
		688	unlikely,
		689	/// This branch of control flow is unlikely to ever be reached.
		690	/// The optimizer may place it in a different page of memory to optimize other branches.
		691	cold,
		692	/// It is difficult to predict whether this branch of control flow will be reached.
		693	/// The optimizer should avoid branching behavior with expensive mispredictions.
		694	unpredictable,
		695	};
		696
678	/// This enum is set by the compiler and communicates which compiler backend is	697	/// This enum is set by the compiler and communicates which compiler backend is
679	/// used to produce machine code.	698	/// used to produce machine code.
680	/// Think carefully before deciding to observe this value. Nearly all code should	699	/// Think carefully before deciding to observe this value. Nearly all code should

lib/std/zig/AstGen.zig+91-53

 @@ -811,18 +811,18 @@ fn expr(gz: *GenZir, scope: *Scope, ri: ResultInfo, node: Ast.Node.Index) InnerE
         .builtin_call_two, .builtin_call_two_comma => {
             if (node_datas[node].lhs == 0) {
                 const params = [_]Ast.Node.Index{};
-                return builtinCall(gz, scope, ri, node, &params);
+                return builtinCall(gz, scope, ri, node, &params, false);
             } else if (node_datas[node].rhs == 0) {
                 const params = [_]Ast.Node.Index{node_datas[node].lhs};
-                return builtinCall(gz, scope, ri, node, &params);
+                return builtinCall(gz, scope, ri, node, &params, false);
             } else {
                 const params = [_]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs };
-                return builtinCall(gz, scope, ri, node, &params);
+                return builtinCall(gz, scope, ri, node, &params, false);
             }
         },
         .builtin_call, .builtin_call_comma => {
             const params = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
-            return builtinCall(gz, scope, ri, node, params);
+            return builtinCall(gz, scope, ri, node, params, false);
         },
         .call_one,
 @@ -1017,16 +1017,16 @@ fn expr(gz: *GenZir, scope: *Scope, ri: ResultInfo, node: Ast.Node.Index) InnerE
         .block_two, .block_two_semicolon => {
             const statements = [2]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs };
             if (node_datas[node].lhs == 0) {
-                return blockExpr(gz, scope, ri, node, statements[0..0]);
+                return blockExpr(gz, scope, ri, node, statements[0..0], .normal);
             } else if (node_datas[node].rhs == 0) {
-                return blockExpr(gz, scope, ri, node, statements[0..1]);
+                return blockExpr(gz, scope, ri, node, statements[0..1], .normal);
             } else {
-                return blockExpr(gz, scope, ri, node, statements[0..2]);
+                return blockExpr(gz, scope, ri, node, statements[0..2], .normal);
             }
         },
         .block, .block_semicolon => {
             const statements = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
-            return blockExpr(gz, scope, ri, node, statements);
+            return blockExpr(gz, scope, ri, node, statements, .normal);
         },
         .enum_literal => return simpleStrTok(gz, ri, main_tokens[node], node, .enum_literal),
         .error_value => return simpleStrTok(gz, ri, node_datas[node].rhs, node, .error_value),
 @@ -1241,7 +1241,7 @@ fn suspendExpr(
     suspend_scope.suspend_node = node;
     defer suspend_scope.unstack();
-    const body_result = try fullBodyExpr(&suspend_scope, &suspend_scope.base, .{ .rl = .none }, body_node);
+    const body_result = try fullBodyExpr(&suspend_scope, &suspend_scope.base, .{ .rl = .none }, body_node, .normal);
     if (!gz.refIsNoReturn(body_result)) {
         _ = try suspend_scope.addBreak(.break_inline, suspend_inst, .void_value);
     }
 @@ -1362,7 +1362,7 @@ fn fnProtoExpr(
                 assert(param_type_node != 0);
                 var param_gz = block_scope.makeSubBlock(scope);
                 defer param_gz.unstack();
-                const param_type = try fullBodyExpr(&param_gz, scope, coerced_type_ri, param_type_node);
+                const param_type = try fullBodyExpr(&param_gz, scope, coerced_type_ri, param_type_node, .normal);
                 const param_inst_expected: Zir.Inst.Index = @enumFromInt(astgen.instructions.len + 1);
                 _ = try param_gz.addBreakWithSrcNode(.break_inline, param_inst_expected, param_type, param_type_node);
                 const main_tokens = tree.nodes.items(.main_token);
 @@ -2040,13 +2040,13 @@ fn comptimeExpr(
                         else
                             stmts[0..2];
-                        const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmt_slice, true);
+                        const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmt_slice, true, .normal);
                         return rvalue(gz, ri, block_ref, node);
                     },
                     .block, .block_semicolon => {
                         const stmts = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
                         // Replace result location and copy back later - see above.
-                        const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmts, true);
+                        const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmts, true, .normal);
                         return rvalue(gz, ri, block_ref, node);
                     },
                     else => unreachable,
 @@ -2071,7 +2071,7 @@ fn comptimeExpr(
         else
             .none,
     };
-    const block_result = try fullBodyExpr(&block_scope, scope, ty_only_ri, node);
+    const block_result = try fullBodyExpr(&block_scope, scope, ty_only_ri, node, .normal);
     if (!gz.refIsNoReturn(block_result)) {
         _ = try block_scope.addBreak(.@"break", block_inst, block_result);
     }
 @@ -2311,6 +2311,7 @@ fn fullBodyExpr(
     scope: *Scope,
     ri: ResultInfo,
     node: Ast.Node.Index,
+    block_kind: BlockKind,
 ) InnerError!Zir.Inst.Ref {
     const tree = gz.astgen.tree;
     const node_tags = tree.nodes.items(.tag);
 @@ -2340,21 +2341,24 @@ fn fullBodyExpr(
         // Labeled blocks are tricky - forwarding result location information properly is non-trivial,
         // plus if this block is exited with a `break_inline` we aren't allowed multiple breaks. This
         // case is rare, so just treat it as a normal expression and create a nested block.
-        return expr(gz, scope, ri, node);
+        return blockExpr(gz, scope, ri, node, statements, block_kind);
     }
     var sub_gz = gz.makeSubBlock(scope);
-    try blockExprStmts(&sub_gz, &sub_gz.base, statements);
+    try blockExprStmts(&sub_gz, &sub_gz.base, statements, block_kind);
     return rvalue(gz, ri, .void_value, node);
 }
+const BlockKind = enum { normal, allow_branch_hint };
 fn blockExpr(
     gz: *GenZir,
     scope: *Scope,
     ri: ResultInfo,
     block_node: Ast.Node.Index,
     statements: []const Ast.Node.Index,
+    kind: BlockKind,
 ) InnerError!Zir.Inst.Ref {
     const astgen = gz.astgen;
     const tree = astgen.tree;
 @@ -2365,7 +2369,7 @@ fn blockExpr(
     if (token_tags[lbrace - 1] == .colon and
         token_tags[lbrace - 2] == .identifier)
     {
-        return labeledBlockExpr(gz, scope, ri, block_node, statements, false);
+        return labeledBlockExpr(gz, scope, ri, block_node, statements, false, kind);
     }
     if (!gz.is_comptime) {
 @@ -2380,7 +2384,7 @@ fn blockExpr(
         var block_scope = gz.makeSubBlock(scope);
         defer block_scope.unstack();
-        try blockExprStmts(&block_scope, &block_scope.base, statements);
+        try blockExprStmts(&block_scope, &block_scope.base, statements, kind);
         if (!block_scope.endsWithNoReturn()) {
             // As our last action before the break, "pop" the error trace if needed
 @@ -2391,7 +2395,7 @@ fn blockExpr(
         try block_scope.setBlockBody(block_inst);
     } else {
         var sub_gz = gz.makeSubBlock(scope);
-        try blockExprStmts(&sub_gz, &sub_gz.base, statements);
+        try blockExprStmts(&sub_gz, &sub_gz.base, statements, kind);
     }
     return rvalue(gz, ri, .void_value, block_node);
 @@ -2436,6 +2440,7 @@ fn labeledBlockExpr(
     block_node: Ast.Node.Index,
     statements: []const Ast.Node.Index,
     force_comptime: bool,
+    block_kind: BlockKind,
 ) InnerError!Zir.Inst.Ref {
     const astgen = gz.astgen;
     const tree = astgen.tree;
 @@ -2476,7 +2481,7 @@ fn labeledBlockExpr(
     if (force_comptime) block_scope.is_comptime = true;
     defer block_scope.unstack();
-    try blockExprStmts(&block_scope, &block_scope.base, statements);
+    try blockExprStmts(&block_scope, &block_scope.base, statements, block_kind);
     if (!block_scope.endsWithNoReturn()) {
         // As our last action before the return, "pop" the error trace if needed
         _ = try gz.addRestoreErrRetIndex(.{ .block = block_inst }, .always, block_node);
 @@ -2495,7 +2500,7 @@ fn labeledBlockExpr(
     }
 }
-fn blockExprStmts(gz: *GenZir, parent_scope: *Scope, statements: []const Ast.Node.Index) !void {
+fn blockExprStmts(gz: *GenZir, parent_scope: *Scope, statements: []const Ast.Node.Index, block_kind: BlockKind) !void {
     const astgen = gz.astgen;
     const tree = astgen.tree;
     const node_tags = tree.nodes.items(.tag);
 @@ -2509,7 +2514,7 @@ fn blockExprStmts(gz: *GenZir, parent_scope: *Scope, statements: []const Ast.Nod
     var noreturn_src_node: Ast.Node.Index = 0;
     var scope = parent_scope;
-    for (statements) |statement| {
+    for (statements, 0..) |statement, stmt_idx| {
         if (noreturn_src_node != 0) {
             try astgen.appendErrorNodeNotes(
                 statement,
 @@ -2524,6 +2529,10 @@ fn blockExprStmts(gz: *GenZir, parent_scope: *Scope, statements: []const Ast.Nod
                 },
             );
         }
+        const allow_branch_hint = switch (block_kind) {
+            .normal => false,
+            .allow_branch_hint => stmt_idx == 0,
+        };
         var inner_node = statement;
         while (true) {
             switch (node_tags[inner_node]) {
 @@ -2567,6 +2576,30 @@ fn blockExprStmts(gz: *GenZir, parent_scope: *Scope, statements: []const Ast.Nod
                 .for_simple,
                 .@"for", => _ = try forExpr(gz, scope, .{ .rl = .none }, inner_node, tree.fullFor(inner_node).?, true),
+                // These cases are here to allow branch hints.
+                .builtin_call_two, .builtin_call_two_comma => {
+                    try emitDbgNode(gz, inner_node);
+                    const ri: ResultInfo = .{ .rl = .none };
+                    const result = if (node_data[inner_node].lhs == 0) r: {
+                        break :r try builtinCall(gz, scope, ri, inner_node, &.{}, allow_branch_hint);
+                    } else if (node_data[inner_node].rhs == 0) r: {
+                        break :r try builtinCall(gz, scope, ri, inner_node, &.{node_data[inner_node].lhs}, allow_branch_hint);
+                    } else r: {
+                        break :r try builtinCall(gz, scope, ri, inner_node, &.{
+                            node_data[inner_node].lhs,
+                            node_data[inner_node].rhs,
+                        }, allow_branch_hint);
+                    };
+                    noreturn_src_node = try addEnsureResult(gz, result, inner_node);
+                },
+                .builtin_call, .builtin_call_comma => {
+                    try emitDbgNode(gz, inner_node);
+                    const ri: ResultInfo = .{ .rl = .none };
+                    const params = tree.extra_data[node_data[inner_node].lhs..node_data[inner_node].rhs];
+                    const result = try builtinCall(gz, scope, ri, inner_node, params, allow_branch_hint);
+                    noreturn_src_node = try addEnsureResult(gz, result, inner_node);
+                },
                 else => noreturn_src_node = try unusedResultExpr(gz, scope, inner_node),
                 // zig fmt: on
             }
 @@ -2827,7 +2860,7 @@ fn addEnsureResult(gz: *GenZir, maybe_unused_result: Zir.Inst.Ref, statement: As
                 .fence,
                 .set_float_mode,
                 .set_align_stack,
-                .set_cold,
+                .branch_hint,
                 => break :b true,
                 else => break :b false,
             },
 @@ -4154,7 +4187,7 @@ fn fnDecl(
                 assert(param_type_node != 0);
                 var param_gz = decl_gz.makeSubBlock(scope);
                 defer param_gz.unstack();
-                const param_type = try fullBodyExpr(&param_gz, params_scope, coerced_type_ri, param_type_node);
+                const param_type = try fullBodyExpr(&param_gz, params_scope, coerced_type_ri, param_type_node, .normal);
                 const param_inst_expected: Zir.Inst.Index = @enumFromInt(astgen.instructions.len + 1);
                 _ = try param_gz.addBreakWithSrcNode(.break_inline, param_inst_expected, param_type, param_type_node);
 @@ -4276,7 +4309,7 @@ fn fnDecl(
     var ret_gz = decl_gz.makeSubBlock(params_scope);
     defer ret_gz.unstack();
     const ret_ref: Zir.Inst.Ref = inst: {
-        const inst = try fullBodyExpr(&ret_gz, params_scope, coerced_type_ri, fn_proto.ast.return_type);
+        const inst = try fullBodyExpr(&ret_gz, params_scope, coerced_type_ri, fn_proto.ast.return_type, .normal);
         if (ret_gz.instructionsSlice().len == 0) {
             // In this case we will send a len=0 body which can be encoded more efficiently.
             break :inst inst;
 @@ -4351,7 +4384,7 @@ fn fnDecl(
         const lbrace_line = astgen.source_line - decl_gz.decl_line;
         const lbrace_column = astgen.source_column;
-        _ = try fullBodyExpr(&fn_gz, params_scope, .{ .rl = .none }, body_node);
+        _ = try fullBodyExpr(&fn_gz, params_scope, .{ .rl = .none }, body_node, .allow_branch_hint);
         try checkUsed(gz, &fn_gz.base, params_scope);
         if (!fn_gz.endsWithNoReturn()) {
 @@ -4552,20 +4585,20 @@ fn globalVarDecl(
     var align_gz = block_scope.makeSubBlock(scope);
     if (var_decl.ast.align_node != 0) {
-        const align_inst = try fullBodyExpr(&align_gz, &align_gz.base, coerced_align_ri, var_decl.ast.align_node);
+        const align_inst = try fullBodyExpr(&align_gz, &align_gz.base, coerced_align_ri, var_decl.ast.align_node, .normal);
         _ = try align_gz.addBreakWithSrcNode(.break_inline, decl_inst, align_inst, node);
     }
     var linksection_gz = align_gz.makeSubBlock(scope);
     if (var_decl.ast.section_node != 0) {
-        const linksection_inst = try fullBodyExpr(&linksection_gz, &linksection_gz.base, coerced_linksection_ri, var_decl.ast.section_node);
+        const linksection_inst = try fullBodyExpr(&linksection_gz, &linksection_gz.base, coerced_linksection_ri, var_decl.ast.section_node, .normal);
         _ = try linksection_gz.addBreakWithSrcNode(.break_inline, decl_inst, linksection_inst, node);
     }
     var addrspace_gz = linksection_gz.makeSubBlock(scope);
     if (var_decl.ast.addrspace_node != 0) {
         const addrspace_ty = try addrspace_gz.addBuiltinValue(var_decl.ast.addrspace_node, .address_space);
-        const addrspace_inst = try fullBodyExpr(&addrspace_gz, &addrspace_gz.base, .{ .rl = .{ .coerced_ty = addrspace_ty } }, var_decl.ast.addrspace_node);
+        const addrspace_inst = try fullBodyExpr(&addrspace_gz, &addrspace_gz.base, .{ .rl = .{ .coerced_ty = addrspace_ty } }, var_decl.ast.addrspace_node, .normal);
         _ = try addrspace_gz.addBreakWithSrcNode(.break_inline, decl_inst, addrspace_inst, node);
     }
 @@ -4622,7 +4655,7 @@ fn comptimeDecl(
     };
     defer decl_block.unstack();
-    const block_result = try fullBodyExpr(&decl_block, &decl_block.base, .{ .rl = .none }, body_node);
+    const block_result = try fullBodyExpr(&decl_block, &decl_block.base, .{ .rl = .none }, body_node, .normal);
     if (decl_block.isEmpty() or !decl_block.refIsNoReturn(block_result)) {
         _ = try decl_block.addBreak(.break_inline, decl_inst, .void_value);
     }
 @@ -4843,7 +4876,7 @@ fn testDecl(
     const lbrace_line = astgen.source_line - decl_block.decl_line;
     const lbrace_column = astgen.source_column;
-    const block_result = try fullBodyExpr(&fn_block, &fn_block.base, .{ .rl = .none }, body_node);
+    const block_result = try fullBodyExpr(&fn_block, &fn_block.base, .{ .rl = .none }, body_node, .normal);
     if (fn_block.isEmpty() or !fn_block.refIsNoReturn(block_result)) {
         // As our last action before the return, "pop" the error trace if needed
 @@ -6112,7 +6145,7 @@ fn orelseCatchExpr(
         break :blk &err_val_scope.base;
     };
-    const else_result = try fullBodyExpr(&else_scope, else_sub_scope, block_scope.break_result_info, rhs);
+    const else_result = try fullBodyExpr(&else_scope, else_sub_scope, block_scope.break_result_info, rhs, .allow_branch_hint);
     if (!else_scope.endsWithNoReturn()) {
         // As our last action before the break, "pop" the error trace if needed
         if (do_err_trace)
 @@ -6280,7 +6313,7 @@ fn boolBinOp(
     var rhs_scope = gz.makeSubBlock(scope);
     defer rhs_scope.unstack();
-    const rhs = try fullBodyExpr(&rhs_scope, &rhs_scope.base, coerced_bool_ri, node_datas[node].rhs);
+    const rhs = try fullBodyExpr(&rhs_scope, &rhs_scope.base, coerced_bool_ri, node_datas[node].rhs, .allow_branch_hint);
     if (!gz.refIsNoReturn(rhs)) {
         _ = try rhs_scope.addBreakWithSrcNode(.break_inline, bool_br, rhs, node_datas[node].rhs);
     }
 @@ -6424,7 +6457,7 @@ fn ifExpr(
         }
     };
-    const then_result = try fullBodyExpr(&then_scope, then_sub_scope, block_scope.break_result_info, then_node);
+    const then_result = try fullBodyExpr(&then_scope, then_sub_scope, block_scope.break_result_info, then_node, .allow_branch_hint);
     try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
     if (!then_scope.endsWithNoReturn()) {
         _ = try then_scope.addBreakWithSrcNode(.@"break", block, then_result, then_node);
 @@ -6466,7 +6499,7 @@ fn ifExpr(
                 break :s &else_scope.base;
             }
         };
-        const else_result = try fullBodyExpr(&else_scope, sub_scope, block_scope.break_result_info, else_node);
+        const else_result = try fullBodyExpr(&else_scope, sub_scope, block_scope.break_result_info, else_node, .allow_branch_hint);
         if (!else_scope.endsWithNoReturn()) {
             // As our last action before the break, "pop" the error trace if needed
             if (do_err_trace)
 @@ -6575,7 +6608,7 @@ fn whileExpr(
     } = c: {
         if (while_full.error_token) |_| {
             const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none };
-            const err_union = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr);
+            const err_union = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr, .normal);
             const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_err_ptr else .is_non_err;
             break :c .{
                 .inst = err_union,
 @@ -6583,14 +6616,14 @@ fn whileExpr(
             };
         } else if (while_full.payload_token) |_| {
             const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none };
-            const optional = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr);
+            const optional = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr, .normal);
             const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_null_ptr else .is_non_null;
             break :c .{
                 .inst = optional,
                 .bool_bit = try cond_scope.addUnNode(tag, optional, while_full.ast.cond_expr),
             };
         } else {
-            const cond = try fullBodyExpr(&cond_scope, &cond_scope.base, coerced_bool_ri, while_full.ast.cond_expr);
+            const cond = try fullBodyExpr(&cond_scope, &cond_scope.base, coerced_bool_ri, while_full.ast.cond_expr, .normal);
             break :c .{
                 .inst = cond,
                 .bool_bit = cond,
 @@ -6715,7 +6748,7 @@ fn whileExpr(
     continue_scope.instructions_top = continue_scope.instructions.items.len;
     {
         try emitDbgNode(&continue_scope, then_node);
-        const unused_result = try fullBodyExpr(&continue_scope, &continue_scope.base, .{ .rl = .none }, then_node);
+        const unused_result = try fullBodyExpr(&continue_scope, &continue_scope.base, .{ .rl = .none }, then_node, .allow_branch_hint);
         _ = try addEnsureResult(&continue_scope, unused_result, then_node);
     }
     try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
 @@ -6761,7 +6794,7 @@ fn whileExpr(
         // control flow apply to outer loops; not this one.
         loop_scope.continue_block = .none;
         loop_scope.break_block = .none;
-        const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node);
+        const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node, .allow_branch_hint);
         if (is_statement) {
             _ = try addEnsureResult(&else_scope, else_result, else_node);
         }
 @@ -7029,7 +7062,7 @@ fn forExpr(
         break :blk capture_sub_scope;
     };
-    const then_result = try fullBodyExpr(&then_scope, then_sub_scope, .{ .rl = .none }, then_node);
+    const then_result = try fullBodyExpr(&then_scope, then_sub_scope, .{ .rl = .none }, then_node, .allow_branch_hint);
     _ = try addEnsureResult(&then_scope, then_result, then_node);
     try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
 @@ -7048,7 +7081,7 @@ fn forExpr(
         // control flow apply to outer loops; not this one.
         loop_scope.continue_block = .none;
         loop_scope.break_block = .none;
-        const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node);
+        const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node, .allow_branch_hint);
         if (is_statement) {
             _ = try addEnsureResult(&else_scope, else_result, else_node);
         }
 @@ -7525,7 +7558,7 @@ fn switchExprErrUnion(
             }
             const target_expr_node = case.ast.target_expr;
-            const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node);
+            const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node, .allow_branch_hint);
             // check capture_scope, not err_scope to avoid false positive unused error capture
             try checkUsed(parent_gz, &case_scope.base, err_scope.parent);
             const uses_err = err_scope.used != 0 or err_scope.discarded != 0;
 @@ -7986,7 +8019,7 @@ fn switchExpr(
                 try case_scope.addDbgVar(.dbg_var_val, dbg_var_tag_name, dbg_var_tag_inst);
             }
             const target_expr_node = case.ast.target_expr;
-            const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node);
+            const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node, .allow_branch_hint);
             try checkUsed(parent_gz, &case_scope.base, sub_scope);
             if (!parent_gz.refIsNoReturn(case_result)) {
                 _ = try case_scope.addBreakWithSrcNode(.@"break", switch_block, case_result, target_expr_node);
 @@ -9154,6 +9187,7 @@ fn builtinCall(
     ri: ResultInfo,
     node: Ast.Node.Index,
     params: []const Ast.Node.Index,
+    allow_branch_hint: bool,
 ) InnerError!Zir.Inst.Ref {
     const astgen = gz.astgen;
     const tree = astgen.tree;
 @@ -9187,6 +9221,18 @@ fn builtinCall(
         return astgen.failNode(node, "'{s}' outside function scope", .{builtin_name});
     switch (info.tag) {
+        .branch_hint => {
+            if (!allow_branch_hint) {
+                return astgen.failNode(node, "'@branchHint' must appear as the first statement in a function or conditional branch", .{});
+            }
+            const hint_ty = try gz.addBuiltinValue(node, .branch_hint);
+            const hint_val = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = hint_ty } }, params[0]);
+            _ = try gz.addExtendedPayload(.branch_hint, Zir.Inst.UnNode{
+                .node = gz.nodeIndexToRelative(node),
+                .operand = hint_val,
+            });
+            return rvalue(gz, ri, .void_value, node);
+        },
         .import => {
             const node_tags = tree.nodes.items(.tag);
             const operand_node = params[0];
 @@ -9294,14 +9340,6 @@ fn builtinCall(
             });
             return rvalue(gz, ri, .void_value, node);
         },
-        .set_cold => {
-            const order = try expr(gz, scope, ri, params[0]);
-            _ = try gz.addExtendedPayload(.set_cold, Zir.Inst.UnNode{
-                .node = gz.nodeIndexToRelative(node),
-                .operand = order,
-            });
-            return rvalue(gz, ri, .void_value, node);
-        },
         .src => {
             // Incorporate the source location into the source hash, so that
 @@ -9963,7 +10001,7 @@ fn cImport(
     defer block_scope.unstack();
     const block_inst = try gz.makeBlockInst(.c_import, node);
-    const block_result = try fullBodyExpr(&block_scope, &block_scope.base, .{ .rl = .none }, body_node);
+    const block_result = try fullBodyExpr(&block_scope, &block_scope.base, .{ .rl = .none }, body_node, .normal);
     _ = try gz.addUnNode(.ensure_result_used, block_result, node);
     if (!gz.refIsNoReturn(block_result)) {
         _ = try block_scope.addBreak(.break_inline, block_inst, .void_value);
 @@ -10046,7 +10084,7 @@ fn callExpr(
         defer arg_block.unstack();
         // `call_inst` is reused to provide the param type.
-        const arg_ref = try fullBodyExpr(&arg_block, &arg_block.base, .{ .rl = .{ .coerced_ty = call_inst }, .ctx = .fn_arg }, param_node);
+        const arg_ref = try fullBodyExpr(&arg_block, &arg_block.base, .{ .rl = .{ .coerced_ty = call_inst }, .ctx = .fn_arg }, param_node, .normal);
         _ = try arg_block.addBreakWithSrcNode(.break_inline, call_index, arg_ref, param_node);
         const body = arg_block.instructionsSlice();

lib/std/zig/AstRlAnnotate.zig+4-1

...	@@ -829,6 +829,10 @@ fn builtinCall(astrl: AstRlAnnotate, block: ?Block, ri: ResultInfo, node: Ast.	...	@@ -829,6 +829,10 @@ fn builtinCall(astrl: AstRlAnnotate, block: ?Block, ri: ResultInfo, node: Ast.
829	}	829	}
830	switch (info.tag) {	830	switch (info.tag) {
831	.import => return false,	831	.import => return false,
		832	.branch_hint => {
		833	_ = try astrl.expr(args[0], block, ResultInfo.type_only);
		834	return false;
		835	},
832	.compile_log, .TypeOf => {	836	.compile_log, .TypeOf => {
833	for (args) \|arg_node\| {	837	for (args) \|arg_node\| {
834	_ = try astrl.expr(arg_node, block, ResultInfo.none);	838	_ = try astrl.expr(arg_node, block, ResultInfo.none);
...	@@ -907,7 +911,6 @@ fn builtinCall(astrl: AstRlAnnotate, block: ?Block, ri: ResultInfo, node: Ast.	...	@@ -907,7 +911,6 @@ fn builtinCall(astrl: AstRlAnnotate, block: ?Block, ri: ResultInfo, node: Ast.
907	.fence,	911	.fence,
908	.set_float_mode,	912	.set_float_mode,
909	.set_align_stack,	913	.set_align_stack,
910	.set_cold,
911	.type_info,	914	.type_info,
912	.work_item_id,	915	.work_item_id,
913	.work_group_size,	916	.work_group_size,

lib/std/zig/BuiltinFn.zig+9-9

 @@ -14,6 +14,7 @@ pub const Tag = enum {
     bit_offset_of,
     int_from_bool,
     bit_size_of,
+    branch_hint,
     breakpoint,
     disable_instrumentation,
     mul_add,
 @@ -82,7 +83,6 @@ pub const Tag = enum {
     return_address,
     select,
     set_align_stack,
-    set_cold,
     set_eval_branch_quota,
     set_float_mode,
     set_runtime_safety,
 @@ -256,6 +256,14 @@ pub const list = list: {
                 .param_count = 1,
             },
         },
+        .{
+            "@branchHint",
+            .{
+                .tag = .branch_hint,
+                .param_count = 1,
+                .illegal_outside_function = true,
+            },
+        },
         .{
             "@breakpoint",
             .{
 @@ -744,14 +752,6 @@ pub const list = list: {
                 .illegal_outside_function = true,
             },
         },
-        .{
-            "@setCold",
-            .{
-                .tag = .set_cold,
-                .param_count = 1,
-                .illegal_outside_function = true,
-            },
-        },
         .{
             "@setEvalBranchQuota",
             .{

lib/std/zig/Zir.zig+7-5

...	@@ -1546,7 +1546,7 @@ pub const Inst = struct {	...	@@ -1546,7 +1546,7 @@ pub const Inst = struct {
1546	=> false,	1546	=> false,
1547		1547
1548	.extended => switch (data.extended.opcode) {	1548	.extended => switch (data.extended.opcode) {
1549	.fence, .set_cold, .breakpoint, .disable_instrumentation => true,	1549	.fence, .branch_hint, .breakpoint, .disable_instrumentation => true,
1550	else => false,	1550	else => false,
1551	},	1551	},
1552	};	1552	};
...	@@ -1954,9 +1954,6 @@ pub const Inst = struct {	...	@@ -1954,9 +1954,6 @@ pub const Inst = struct {
1954	/// Implement builtin `@setAlignStack`.	1954	/// Implement builtin `@setAlignStack`.
1955	/// `operand` is payload index to `UnNode`.	1955	/// `operand` is payload index to `UnNode`.
1956	set_align_stack,	1956	set_align_stack,
1957	/// Implements `@setCold`.
1958	/// `operand` is payload index to `UnNode`.
1959	set_cold,
1960	/// Implements the `@errorCast` builtin.	1957	/// Implements the `@errorCast` builtin.
1961	/// `operand` is payload index to `BinNode`. `lhs` is dest type, `rhs` is operand.	1958	/// `operand` is payload index to `BinNode`. `lhs` is dest type, `rhs` is operand.
1962	error_cast,	1959	error_cast,
...	@@ -2051,6 +2048,10 @@ pub const Inst = struct {	...	@@ -2051,6 +2048,10 @@ pub const Inst = struct {
2051	/// `operand` is `src_node: i32`.	2048	/// `operand` is `src_node: i32`.
2052	/// `small` is an `Inst.BuiltinValue`.	2049	/// `small` is an `Inst.BuiltinValue`.
2053	builtin_value,	2050	builtin_value,
		2051	/// Provide a `@branchHint` for the current block.
		2052	/// `operand` is payload index to `UnNode`.
		2053	/// `small` is unused.
		2054	branch_hint,
2054		2055
2055	pub const InstData = struct {	2056	pub const InstData = struct {
2056	opcode: Extended,	2057	opcode: Extended,
...	@@ -3142,6 +3143,7 @@ pub const Inst = struct {	...	@@ -3142,6 +3143,7 @@ pub const Inst = struct {
3142	export_options,	3143	export_options,
3143	extern_options,	3144	extern_options,
3144	type_info,	3145	type_info,
		3146	branch_hint,
3145	// Values	3147	// Values
3146	calling_convention_c,	3148	calling_convention_c,
3147	calling_convention_inline,	3149	calling_convention_inline,
...	@@ -3962,7 +3964,6 @@ fn findDeclsInner(	...	@@ -3962,7 +3964,6 @@ fn findDeclsInner(
3962	.fence,	3964	.fence,
3963	.set_float_mode,	3965	.set_float_mode,
3964	.set_align_stack,	3966	.set_align_stack,
3965	.set_cold,
3966	.error_cast,	3967	.error_cast,
3967	.await_nosuspend,	3968	.await_nosuspend,
3968	.breakpoint,	3969	.breakpoint,
...	@@ -3986,6 +3987,7 @@ fn findDeclsInner(	...	@@ -3986,6 +3987,7 @@ fn findDeclsInner(
3986	.closure_get,	3987	.closure_get,
3987	.field_parent_ptr,	3988	.field_parent_ptr,
3988	.builtin_value,	3989	.builtin_value,
		3990	.branch_hint,
3989	=> return,	3991	=> return,
3990		3992
3991	// `@TypeOf` has a body.	3993	// `@TypeOf` has a body.

src/Air.zig+109-6

...	@@ -433,13 +433,18 @@ pub const Inst = struct {	...	@@ -433,13 +433,18 @@ pub const Inst = struct {
433	/// In the case of non-error, control flow proceeds to the next instruction	433	/// In the case of non-error, control flow proceeds to the next instruction
434	/// after the `try`, with the result of this instruction being the unwrapped	434	/// after the `try`, with the result of this instruction being the unwrapped
435	/// payload value, as if `unwrap_errunion_payload` was executed on the operand.	435	/// payload value, as if `unwrap_errunion_payload` was executed on the operand.
		436	/// The error branch is considered to have a branch hint of `.unlikely`.
436	/// Uses the `pl_op` field. Payload is `Try`.	437	/// Uses the `pl_op` field. Payload is `Try`.
437	@"try",	438	@"try",
		439	/// Same as `try` except the error branch hint is `.cold`.
		440	try_cold,
438	/// Same as `try` except the operand is a pointer to an error union, and the	441	/// Same as `try` except the operand is a pointer to an error union, and the
439	/// result is a pointer to the payload. Result is as if `unwrap_errunion_payload_ptr`	442	/// result is a pointer to the payload. Result is as if `unwrap_errunion_payload_ptr`
440	/// was executed on the operand.	443	/// was executed on the operand.
441	/// Uses the `ty_pl` field. Payload is `TryPtr`.	444	/// Uses the `ty_pl` field. Payload is `TryPtr`.
442	try_ptr,	445	try_ptr,
		446	/// Same as `try_ptr` except the error branch hint is `.cold`.
		447	try_ptr_cold,
443	/// Notes the beginning of a source code statement and marks the line and column.	448	/// Notes the beginning of a source code statement and marks the line and column.
444	/// Result type is always void.	449	/// Result type is always void.
445	/// Uses the `dbg_stmt` field.	450	/// Uses the `dbg_stmt` field.
...	@@ -1116,11 +1121,20 @@ pub const Call = struct {	...	@@ -1116,11 +1121,20 @@ pub const Call = struct {
1116	pub const CondBr = struct {	1121	pub const CondBr = struct {
1117	then_body_len: u32,	1122	then_body_len: u32,
1118	else_body_len: u32,	1123	else_body_len: u32,
		1124	branch_hints: BranchHints,
		1125	pub const BranchHints = packed struct(u32) {
		1126	true: std.builtin.BranchHint,
		1127	false: std.builtin.BranchHint,
		1128	_: u26 = 0,
		1129	};
1119	};	1130	};
1120		1131
1121	/// Trailing:	1132	/// Trailing:
1122	/// * 0. `Case` for each `cases_len`	1133	/// * 0. `BranchHint` for each `cases_len + 1`. bit-packed into `u32`
1123	/// * 1. the else body, according to `else_body_len`.	1134	/// elems such that each `u32` contains up to 10x `BranchHint`.
		1135	/// LSBs are first case. Final hint is `else`.
		1136	/// * 1. `Case` for each `cases_len`
		1137	/// * 2. the else body, according to `else_body_len`.
1124	pub const SwitchBr = struct {	1138	pub const SwitchBr = struct {
1125	cases_len: u32,	1139	cases_len: u32,
1126	else_body_len: u32,	1140	else_body_len: u32,
...	@@ -1380,6 +1394,7 @@ pub fn typeOfIndex(air: const Air, inst: Air.Inst.Index, ip: const InternPool)	...	@@ -1380,6 +1394,7 @@ pub fn typeOfIndex(air: const Air, inst: Air.Inst.Index, ip: const InternPool)
1380	.ptr_add,	1394	.ptr_add,
1381	.ptr_sub,	1395	.ptr_sub,
1382	.try_ptr,	1396	.try_ptr,
		1397	.try_ptr_cold,
1383	=> return datas[@intFromEnum(inst)].ty_pl.ty.toType(),	1398	=> return datas[@intFromEnum(inst)].ty_pl.ty.toType(),
1384		1399
1385	.not,	1400	.not,
...	@@ -1500,7 +1515,7 @@ pub fn typeOfIndex(air: const Air, inst: Air.Inst.Index, ip: const InternPool)	...	@@ -1500,7 +1515,7 @@ pub fn typeOfIndex(air: const Air, inst: Air.Inst.Index, ip: const InternPool)
1500	return air.typeOf(extra.lhs, ip);	1515	return air.typeOf(extra.lhs, ip);
1501	},	1516	},
1502		1517
1503	.@"try" => {	1518	.@"try", .try_cold => {
1504	const err_union_ty = air.typeOf(datas[@intFromEnum(inst)].pl_op.operand, ip);	1519	const err_union_ty = air.typeOf(datas[@intFromEnum(inst)].pl_op.operand, ip);
1505	return Type.fromInterned(ip.indexToKey(err_union_ty.ip_index).error_union_type.payload_type);	1520	return Type.fromInterned(ip.indexToKey(err_union_ty.ip_index).error_union_type.payload_type);
1506	},	1521	},
...	@@ -1524,9 +1539,8 @@ pub fn extraData(air: Air, comptime T: type, index: usize) struct { data: T, end	...	@@ -1524,9 +1539,8 @@ pub fn extraData(air: Air, comptime T: type, index: usize) struct { data: T, end
1524	inline for (fields) \|field\| {	1539	inline for (fields) \|field\| {
1525	@field(result, field.name) = switch (field.type) {	1540	@field(result, field.name) = switch (field.type) {
1526	u32 => air.extra[i],	1541	u32 => air.extra[i],
1527	Inst.Ref => @as(Inst.Ref, @enumFromInt(air.extra[i])),	1542	InternPool.Index, Inst.Ref => @enumFromInt(air.extra[i]),
1528	i32 => @as(i32, @bitCast(air.extra[i])),	1543	i32, CondBr.BranchHints => @bitCast(air.extra[i]),
1529	InternPool.Index => @as(InternPool.Index, @enumFromInt(air.extra[i])),
1530	else => @compileError("bad field type: " ++ @typeName(field.type)),	1544	else => @compileError("bad field type: " ++ @typeName(field.type)),
1531	};	1545	};
1532	i += 1;	1546	i += 1;
...	@@ -1593,7 +1607,9 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {	...	@@ -1593,7 +1607,9 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1593	.cond_br,	1607	.cond_br,
1594	.switch_br,	1608	.switch_br,
1595	.@"try",	1609	.@"try",
		1610	.try_cold,
1596	.try_ptr,	1611	.try_ptr,
		1612	.try_ptr_cold,
1597	.dbg_stmt,	1613	.dbg_stmt,
1598	.dbg_inline_block,	1614	.dbg_inline_block,
1599	.dbg_var_ptr,	1615	.dbg_var_ptr,
...	@@ -1796,4 +1812,91 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {	...	@@ -1796,4 +1812,91 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1796	};	1812	};
1797	}	1813	}
1798		1814
		1815	pub const UnwrappedSwitch = struct {
		1816	air: *const Air,
		1817	operand: Inst.Ref,
		1818	cases_len: u32,
		1819	else_body_len: u32,
		1820	branch_hints_start: u32,
		1821	cases_start: u32,
		1822
		1823	/// Asserts that `case_idx < us.cases_len`.
		1824	pub fn getHint(us: UnwrappedSwitch, case_idx: u32) std.builtin.BranchHint {
		1825	assert(case_idx < us.cases_len);
		1826	return us.getHintInner(case_idx);
		1827	}
		1828	pub fn getElseHint(us: UnwrappedSwitch) std.builtin.BranchHint {
		1829	return us.getHintInner(us.cases_len);
		1830	}
		1831	fn getHintInner(us: UnwrappedSwitch, idx: u32) std.builtin.BranchHint {
		1832	const bag = us.air.extra[us.branch_hints_start..][idx / 10];
		1833	const bits: u3 = @truncate(bag >> @intCast(3 * (idx % 10)));
		1834	return @enumFromInt(bits);
		1835	}
		1836
		1837	pub fn iterateCases(us: UnwrappedSwitch) CaseIterator {
		1838	return .{
		1839	.air = us.air,
		1840	.cases_len = us.cases_len,
		1841	.else_body_len = us.else_body_len,
		1842	.next_case = 0,
		1843	.extra_index = us.cases_start,
		1844	};
		1845	}
		1846	pub const CaseIterator = struct {
		1847	air: *const Air,
		1848	cases_len: u32,
		1849	else_body_len: u32,
		1850	next_case: u32,
		1851	extra_index: u32,
		1852
		1853	pub fn next(it: *CaseIterator) ?Case {
		1854	if (it.next_case == it.cases_len) return null;
		1855	const idx = it.next_case;
		1856	it.next_case += 1;
		1857
		1858	const extra = it.air.extraData(SwitchBr.Case, it.extra_index);
		1859	var extra_index = extra.end;
		1860	const items: []const Inst.Ref = @ptrCast(it.air.extra[extra_index..][0..extra.data.items_len]);
		1861	extra_index += items.len;
		1862	const body: []const Inst.Index = @ptrCast(it.air.extra[extra_index..][0..extra.data.body_len]);
		1863	extra_index += body.len;
		1864	it.extra_index = @intCast(extra_index);
		1865
		1866	return .{
		1867	.idx = idx,
		1868	.items = items,
		1869	.body = body,
		1870	};
		1871	}
		1872	/// Only valid to call once all cases have been iterated, i.e. `next` returns `null`.
		1873	/// Returns the body of the "default" (`else`) case.
		1874	pub fn elseBody(it: *CaseIterator) []const Inst.Index {
		1875	assert(it.next_case == it.cases_len);
		1876	return @ptrCast(it.air.extra[it.extra_index..][0..it.else_body_len]);
		1877	}
		1878	pub const Case = struct {
		1879	idx: u32,
		1880	items: []const Inst.Ref,
		1881	body: []const Inst.Index,
		1882	};
		1883	};
		1884	};
		1885
		1886	pub fn unwrapSwitch(air: *const Air, switch_inst: Inst.Index) UnwrappedSwitch {
		1887	const inst = air.instructions.get(@intFromEnum(switch_inst));
		1888	assert(inst.tag == .switch_br);
		1889	const pl_op = inst.data.pl_op;
		1890	const extra = air.extraData(SwitchBr, pl_op.payload);
		1891	const hint_bag_count = std.math.divCeil(usize, extra.data.cases_len + 1, 10) catch unreachable;
		1892	return .{
		1893	.air = air,
		1894	.operand = pl_op.operand,
		1895	.cases_len = extra.data.cases_len,
		1896	.else_body_len = extra.data.else_body_len,
		1897	.branch_hints_start = @intCast(extra.end),
		1898	.cases_start = @intCast(extra.end + hint_bag_count),
		1899	};
		1900	}
		1901
1799	pub const typesFullyResolved = @import("Air/types_resolved.zig").typesFullyResolved;	1902	pub const typesFullyResolved = @import("Air/types_resolved.zig").typesFullyResolved;

src/Air/types_resolved.zig+9-22

 @@ -344,7 +344,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
                 if (!checkRef(data.pl_op.operand, zcu)) return false;
             },
-            .@"try" => {
+            .@"try", .try_cold => {
                 const extra = air.extraData(Air.Try, data.pl_op.payload);
                 if (!checkRef(data.pl_op.operand, zcu)) return false;
                 if (!checkBody(
 @@ -354,7 +354,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
                 )) return false;
             },
-            .try_ptr => {
+            .try_ptr, .try_ptr_cold => {
                 const extra = air.extraData(Air.TryPtr, data.ty_pl.payload);
                 if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
                 if (!checkRef(extra.data.ptr, zcu)) return false;
 @@ -381,27 +381,14 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
             },
             .switch_br => {
-                const extra = air.extraData(Air.SwitchBr, data.pl_op.payload);
+                const switch_br = air.unwrapSwitch(inst);
-                if (!checkRef(data.pl_op.operand, zcu)) return false;
+                if (!checkRef(switch_br.operand, zcu)) return false;
-                var extra_index = extra.end;
+                var it = switch_br.iterateCases();
-                for (0..extra.data.cases_len) |_| {
+                while (it.next()) |case| {
-                    const case = air.extraData(Air.SwitchBr.Case, extra_index);
+                    for (case.items) |item| if (!checkRef(item, zcu)) return false;
-                    extra_index = case.end;
+                    if (!checkBody(air, case.body, zcu)) return false;
-                    const items: []const Air.Inst.Ref = @ptrCast(air.extra[extra_index..][0..case.data.items_len]);
-                    extra_index += case.data.items_len;
-                    for (items) |item| if (!checkRef(item, zcu)) return false;
-                    if (!checkBody(
-                        air,
-                        @ptrCast(air.extra[extra_index..][0..case.data.body_len]),
-                        zcu,
-                    )) return false;
-                    extra_index += case.data.body_len;
                 }
-                if (!checkBody(
+                if (!checkBody(air, it.elseBody(), zcu)) return false;
-                    air,
-                    @ptrCast(air.extra[extra_index..][0..extra.data.else_body_len]),
-                    zcu,
-                )) return false;
             },
             .assembly => {

src/InternPool.zig+17-18

 @@ -2121,6 +2121,17 @@ pub const Key = union(enum) {
             @atomicStore(FuncAnalysis, analysis_ptr, analysis, .release);
         }
+        pub fn setBranchHint(func: Func, ip: *InternPool, hint: std.builtin.BranchHint) void {
+            const extra_mutex = &ip.getLocal(func.tid).mutate.extra.mutex;
+            extra_mutex.lock();
+            defer extra_mutex.unlock();
+            const analysis_ptr = func.analysisPtr(ip);
+            var analysis = analysis_ptr.*;
+            analysis.branch_hint = hint;
+            @atomicStore(FuncAnalysis, analysis_ptr, analysis, .release);
+        }
         /// Returns a pointer that becomes invalid after any additions to the `InternPool`.
         fn zirBodyInstPtr(func: Func, ip: *InternPool) *TrackedInst.Index {
             const extra = ip.getLocalShared(func.tid).extra.acquire();
 @@ -5575,7 +5586,7 @@ pub const Tag = enum(u8) {
 /// to be part of the type of the function.
 pub const FuncAnalysis = packed struct(u32) {
     state: State,
-    is_cold: bool,
+    branch_hint: std.builtin.BranchHint,
     is_noinline: bool,
     calls_or_awaits_errorable_fn: bool,
     stack_alignment: Alignment,
 @@ -5583,7 +5594,7 @@ pub const FuncAnalysis = packed struct(u32) {
     inferred_error_set: bool,
     disable_instrumentation: bool,
-    _: u19 = 0,
+    _: u17 = 0,
     pub const State = enum(u2) {
         /// The runtime function has never been referenced.
 @@ -8636,7 +8647,7 @@ pub fn getFuncDecl(
     const func_decl_extra_index = addExtraAssumeCapacity(extra, Tag.FuncDecl{
         .analysis = .{
             .state = .unreferenced,
-            .is_cold = false,
+            .branch_hint = .none,
             .is_noinline = key.is_noinline,
             .calls_or_awaits_errorable_fn = false,
             .stack_alignment = .none,
 @@ -8740,7 +8751,7 @@ pub fn getFuncDeclIes(
     const func_decl_extra_index = addExtraAssumeCapacity(extra, Tag.FuncDecl{
         .analysis = .{
             .state = .unreferenced,
-            .is_cold = false,
+            .branch_hint = .none,
             .is_noinline = key.is_noinline,
             .calls_or_awaits_errorable_fn = false,
             .stack_alignment = .none,
 @@ -8932,7 +8943,7 @@ pub fn getFuncInstance(
     const func_extra_index = addExtraAssumeCapacity(extra, Tag.FuncInstance{
         .analysis = .{
             .state = .unreferenced,
-            .is_cold = false,
+            .branch_hint = .none,
             .is_noinline = arg.is_noinline,
             .calls_or_awaits_errorable_fn = false,
             .stack_alignment = .none,
 @@ -9032,7 +9043,7 @@ pub fn getFuncInstanceIes(
     const func_extra_index = addExtraAssumeCapacity(extra, Tag.FuncInstance{
         .analysis = .{
             .state = .unreferenced,
-            .is_cold = false,
+            .branch_hint = .none,
             .is_noinline = arg.is_noinline,
             .calls_or_awaits_errorable_fn = false,
             .stack_alignment = .none,
 @@ -11853,18 +11864,6 @@ pub fn funcSetDisableInstrumentation(ip: *InternPool, func: Index) void {
     @atomicStore(FuncAnalysis, analysis_ptr, analysis, .release);
 }
-pub fn funcSetCold(ip: *InternPool, func: Index, is_cold: bool) void {
-    const unwrapped_func = func.unwrap(ip);
-    const extra_mutex = &ip.getLocal(unwrapped_func.tid).mutate.extra.mutex;
-    extra_mutex.lock();
-    defer extra_mutex.unlock();
-    const analysis_ptr = ip.funcAnalysisPtr(func);
-    var analysis = analysis_ptr.*;
-    analysis.is_cold = is_cold;
-    @atomicStore(FuncAnalysis, analysis_ptr, analysis, .release);
-}
 pub fn funcZirBodyInst(ip: *const InternPool, func: Index) TrackedInst.Index {
     const unwrapped_func = func.unwrap(ip);
     const item = unwrapped_func.getItem(ip);

src/Liveness.zig+15-21

 @@ -658,10 +658,10 @@ pub fn categorizeOperand(
             return .complex;
         },
-        .@"try" => {
+        .@"try", .try_cold => {
             return .complex;
         },
-        .try_ptr => {
+        .try_ptr, .try_ptr_cold => {
             return .complex;
         },
         .loop => {
 @@ -1254,8 +1254,8 @@ fn analyzeInst(
         },
         .loop => return analyzeInstLoop(a, pass, data, inst),
-        .@"try" => return analyzeInstCondBr(a, pass, data, inst, .@"try"),
+        .@"try", .try_cold => return analyzeInstCondBr(a, pass, data, inst, .@"try"),
-        .try_ptr => return analyzeInstCondBr(a, pass, data, inst, .try_ptr),
+        .try_ptr, .try_ptr_cold => return analyzeInstCondBr(a, pass, data, inst, .try_ptr),
         .cond_br => return analyzeInstCondBr(a, pass, data, inst, .cond_br),
         .switch_br => return analyzeInstSwitchBr(a, pass, data, inst),
 @@ -1674,21 +1674,18 @@ fn analyzeInstSwitchBr(
     const inst_datas = a.air.instructions.items(.data);
     const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
     const condition = pl_op.operand;
-    const switch_br = a.air.extraData(Air.SwitchBr, pl_op.payload);
+    const switch_br = a.air.unwrapSwitch(inst);
     const gpa = a.gpa;
-    const ncases = switch_br.data.cases_len;
+    const ncases = switch_br.cases_len;
     switch (pass) {
         .loop_analysis => {
-            var air_extra_index: usize = switch_br.end;
+            var it = switch_br.iterateCases();
-            for (0..ncases) |_| {
+            while (it.next()) |case| {
-                const case = a.air.extraData(Air.SwitchBr.Case, air_extra_index);
+                try analyzeBody(a, pass, data, case.body);
-                const case_body: []const Air.Inst.Index = @ptrCast(a.air.extra[case.end + case.data.items_len ..][0..case.data.body_len]);
-                air_extra_index = case.end + case.data.items_len + case_body.len;
-                try analyzeBody(a, pass, data, case_body);
             }
             { // else
-                const else_body: []const Air.Inst.Index = @ptrCast(a.air.extra[air_extra_index..][0..switch_br.data.else_body_len]);
+                const else_body = it.elseBody();
                 try analyzeBody(a, pass, data, else_body);
             }
         },
 @@ -1706,16 +1703,13 @@ fn analyzeInstSwitchBr(
             @memset(case_live_sets, .{});
             defer for (case_live_sets) |*live_set| live_set.deinit(gpa);
-            var air_extra_index: usize = switch_br.end;
+            var case_it = switch_br.iterateCases();
-            for (case_live_sets[0..ncases]) |*live_set| {
+            while (case_it.next()) |case| {
-                const case = a.air.extraData(Air.SwitchBr.Case, air_extra_index);
+                try analyzeBody(a, pass, data, case.body);
-                const case_body: []const Air.Inst.Index = @ptrCast(a.air.extra[case.end + case.data.items_len ..][0..case.data.body_len]);
+                case_live_sets[case.idx] = data.live_set.move();
-                air_extra_index = case.end + case.data.items_len + case_body.len;
-                try analyzeBody(a, pass, data, case_body);
-                live_set.* = data.live_set.move();
             }
             { // else
-                const else_body: []const Air.Inst.Index = @ptrCast(a.air.extra[air_extra_index..][0..switch_br.data.else_body_len]);
+                const else_body = case_it.elseBody();
                 try analyzeBody(a, pass, data, else_body);
                 case_live_sets[ncases] = data.live_set.move();
             }

src/Liveness/Verify.zig+11-22

...	@@ -374,7 +374,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {	...	@@ -374,7 +374,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
374	},	374	},
375		375
376	// control flow	376	// control flow
377	.@"try" => {	377	.@"try", .try_cold => {
378	const pl_op = data[@intFromEnum(inst)].pl_op;	378	const pl_op = data[@intFromEnum(inst)].pl_op;
379	const extra = self.air.extraData(Air.Try, pl_op.payload);	379	const extra = self.air.extraData(Air.Try, pl_op.payload);
380	const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);	380	const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
...	@@ -396,7 +396,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {	...	@@ -396,7 +396,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
396		396
397	try self.verifyInst(inst);	397	try self.verifyInst(inst);
398	},	398	},
399	.try_ptr => {	399	.try_ptr, .try_ptr_cold => {
400	const ty_pl = data[@intFromEnum(inst)].ty_pl;	400	const ty_pl = data[@intFromEnum(inst)].ty_pl;
401	const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);	401	const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
402	const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);	402	const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
...	@@ -509,44 +509,33 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {	...	@@ -509,44 +509,33 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
509	try self.verifyInst(inst);	509	try self.verifyInst(inst);
510	},	510	},
511	.switch_br => {	511	.switch_br => {
512	const pl_op = data[@intFromEnum(inst)].pl_op;	512	const switch_br = self.air.unwrapSwitch(inst);
513	const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
514	var extra_index = switch_br.end;
515	var case_i: u32 = 0;
516	const switch_br_liveness = try self.liveness.getSwitchBr(	513	const switch_br_liveness = try self.liveness.getSwitchBr(
517	self.gpa,	514	self.gpa,
518	inst,	515	inst,
519	switch_br.data.cases_len + 1,	516	switch_br.cases_len + 1,
520	);	517	);
521	defer self.gpa.free(switch_br_liveness.deaths);	518	defer self.gpa.free(switch_br_liveness.deaths);
522		519
523	try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));	520	try self.verifyOperand(inst, switch_br.operand, self.liveness.operandDies(inst, 0));
524		521
525	var live = self.live.move();	522	var live = self.live.move();
526	defer live.deinit(self.gpa);	523	defer live.deinit(self.gpa);
527		524
528	while (case_i < switch_br.data.cases_len) : (case_i += 1) {	525	var it = switch_br.iterateCases();
529	const case = self.air.extraData(Air.SwitchBr.Case, extra_index);	526	while (it.next()) \|case\| {
530	const items = @as(
531	[]const Air.Inst.Ref,
532	@ptrCast(self.air.extra[case.end..][0..case.data.items_len]),
533	);
534	const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
535	extra_index = case.end + items.len + case_body.len;
536
537	self.live.deinit(self.gpa);	527	self.live.deinit(self.gpa);
538	self.live = try live.clone(self.gpa);	528	self.live = try live.clone(self.gpa);
539		529
540	for (switch_br_liveness.deaths[case_i]) \|death\| try self.verifyDeath(inst, death);	530	for (switch_br_liveness.deaths[case.idx]) \|death\| try self.verifyDeath(inst, death);
541	try self.verifyBody(case_body);	531	try self.verifyBody(case.body);
542	}	532	}
543		533
544	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);	534	const else_body = it.elseBody();
545	if (else_body.len > 0) {	535	if (else_body.len > 0) {
546	self.live.deinit(self.gpa);	536	self.live.deinit(self.gpa);
547	self.live = try live.clone(self.gpa);	537	self.live = try live.clone(self.gpa);
548		538	for (switch_br_liveness.deaths[switch_br.cases_len]) \|death\| try self.verifyDeath(inst, death);
549	for (switch_br_liveness.deaths[case_i]) \|death\| try self.verifyDeath(inst, death);
550	try self.verifyBody(else_body);	539	try self.verifyBody(else_body);
551	}	540	}
552		541

src/Sema.zig+258-107

 @@ -118,6 +118,10 @@ dependencies: std.AutoArrayHashMapUnmanaged(InternPool.Dependee, void) = .{},
 /// by `analyzeCall`.
 allow_memoize: bool = true,
+/// The `BranchHint` for the current branch of runtime control flow.
+/// This state is on `Sema` so that `cold` hints can be propagated up through blocks with less special handling.
+branch_hint: ?std.builtin.BranchHint = null,
 const MaybeComptimeAlloc = struct {
     /// The runtime index of the `alloc` instruction.
     runtime_index: Value.RuntimeIndex,
 @@ -892,7 +896,12 @@ pub fn deinit(sema: *Sema) void {
 /// Performs semantic analysis of a ZIR body which is behind a runtime condition. If comptime
 /// control flow happens here, Sema will convert it to runtime control flow by introducing post-hoc
 /// blocks where necessary.
-fn analyzeBodyRuntimeBreak(sema: *Sema, block: *Block, body: []const Zir.Inst.Index) !void {
+/// Returns the branch hint for this branch.
+fn analyzeBodyRuntimeBreak(sema: *Sema, block: *Block, body: []const Zir.Inst.Index) !std.builtin.BranchHint {
+    const parent_hint = sema.branch_hint;
+    defer sema.branch_hint = parent_hint;
+    sema.branch_hint = null;
     sema.analyzeBodyInner(block, body) catch |err| switch (err) {
         error.ComptimeBreak => {
             const zir_datas = sema.code.instructions.items(.data);
 @@ -902,6 +911,8 @@ fn analyzeBodyRuntimeBreak(sema: *Sema, block: *Block, body: []const Zir.Inst.In
         },
         else => |e| return e,
     };
+    return sema.branch_hint orelse .none;
 }
 /// Semantically analyze a ZIR function body. It is guranteed by AstGen that such a body cannot
 @@ -1304,11 +1315,6 @@ fn analyzeBodyInner(
                         i += 1;
                         continue;
                     },
-                    .set_cold => {
-                        try sema.zirSetCold(block, extended);
-                        i += 1;
-                        continue;
-                    },
                     .breakpoint => {
                         if (!block.is_comptime) {
                             _ = try block.addNoOp(.breakpoint);
 @@ -1326,6 +1332,11 @@ fn analyzeBodyInner(
                         i += 1;
                         continue;
                     },
+                    .branch_hint => {
+                        try sema.zirBranchHint(block, extended);
+                        i += 1;
+                        continue;
+                    },
                     .value_placeholder => unreachable, // never appears in a body
                     .field_parent_ptr => try sema.zirFieldParentPtr(block, extended),
                     .builtin_value => try sema.zirBuiltinValue(extended),
 @@ -5727,6 +5738,13 @@ fn zirPanic(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void
     if (block.is_comptime) {
         return sema.fail(block, src, "encountered @panic at comptime", .{});
     }
+    // We only apply the first hint in a branch.
+    // This allows user-provided hints to override implicit cold hints.
+    if (sema.branch_hint == null) {
+        sema.branch_hint = .cold;
+    }
     try sema.panicWithMsg(block, src, coerced_msg, .@"@panic");
 }
 @@ -6418,25 +6436,6 @@ fn zirSetAlignStack(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.Inst
     sema.allow_memoize = false;
 }
-fn zirSetCold(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!void {
-    const pt = sema.pt;
-    const zcu = pt.zcu;
-    const ip = &zcu.intern_pool;
-    const extra = sema.code.extraData(Zir.Inst.UnNode, extended.operand).data;
-    const operand_src = block.builtinCallArgSrc(extra.node, 0);
-    const is_cold = try sema.resolveConstBool(block, operand_src, extra.operand, .{
-        .needed_comptime_reason = "operand to @setCold must be comptime-known",
-    });
-    // TODO: should `@setCold` apply to the parent in an inline call?
-    // See also #20642 and friends.
-    const func = switch (sema.owner.unwrap()) {
-        .func => |func| func,
-        .cau => return, // does nothing outside a function
-    };
-    ip.funcSetCold(func, is_cold);
-    sema.allow_memoize = false;
-}
 fn zirDisableInstrumentation(sema: *Sema) CompileError!void {
     const pt = sema.pt;
     const zcu = pt.zcu;
 @@ -6891,13 +6890,20 @@ fn popErrorReturnTrace(
             @typeInfo(Air.Block).Struct.fields.len + 1); // +1 for the sole .cond_br instruction in the .block
         const cond_br_inst: Air.Inst.Index = @enumFromInt(sema.air_instructions.len);
-        try sema.air_instructions.append(gpa, .{ .tag = .cond_br, .data = .{ .pl_op = .{
+        try sema.air_instructions.append(gpa, .{
-            .operand = is_non_error_inst,
+            .tag = .cond_br,
-            .payload = sema.addExtraAssumeCapacity(Air.CondBr{
+            .data = .{
-                .then_body_len = @intCast(then_block.instructions.items.len),
+                .pl_op = .{
-                .else_body_len = @intCast(else_block.instructions.items.len),
+                    .operand = is_non_error_inst,
-            }),
+                    .payload = sema.addExtraAssumeCapacity(Air.CondBr{
-        } } });
+                        .then_body_len = @intCast(then_block.instructions.items.len),
+                        .else_body_len = @intCast(else_block.instructions.items.len),
+                        // weight against error branch
+                        .branch_hints = .{ .true = .likely, .false = .unlikely },
+                    }),
+                },
+            },
+        });
         sema.air_extra.appendSliceAssumeCapacity(@ptrCast(then_block.instructions.items));
         sema.air_extra.appendSliceAssumeCapacity(@ptrCast(else_block.instructions.items));
 @@ -10954,6 +10960,11 @@ const SwitchProngAnalysis = struct {
             sema.code.instructions.items(.data)[@intFromEnum(spa.switch_block_inst)].pl_node.src_node,
         );
+        // We can propagate `.cold` hints from this branch since it's comptime-known
+        // to be taken from the parent branch.
+        const parent_hint = sema.branch_hint;
+        defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null;
         if (has_tag_capture) {
             const tag_ref = try spa.analyzeTagCapture(child_block, capture_src, inline_case_capture);
             sema.inst_map.putAssumeCapacity(spa.tag_capture_inst, tag_ref);
 @@ -10990,6 +11001,7 @@ const SwitchProngAnalysis = struct {
     /// Analyze a switch prong which may have peers at runtime.
     /// Uses `analyzeBodyRuntimeBreak`. Sets up captures as needed.
+    /// Returns the `BranchHint` for the prong.
     fn analyzeProngRuntime(
         spa: SwitchProngAnalysis,
         case_block: *Block,
 @@ -11007,7 +11019,7 @@ const SwitchProngAnalysis = struct {
         /// Whether this prong has an inline tag capture. If `true`, then
         /// `inline_case_capture` cannot be `.none`.
         has_tag_capture: bool,
-    ) CompileError!void {
+    ) CompileError!std.builtin.BranchHint {
         const sema = spa.sema;
         if (has_tag_capture) {
 @@ -11033,7 +11045,7 @@ const SwitchProngAnalysis = struct {
                 if (sema.typeOf(capture_ref).isNoReturn(sema.pt.zcu)) {
                     // No need to analyze any further, the prong is unreachable
-                    return;
+                    return .none;
                 }
                 sema.inst_map.putAssumeCapacity(spa.switch_block_inst, capture_ref);
 @@ -11302,10 +11314,17 @@ const SwitchProngAnalysis = struct {
                 const prong_count = field_indices.len - in_mem_coercible.count();
-                const estimated_extra = prong_count * 6; // 2 for Case, 1 item, probably 3 insts
+                const estimated_extra = prong_count * 6 + (prong_count / 10); // 2 for Case, 1 item, probably 3 insts; plus hints
                 var cases_extra = try std.ArrayList(u32).initCapacity(sema.gpa, estimated_extra);
                 defer cases_extra.deinit();
+                {
+                    // All branch hints are `.none`, so just add zero elems.
+                    comptime assert(@intFromEnum(std.builtin.BranchHint.none) == 0);
+                    const need_elems = std.math.divCeil(usize, prong_count + 1, 10) catch unreachable;
+                    try cases_extra.appendNTimes(0, need_elems);
+                }
                 {
                     // Non-bitcast cases
                     var it = in_mem_coercible.iterator(.{ .kind = .unset });
 @@ -11728,7 +11747,7 @@ fn zirSwitchBlockErrUnion(sema: *Sema, block: *Block, inst: Zir.Inst.Index) Comp
     sub_block.need_debug_scope = null; // this body is emitted regardless
     defer sub_block.instructions.deinit(gpa);
-    try sema.analyzeBodyRuntimeBreak(&sub_block, non_error_case.body);
+    const non_error_hint = try sema.analyzeBodyRuntimeBreak(&sub_block, non_error_case.body);
     const true_instructions = try sub_block.instructions.toOwnedSlice(gpa);
     defer gpa.free(true_instructions);
 @@ -11782,6 +11801,7 @@ fn zirSwitchBlockErrUnion(sema: *Sema, block: *Block, inst: Zir.Inst.Index) Comp
             .payload = sema.addExtraAssumeCapacity(Air.CondBr{
                 .then_body_len = @intCast(true_instructions.len),
                 .else_body_len = @intCast(sub_block.instructions.items.len),
+                .branch_hints = .{ .true = non_error_hint, .false = .none },
             }),
         } },
     });
 @@ -12486,6 +12506,9 @@ fn analyzeSwitchRuntimeBlock(
     var cases_extra = try std.ArrayListUnmanaged(u32).initCapacity(gpa, estimated_cases_extra);
     defer cases_extra.deinit(gpa);
+    var branch_hints = try std.ArrayListUnmanaged(std.builtin.BranchHint).initCapacity(gpa, scalar_cases_len);
+    defer branch_hints.deinit(gpa);
     var case_block = child_block.makeSubBlock();
     case_block.runtime_loop = null;
     case_block.runtime_cond = operand_src;
 @@ -12516,10 +12539,13 @@ fn analyzeSwitchRuntimeBlock(
             break :blk field_ty.zigTypeTag(zcu) != .NoReturn;
         } else true;
-        if (err_set and try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) {
+        const prong_hint: std.builtin.BranchHint = if (err_set and
-            // nothing to do here
+            try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap))
-        } else if (analyze_body) {
+        h: {
-            try spa.analyzeProngRuntime(
+            // nothing to do here. weight against error branch
+            break :h .unlikely;
+        } else if (analyze_body) h: {
+            break :h try spa.analyzeProngRuntime(
                 &case_block,
                 .normal,
                 body,
 @@ -12532,10 +12558,12 @@ fn analyzeSwitchRuntimeBlock(
                 if (info.is_inline) item else .none,
                 info.has_tag_capture,
             );
-        } else {
+        } else h: {
             _ = try case_block.addNoOp(.unreach);
-        }
+            break :h .none;
+        };
+        try branch_hints.append(gpa, prong_hint);
         try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
         cases_extra.appendAssumeCapacity(1); // items_len
         cases_extra.appendAssumeCapacity(@intCast(case_block.instructions.items.len));
 @@ -12545,6 +12573,7 @@ fn analyzeSwitchRuntimeBlock(
     var is_first = true;
     var prev_cond_br: Air.Inst.Index = undefined;
+    var prev_hint: std.builtin.BranchHint = undefined;
     var first_else_body: []const Air.Inst.Index = &.{};
     defer gpa.free(first_else_body);
     var prev_then_body: []const Air.Inst.Index = &.{};
 @@ -12606,7 +12635,7 @@ fn analyzeSwitchRuntimeBlock(
                     } }));
                     emit_bb = true;
-                    try spa.analyzeProngRuntime(
+                    const prong_hint = try spa.analyzeProngRuntime(
                         &case_block,
                         .normal,
                         body,
 @@ -12619,6 +12648,7 @@ fn analyzeSwitchRuntimeBlock(
                         item_ref,
                         info.has_tag_capture,
                     );
+                    try branch_hints.append(gpa, prong_hint);
                     try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
                     cases_extra.appendAssumeCapacity(1); // items_len
 @@ -12649,8 +12679,8 @@ fn analyzeSwitchRuntimeBlock(
                 } }));
                 emit_bb = true;
-                if (analyze_body) {
+                const prong_hint: std.builtin.BranchHint = if (analyze_body) h: {
-                    try spa.analyzeProngRuntime(
+                    break :h try spa.analyzeProngRuntime(
                         &case_block,
                         .normal,
                         body,
 @@ -12663,9 +12693,11 @@ fn analyzeSwitchRuntimeBlock(
                         item,
                         info.has_tag_capture,
                     );
-                } else {
+                } else h: {
                     _ = try case_block.addNoOp(.unreach);
-                }
+                    break :h .none;
+                };
+                try branch_hints.append(gpa, prong_hint);
                 try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
                 cases_extra.appendAssumeCapacity(1); // items_len
 @@ -12697,10 +12729,13 @@ fn analyzeSwitchRuntimeBlock(
             const body = sema.code.bodySlice(extra_index, info.body_len);
             extra_index += info.body_len;
-            if (err_set and try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) {
+            const prong_hint: std.builtin.BranchHint = if (err_set and
-                // nothing to do here
+                try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap))
-            } else if (analyze_body) {
+            h: {
-                try spa.analyzeProngRuntime(
+                // nothing to do here. weight against error branch
+                break :h .unlikely;
+            } else if (analyze_body) h: {
+                break :h try spa.analyzeProngRuntime(
                     &case_block,
                     .normal,
                     body,
 @@ -12713,10 +12748,12 @@ fn analyzeSwitchRuntimeBlock(
                     .none,
                     false,
                 );
-            } else {
+            } else h: {
                 _ = try case_block.addNoOp(.unreach);
-            }
+                break :h .none;
+            };
+            try branch_hints.append(gpa, prong_hint);
             try cases_extra.ensureUnusedCapacity(gpa, 2 + items.len +
                 case_block.instructions.items.len);
 @@ -12784,23 +12821,24 @@ fn analyzeSwitchRuntimeBlock(
             const body = sema.code.bodySlice(extra_index, info.body_len);
             extra_index += info.body_len;
-            if (err_set and try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) {
+            const prong_hint: std.builtin.BranchHint = if (err_set and
-                // nothing to do here
+                try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap))
-            } else {
+            h: {
-                try spa.analyzeProngRuntime(
+                // nothing to do here. weight against error branch
-                    &case_block,
+                break :h .unlikely;
-                    .normal,
+            } else try spa.analyzeProngRuntime(
-                    body,
+                &case_block,
-                    info.capture,
+                .normal,
-                    child_block.src(.{ .switch_capture = .{
+                body,
-                        .switch_node_offset = switch_node_offset,
+                info.capture,
-                        .case_idx = .{ .kind = .multi, .index = @intCast(multi_i) },
+                child_block.src(.{ .switch_capture = .{
-                    } }),
+                    .switch_node_offset = switch_node_offset,
-                    items,
+                    .case_idx = .{ .kind = .multi, .index = @intCast(multi_i) },
-                    .none,
+                } }),
-                    false,
+                items,
-                );
+                .none,
-            }
+                false,
+            );
             if (is_first) {
                 is_first = false;
 @@ -12812,10 +12850,10 @@ fn analyzeSwitchRuntimeBlock(
                     @typeInfo(Air.CondBr).Struct.fields.len + prev_then_body.len + cond_body.len,
                 );
-                sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload =
+                sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload = sema.addExtraAssumeCapacity(Air.CondBr{
-                    sema.addExtraAssumeCapacity(Air.CondBr{
                     .then_body_len = @intCast(prev_then_body.len),
                     .else_body_len = @intCast(cond_body.len),
+                    .branch_hints = .{ .true = prev_hint, .false = .none },
                 });
                 sema.air_extra.appendSliceAssumeCapacity(@ptrCast(prev_then_body));
                 sema.air_extra.appendSliceAssumeCapacity(@ptrCast(cond_body));
 @@ -12823,6 +12861,7 @@ fn analyzeSwitchRuntimeBlock(
             gpa.free(prev_then_body);
             prev_then_body = try case_block.instructions.toOwnedSlice(gpa);
             prev_cond_br = new_cond_br;
+            prev_hint = prong_hint;
         }
     }
 @@ -12854,8 +12893,8 @@ fn analyzeSwitchRuntimeBlock(
                     if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src);
                     emit_bb = true;
-                    if (analyze_body) {
+                    const prong_hint: std.builtin.BranchHint = if (analyze_body) h: {
-                        try spa.analyzeProngRuntime(
+                        break :h try spa.analyzeProngRuntime(
                             &case_block,
                             .special,
                             special.body,
 @@ -12868,9 +12907,11 @@ fn analyzeSwitchRuntimeBlock(
                             item_ref,
                             special.has_tag_capture,
                         );
-                    } else {
+                    } else h: {
                         _ = try case_block.addNoOp(.unreach);
-                    }
+                        break :h .none;
+                    };
+                    try branch_hints.append(gpa, prong_hint);
                     try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
                     cases_extra.appendAssumeCapacity(1); // items_len
 @@ -12903,7 +12944,7 @@ fn analyzeSwitchRuntimeBlock(
                     if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src);
                     emit_bb = true;
-                    try spa.analyzeProngRuntime(
+                    const prong_hint = try spa.analyzeProngRuntime(
                         &case_block,
                         .special,
                         special.body,
 @@ -12916,6 +12957,7 @@ fn analyzeSwitchRuntimeBlock(
                         item_ref,
                         special.has_tag_capture,
                     );
+                    try branch_hints.append(gpa, prong_hint);
                     try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
                     cases_extra.appendAssumeCapacity(1); // items_len
 @@ -12937,7 +12979,7 @@ fn analyzeSwitchRuntimeBlock(
                     if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src);
                     emit_bb = true;
-                    try spa.analyzeProngRuntime(
+                    const prong_hint = try spa.analyzeProngRuntime(
                         &case_block,
                         .special,
                         special.body,
 @@ -12950,6 +12992,7 @@ fn analyzeSwitchRuntimeBlock(
                         item_ref,
                         special.has_tag_capture,
                     );
+                    try branch_hints.append(gpa, prong_hint);
                     try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
                     cases_extra.appendAssumeCapacity(1); // items_len
 @@ -12968,7 +13011,7 @@ fn analyzeSwitchRuntimeBlock(
                     if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src);
                     emit_bb = true;
-                    try spa.analyzeProngRuntime(
+                    const prong_hint = try spa.analyzeProngRuntime(
                         &case_block,
                         .special,
                         special.body,
 @@ -12981,6 +13024,7 @@ fn analyzeSwitchRuntimeBlock(
                         .bool_true,
                         special.has_tag_capture,
                     );
+                    try branch_hints.append(gpa, prong_hint);
                     try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
                     cases_extra.appendAssumeCapacity(1); // items_len
 @@ -12997,7 +13041,7 @@ fn analyzeSwitchRuntimeBlock(
                     if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src);
                     emit_bb = true;
-                    try spa.analyzeProngRuntime(
+                    const prong_hint = try spa.analyzeProngRuntime(
                         &case_block,
                         .special,
                         special.body,
 @@ -13010,6 +13054,7 @@ fn analyzeSwitchRuntimeBlock(
                         .bool_false,
                         special.has_tag_capture,
                     );
+                    try branch_hints.append(gpa, prong_hint);
                     try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
                     cases_extra.appendAssumeCapacity(1); // items_len
 @@ -13045,12 +13090,13 @@ fn analyzeSwitchRuntimeBlock(
             } else false
         else
             true;
-        if (special.body.len != 0 and err_set and
+        const else_hint: std.builtin.BranchHint = if (special.body.len != 0 and err_set and
             try sema.maybeErrorUnwrap(&case_block, special.body, operand, operand_src, allow_err_code_unwrap))
-        {
+        h: {
-            // nothing to do here
+            // nothing to do here. weight against error branch
-        } else if (special.body.len != 0 and analyze_body and !special.is_inline) {
+            break :h .unlikely;
-            try spa.analyzeProngRuntime(
+        } else if (special.body.len != 0 and analyze_body and !special.is_inline) h: {
+            break :h try spa.analyzeProngRuntime(
                 &case_block,
                 .special,
                 special.body,
 @@ -13063,7 +13109,7 @@ fn analyzeSwitchRuntimeBlock(
                 .none,
                 false,
             );
-        } else {
+        } else h: {
             // We still need a terminator in this block, but we have proven
             // that it is unreachable.
             if (case_block.wantSafety()) {
 @@ -13072,33 +13118,57 @@ fn analyzeSwitchRuntimeBlock(
             } else {
                 _ = try case_block.addNoOp(.unreach);
             }
-        }
+            // Safety check / unreachable branches are cold.
+            break :h .cold;
+        };
         if (is_first) {
+            try branch_hints.append(gpa, else_hint);
             final_else_body = case_block.instructions.items;
         } else {
+            try branch_hints.append(gpa, .none); // we have the range conditionals first
             try sema.air_extra.ensureUnusedCapacity(gpa, prev_then_body.len +
                 @typeInfo(Air.CondBr).Struct.fields.len + case_block.instructions.items.len);
-            sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload =
+            sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload = sema.addExtraAssumeCapacity(Air.CondBr{
-                sema.addExtraAssumeCapacity(Air.CondBr{
                 .then_body_len = @intCast(prev_then_body.len),
                 .else_body_len = @intCast(case_block.instructions.items.len),
+                .branch_hints = .{ .true = prev_hint, .false = else_hint },
             });
             sema.air_extra.appendSliceAssumeCapacity(@ptrCast(prev_then_body));
             sema.air_extra.appendSliceAssumeCapacity(@ptrCast(case_block.instructions.items));
             final_else_body = first_else_body;
         }
+    } else {
+        try branch_hints.append(gpa, .none);
     }
+    assert(branch_hints.items.len == cases_len + 1);
     try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.SwitchBr).Struct.fields.len +
-        cases_extra.items.len + final_else_body.len);
+        cases_extra.items.len + final_else_body.len +
+        (std.math.divCeil(usize, branch_hints.items.len, 10) catch unreachable)); // branch hints
     const payload_index = sema.addExtraAssumeCapacity(Air.SwitchBr{
         .cases_len = @intCast(cases_len),
         .else_body_len = @intCast(final_else_body.len),
     });
+    {
+        // Add branch hints.
+        var cur_bag: u32 = 0;
+        for (branch_hints.items, 0..) |hint, idx| {
+            const idx_in_bag = idx % 10;
+            cur_bag |= @as(u32, @intFromEnum(hint)) << @intCast(idx_in_bag * 3);
+            if (idx_in_bag == 9) {
+                sema.air_extra.appendAssumeCapacity(cur_bag);
+                cur_bag = 0;
+            }
+        }
+        if (branch_hints.items.len % 10 != 0) {
+            sema.air_extra.appendAssumeCapacity(cur_bag);
+        }
+    }
     sema.air_extra.appendSliceAssumeCapacity(@ptrCast(cases_extra.items));
     sema.air_extra.appendSliceAssumeCapacity(@ptrCast(final_else_body));
 @@ -19159,6 +19229,10 @@ fn zirBoolBr(
     const lhs_result: Air.Inst.Ref = if (is_bool_or) .bool_true else .bool_false;
     _ = try lhs_block.addBr(block_inst, lhs_result);
+    const parent_hint = sema.branch_hint;
+    defer sema.branch_hint = parent_hint;
+    sema.branch_hint = null;
     const rhs_result = try sema.resolveInlineBody(rhs_block, body, inst);
     const rhs_noret = sema.typeOf(rhs_result).isNoReturn(zcu);
     const coerced_rhs_result = if (!rhs_noret) rhs: {
 @@ -19167,7 +19241,17 @@ fn zirBoolBr(
         break :rhs coerced_result;
     } else rhs_result;
-    const result = sema.finishCondBr(parent_block, &child_block, &then_block, &else_block, lhs, block_inst);
+    const rhs_hint = sema.branch_hint orelse .none;
+    const result = try sema.finishCondBr(
+        parent_block,
+        &child_block,
+        &then_block,
+        &else_block,
+        lhs,
+        block_inst,
+        if (is_bool_or) .{ .true = .none, .false = rhs_hint } else .{ .true = rhs_hint, .false = .none },
+    );
     if (!rhs_noret) {
         if (try sema.resolveDefinedValue(rhs_block, rhs_src, coerced_rhs_result)) |rhs_val| {
             if (is_bool_or and rhs_val.toBool()) {
 @@ -19189,6 +19273,7 @@ fn finishCondBr(
     else_block: *Block,
     cond: Air.Inst.Ref,
     block_inst: Air.Inst.Index,
+    branch_hints: Air.CondBr.BranchHints,
 ) !Air.Inst.Ref {
     const gpa = sema.gpa;
 @@ -19199,6 +19284,7 @@ fn finishCondBr(
     const cond_br_payload = sema.addExtraAssumeCapacity(Air.CondBr{
         .then_body_len = @intCast(then_block.instructions.items.len),
         .else_body_len = @intCast(else_block.instructions.items.len),
+        .branch_hints = branch_hints,
     });
     sema.air_extra.appendSliceAssumeCapacity(@ptrCast(then_block.instructions.items));
     sema.air_extra.appendSliceAssumeCapacity(@ptrCast(else_block.instructions.items));
 @@ -19333,6 +19419,11 @@ fn zirCondbr(
     if (try sema.resolveDefinedValue(parent_block, cond_src, cond)) |cond_val| {
         const body = if (cond_val.toBool()) then_body else else_body;
+        // We can propagate `.cold` hints from this branch since it's comptime-known
+        // to be taken from the parent branch.
+        const parent_hint = sema.branch_hint;
+        defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null;
         try sema.maybeErrorUnwrapCondbr(parent_block, body, extra.data.condition, cond_src);
         // We use `analyzeBodyInner` since we want to propagate any comptime control flow to the caller.
         return sema.analyzeBodyInner(parent_block, body);
 @@ -19349,7 +19440,7 @@ fn zirCondbr(
     sub_block.need_debug_scope = null; // this body is emitted regardless
     defer sub_block.instructions.deinit(gpa);
-    try sema.analyzeBodyRuntimeBreak(&sub_block, then_body);
+    const true_hint = try sema.analyzeBodyRuntimeBreak(&sub_block, then_body);
     const true_instructions = try sub_block.instructions.toOwnedSlice(gpa);
     defer gpa.free(true_instructions);
 @@ -19365,11 +19456,13 @@ fn zirCondbr(
         break :blk try sub_block.addTyOp(.unwrap_errunion_err, result_ty, err_operand);
     };
-    if (err_cond != null and try sema.maybeErrorUnwrap(&sub_block, else_body, err_cond.?, cond_src, false)) {
+    const false_hint: std.builtin.BranchHint = if (err_cond != null and
-        // nothing to do
+        try sema.maybeErrorUnwrap(&sub_block, else_body, err_cond.?, cond_src, false))
-    } else {
+    h: {
-        try sema.analyzeBodyRuntimeBreak(&sub_block, else_body);
+        // nothing to do here. weight against error branch
-    }
+        break :h .unlikely;
+    } else try sema.analyzeBodyRuntimeBreak(&sub_block, else_body);
     try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.CondBr).Struct.fields.len +
         true_instructions.len + sub_block.instructions.items.len);
     _ = try parent_block.addInst(.{
 @@ -19379,6 +19472,7 @@ fn zirCondbr(
             .payload = sema.addExtraAssumeCapacity(Air.CondBr{
                 .then_body_len = @intCast(true_instructions.len),
                 .else_body_len = @intCast(sub_block.instructions.items.len),
+                .branch_hints = .{ .true = true_hint, .false = false_hint },
             }),
         } },
     });
 @@ -19403,6 +19497,11 @@ fn zirTry(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileError!
     }
     const is_non_err = try sema.analyzeIsNonErrComptimeOnly(parent_block, operand_src, err_union);
     if (is_non_err != .none) {
+        // We can propagate `.cold` hints from this branch since it's comptime-known
+        // to be taken from the parent branch.
+        const parent_hint = sema.branch_hint;
+        defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null;
         const is_non_err_val = (try sema.resolveDefinedValue(parent_block, operand_src, is_non_err)).?;
         if (is_non_err_val.toBool()) {
             return sema.analyzeErrUnionPayload(parent_block, src, err_union_ty, err_union, operand_src, false);
 @@ -19416,13 +19515,19 @@ fn zirTry(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileError!
     var sub_block = parent_block.makeSubBlock();
     defer sub_block.instructions.deinit(sema.gpa);
+    const parent_hint = sema.branch_hint;
+    defer sema.branch_hint = parent_hint;
     // This body is guaranteed to end with noreturn and has no breaks.
     try sema.analyzeBodyInner(&sub_block, body);
+    // The only interesting hint here is `.cold`, which can come from e.g. `errdefer @panic`.
+    const is_cold = sema.branch_hint == .cold;
     try sema.air_extra.ensureUnusedCapacity(sema.gpa, @typeInfo(Air.Try).Struct.fields.len +
         sub_block.instructions.items.len);
     const try_inst = try parent_block.addInst(.{
-        .tag = .@"try",
+        .tag = if (is_cold) .try_cold else .@"try",
         .data = .{ .pl_op = .{
             .operand = err_union,
             .payload = sema.addExtraAssumeCapacity(Air.Try{
 @@ -19452,6 +19557,11 @@ fn zirTryPtr(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileErr
     }
     const is_non_err = try sema.analyzeIsNonErrComptimeOnly(parent_block, operand_src, err_union);
     if (is_non_err != .none) {
+        // We can propagate `.cold` hints from this branch since it's comptime-known
+        // to be taken from the parent branch.
+        const parent_hint = sema.branch_hint;
+        defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null;
         const is_non_err_val = (try sema.resolveDefinedValue(parent_block, operand_src, is_non_err)).?;
         if (is_non_err_val.toBool()) {
             return sema.analyzeErrUnionPayloadPtr(parent_block, src, operand, false, false);
 @@ -19465,9 +19575,15 @@ fn zirTryPtr(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileErr
     var sub_block = parent_block.makeSubBlock();
     defer sub_block.instructions.deinit(sema.gpa);
+    const parent_hint = sema.branch_hint;
+    defer sema.branch_hint = parent_hint;
     // This body is guaranteed to end with noreturn and has no breaks.
     try sema.analyzeBodyInner(&sub_block, body);
+    // The only interesting hint here is `.cold`, which can come from e.g. `errdefer @panic`.
+    const is_cold = sema.branch_hint == .cold;
     const operand_ty = sema.typeOf(operand);
     const ptr_info = operand_ty.ptrInfo(zcu);
     const res_ty = try pt.ptrTypeSema(.{
 @@ -19483,7 +19599,7 @@ fn zirTryPtr(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileErr
     try sema.air_extra.ensureUnusedCapacity(sema.gpa, @typeInfo(Air.TryPtr).Struct.fields.len +
         sub_block.instructions.items.len);
     const try_inst = try parent_block.addInst(.{
-        .tag = .try_ptr,
+        .tag = if (is_cold) .try_ptr_cold else .try_ptr,
         .data = .{ .ty_pl = .{
             .ty = res_ty_ref,
             .payload = sema.addExtraAssumeCapacity(Air.TryPtr{
 @@ -19735,6 +19851,8 @@ fn retWithErrTracing(
     const cond_br_payload = sema.addExtraAssumeCapacity(Air.CondBr{
         .then_body_len = @intCast(then_block.instructions.items.len),
         .else_body_len = @intCast(else_block.instructions.items.len),
+        // weight against error branch
+        .branch_hints = .{ .true = .likely, .false = .unlikely },
     });
     sema.air_extra.appendSliceAssumeCapacity(@ptrCast(then_block.instructions.items));
     sema.air_extra.appendSliceAssumeCapacity(@ptrCast(else_block.instructions.items));
 @@ -26747,6 +26865,7 @@ fn zirBuiltinValue(sema: *Sema, extended: Zir.Inst.Extended.InstData) CompileErr
         .export_options => "ExportOptions",
         .extern_options => "ExternOptions",
         .type_info => "Type",
+        .branch_hint => "BranchHint",
         // Values are handled here.
         .calling_convention_c => {
 @@ -26772,6 +26891,27 @@ fn zirBuiltinValue(sema: *Sema, extended: Zir.Inst.Extended.InstData) CompileErr
     return Air.internedToRef(ty.toIntern());
 }
+fn zirBranchHint(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!void {
+    const pt = sema.pt;
+    const zcu = pt.zcu;
+    const extra = sema.code.extraData(Zir.Inst.UnNode, extended.operand).data;
+    const uncoerced_hint = try sema.resolveInst(extra.operand);
+    const operand_src = block.builtinCallArgSrc(extra.node, 0);
+    const hint_ty = try pt.getBuiltinType("BranchHint");
+    const coerced_hint = try sema.coerce(block, hint_ty, uncoerced_hint, operand_src);
+    const hint_val = try sema.resolveConstDefinedValue(block, operand_src, coerced_hint, .{
+        .needed_comptime_reason = "operand to '@branchHint' must be comptime-known",
+    });
+    // We only apply the first hint in a branch.
+    // This allows user-provided hints to override implicit cold hints.
+    if (sema.branch_hint == null) {
+        sema.branch_hint = zcu.toEnum(std.builtin.BranchHint, hint_val);
+    }
+}
 fn requireRuntimeBlock(sema: *Sema, block: *Block, src: LazySrcLoc, runtime_src: ?LazySrcLoc) !void {
     if (block.is_comptime) {
         const msg = msg: {
 @@ -27327,13 +27467,17 @@ fn addSafetyCheckExtra(
     sema.air_instructions.appendAssumeCapacity(.{
         .tag = .cond_br,
-        .data = .{ .pl_op = .{
+        .data = .{
-            .operand = ok,
+            .pl_op = .{
-            .payload = sema.addExtraAssumeCapacity(Air.CondBr{
+                .operand = ok,
-                .then_body_len = 1,
+                .payload = sema.addExtraAssumeCapacity(Air.CondBr{
-                .else_body_len = @intCast(fail_block.instructions.items.len),
+                    .then_body_len = 1,
-            }),
+                    .else_body_len = @intCast(fail_block.instructions.items.len),
-        } },
+                    // safety check failure branch is cold
+                    .branch_hints = .{ .true = .likely, .false = .cold },
+                }),
+            },
+        },
     });
     sema.air_extra.appendAssumeCapacity(@intFromEnum(br_inst));
     sema.air_extra.appendSliceAssumeCapacity(@ptrCast(fail_block.instructions.items));
 @@ -27530,6 +27674,7 @@ fn safetyCheckFormatted(
     try sema.addSafetyCheckExtra(parent_block, ok, &fail_block);
 }
+/// This does not set `sema.branch_hint`.
 fn safetyPanic(sema: *Sema, block: *Block, src: LazySrcLoc, panic_id: Zcu.PanicId) CompileError!void {
     const msg_nav_index = try sema.preparePanicId(block, src, panic_id);
     const msg_inst = try sema.analyzeNavVal(block, src, msg_nav_index);
 @@ -37179,7 +37324,7 @@ pub fn addExtraAssumeCapacity(sema: *Sema, extra: anytype) u32 {
     inline for (fields) |field| {
         sema.air_extra.appendAssumeCapacity(switch (field.type) {
             u32 => @field(extra, field.name),
-            i32 => @bitCast(@field(extra, field.name)),
+            i32, Air.CondBr.BranchHints => @bitCast(@field(extra, field.name)),
             Air.Inst.Ref, InternPool.Index => @intFromEnum(@field(extra, field.name)),
             else => @compileError("bad field type: " ++ @typeName(field.type)),
         });
 @@ -38247,6 +38392,12 @@ fn maybeDerefSliceAsArray(
 fn analyzeUnreachable(sema: *Sema, block: *Block, src: LazySrcLoc, safety_check: bool) !void {
     if (safety_check and block.wantSafety()) {
+        // We only apply the first hint in a branch.
+        // This allows user-provided hints to override implicit cold hints.
+        if (sema.branch_hint == null) {
+            sema.branch_hint = .cold;
+        }
         try sema.safetyPanic(block, src, .unreach);
     } else {
         _ = try block.addNoOp(.unreach);

src/Zcu/PerThread.zig+2

...	@@ -2188,6 +2188,8 @@ fn analyzeFnBody(pt: Zcu.PerThread, func_index: InternPool.Index) Zcu.SemaError!	...	@@ -2188,6 +2188,8 @@ fn analyzeFnBody(pt: Zcu.PerThread, func_index: InternPool.Index) Zcu.SemaError!
2188	});	2188	});
2189	}	2189	}
2190		2190
		2191	func.setBranchHint(ip, sema.branch_hint orelse .none);
		2192
2191	// If we don't get an error return trace from a caller, create our own.	2193	// If we don't get an error return trace from a caller, create our own.
2192	if (func.analysisUnordered(ip).calls_or_awaits_errorable_fn and	2194	if (func.analysisUnordered(ip).calls_or_awaits_errorable_fn and
2193	zcu.comp.config.any_error_tracing and	2195	zcu.comp.config.any_error_tracing and

src/arch/aarch64/CodeGen.zig+16-22

 @@ -795,7 +795,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .addrspace_cast  => return self.fail("TODO implement addrspace_cast", .{}),
             .@"try"          => try self.airTry(inst),
+            .try_cold        => try self.airTry(inst),
             .try_ptr         => try self.airTryPtr(inst),
+            .try_ptr_cold    => try self.airTryPtr(inst),
             .dbg_stmt         => try self.airDbgStmt(inst),
             .dbg_inline_block => try self.airDbgInlineBlock(inst),
 @@ -5092,25 +5094,17 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
 }
 fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
-    const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
+    const switch_br = self.air.unwrapSwitch(inst);
-    const condition_ty = self.typeOf(pl_op.operand);
+    const condition_ty = self.typeOf(switch_br.operand);
-    const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
     const liveness = try self.liveness.getSwitchBr(
         self.gpa,
         inst,
-        switch_br.data.cases_len + 1,
+        switch_br.cases_len + 1,
     );
     defer self.gpa.free(liveness.deaths);
-    var extra_index: usize = switch_br.end;
+    var it = switch_br.iterateCases();
-    var case_i: u32 = 0;
+    while (it.next()) |case| {
-    while (case_i < switch_br.data.cases_len) : (case_i += 1) {
-        const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
-        const items = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[case.end..][0..case.data.items_len]));
-        assert(items.len > 0);
-        const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
-        extra_index = case.end + items.len + case_body.len;
         // For every item, we compare it to condition and branch into
         // the prong if they are equal. After we compared to all
         // items, we branch into the next prong (or if no other prongs
 @@ -5126,11 +5120,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
         // prong:      ...
         //             ...
         // out:        ...
-        const branch_into_prong_relocs = try self.gpa.alloc(u32, items.len);
+        const branch_into_prong_relocs = try self.gpa.alloc(u32, case.items.len);
         defer self.gpa.free(branch_into_prong_relocs);
-        for (items, 0..) |item, idx| {
+        for (case.items, 0..) |item, idx| {
-            const cmp_result = try self.cmp(.{ .inst = pl_op.operand }, .{ .inst = item }, condition_ty, .neq);
+            const cmp_result = try self.cmp(.{ .inst = switch_br.operand }, .{ .inst = item }, condition_ty, .neq);
             branch_into_prong_relocs[idx] = try self.condBr(cmp_result);
         }
 @@ -5156,11 +5150,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
             _ = self.branch_stack.pop();
         }
-        try self.ensureProcessDeathCapacity(liveness.deaths[case_i].len);
+        try self.ensureProcessDeathCapacity(liveness.deaths[case.idx].len);
-        for (liveness.deaths[case_i]) |operand| {
+        for (liveness.deaths[case.idx]) |operand| {
             self.processDeath(operand);
         }
-        try self.genBody(case_body);
+        try self.genBody(case.body);
         // Revert to the previous register and stack allocation state.
         var saved_case_branch = self.branch_stack.pop();
 @@ -5178,8 +5172,8 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
         try self.performReloc(branch_away_from_prong_reloc);
     }
-    if (switch_br.data.else_body_len > 0) {
+    if (switch_br.else_body_len > 0) {
-        const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);
+        const else_body = it.elseBody();
         // Capture the state of register and stack allocation state so that we can revert to it.
         const parent_next_stack_offset = self.next_stack_offset;
 @@ -5218,7 +5212,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
         // in airCondBr.
     }
-    return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
+    return self.finishAir(inst, .unreach, .{ switch_br.operand, .none, .none });
 }
 fn performReloc(self: *Self, inst: Mir.Inst.Index) !void {

src/arch/arm/CodeGen.zig+16-22

 @@ -782,7 +782,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .addrspace_cast  => return self.fail("TODO implement addrspace_cast", .{}),
             .@"try"          => try self.airTry(inst),
+            .try_cold        => try self.airTry(inst),
             .try_ptr         => try self.airTryPtr(inst),
+            .try_ptr_cold    => try self.airTryPtr(inst),
             .dbg_stmt         => try self.airDbgStmt(inst),
             .dbg_inline_block => try self.airDbgInlineBlock(inst),
 @@ -5040,25 +5042,17 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
 }
 fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
-    const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
+    const switch_br = self.air.unwrapSwitch(inst);
-    const condition_ty = self.typeOf(pl_op.operand);
+    const condition_ty = self.typeOf(switch_br.operand);
-    const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
     const liveness = try self.liveness.getSwitchBr(
         self.gpa,
         inst,
-        switch_br.data.cases_len + 1,
+        switch_br.cases_len + 1,
     );
     defer self.gpa.free(liveness.deaths);
-    var extra_index: usize = switch_br.end;
+    var it = switch_br.iterateCases();
-    var case_i: u32 = 0;
+    while (it.next()) |case| {
-    while (case_i < switch_br.data.cases_len) : (case_i += 1) {
-        const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
-        const items: []const Air.Inst.Ref = @ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
-        assert(items.len > 0);
-        const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
-        extra_index = case.end + items.len + case_body.len;
         // For every item, we compare it to condition and branch into
         // the prong if they are equal. After we compared to all
         // items, we branch into the next prong (or if no other prongs
 @@ -5074,11 +5068,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
         // prong:      ...
         //             ...
         // out:        ...
-        const branch_into_prong_relocs = try self.gpa.alloc(u32, items.len);
+        const branch_into_prong_relocs = try self.gpa.alloc(u32, case.items.len);
         defer self.gpa.free(branch_into_prong_relocs);
-        for (items, 0..) |item, idx| {
+        for (case.items, 0..) |item, idx| {
-            const cmp_result = try self.cmp(.{ .inst = pl_op.operand }, .{ .inst = item }, condition_ty, .neq);
+            const cmp_result = try self.cmp(.{ .inst = switch_br.operand }, .{ .inst = item }, condition_ty, .neq);
             branch_into_prong_relocs[idx] = try self.condBr(cmp_result);
         }
 @@ -5104,11 +5098,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
             _ = self.branch_stack.pop();
         }
-        try self.ensureProcessDeathCapacity(liveness.deaths[case_i].len);
+        try self.ensureProcessDeathCapacity(liveness.deaths[case.idx].len);
-        for (liveness.deaths[case_i]) |operand| {
+        for (liveness.deaths[case.idx]) |operand| {
             self.processDeath(operand);
         }
-        try self.genBody(case_body);
+        try self.genBody(case.body);
         // Revert to the previous register and stack allocation state.
         var saved_case_branch = self.branch_stack.pop();
 @@ -5126,8 +5120,8 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
         try self.performReloc(branch_away_from_prong_reloc);
     }
-    if (switch_br.data.else_body_len > 0) {
+    if (switch_br.else_body_len > 0) {
-        const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);
+        const else_body = it.elseBody();
         // Capture the state of register and stack allocation state so that we can revert to it.
         const parent_next_stack_offset = self.next_stack_offset;
 @@ -5166,7 +5160,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
         // in airCondBr.
     }
-    return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
+    return self.finishAir(inst, .unreach, .{ switch_br.operand, .none, .none });
 }
 fn performReloc(self: *Self, inst: Mir.Inst.Index) !void {

src/arch/riscv64/CodeGen.zig+16-23

...	@@ -1640,7 +1640,9 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {	...	@@ -1640,7 +1640,9 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1640	.addrspace_cast => return func.fail("TODO: addrspace_cast", .{}),	1640	.addrspace_cast => return func.fail("TODO: addrspace_cast", .{}),
1641		1641
1642	.@"try" => try func.airTry(inst),	1642	.@"try" => try func.airTry(inst),
		1643	.try_cold => try func.airTry(inst),
1643	.try_ptr => return func.fail("TODO: try_ptr", .{}),	1644	.try_ptr => return func.fail("TODO: try_ptr", .{}),
		1645	.try_ptr_cold => return func.fail("TODO: try_ptr_cold", .{}),
1644		1646
1645	.dbg_var_ptr,	1647	.dbg_var_ptr,
1646	.dbg_var_val,	1648	.dbg_var_val,
...	@@ -5659,38 +5661,30 @@ fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !	...	@@ -5659,38 +5661,30 @@ fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
5659	}	5661	}
5660		5662
5661	fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {	5663	fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5662	const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	5664	const switch_br = func.air.unwrapSwitch(inst);
5663	const condition_ty = func.typeOf(pl_op.operand);	5665
5664	const switch_br = func.air.extraData(Air.SwitchBr, pl_op.payload);	5666	const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.cases_len + 1);
5665	var extra_index: usize = switch_br.end;
5666	var case_i: u32 = 0;
5667	const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.data.cases_len + 1);
5668	defer func.gpa.free(liveness.deaths);	5667	defer func.gpa.free(liveness.deaths);
5669		5668
5670	const condition = try func.resolveInst(pl_op.operand);	5669	const condition = try func.resolveInst(switch_br.operand);
		5670	const condition_ty = func.typeOf(switch_br.operand);
5671		5671
5672	// If the condition dies here in this switch instruction, process	5672	// If the condition dies here in this switch instruction, process
5673	// that death now instead of later as this has an effect on	5673	// that death now instead of later as this has an effect on
5674	// whether it needs to be spilled in the branches	5674	// whether it needs to be spilled in the branches
5675	if (func.liveness.operandDies(inst, 0)) {	5675	if (func.liveness.operandDies(inst, 0)) {
5676	if (pl_op.operand.toIndex()) \|op_inst\| try func.processDeath(op_inst);	5676	if (switch_br.operand.toIndex()) \|op_inst\| try func.processDeath(op_inst);
5677	}	5677	}
5678		5678
5679	func.scope_generation += 1;	5679	func.scope_generation += 1;
5680	const state = try func.saveState();	5680	const state = try func.saveState();
5681		5681
5682	while (case_i < switch_br.data.cases_len) : (case_i += 1) {	5682	var it = switch_br.iterateCases();
5683	const case = func.air.extraData(Air.SwitchBr.Case, extra_index);	5683	while (it.next()) \|case\| {
5684	const items: []const Air.Inst.Ref =	5684	var relocs = try func.gpa.alloc(Mir.Inst.Index, case.items.len);
5685	@ptrCast(func.air.extra[case.end..][0..case.data.items_len]);
5686	const case_body: []const Air.Inst.Index =
5687	@ptrCast(func.air.extra[case.end + items.len ..][0..case.data.body_len]);
5688	extra_index = case.end + items.len + case_body.len;
5689
5690	var relocs = try func.gpa.alloc(Mir.Inst.Index, items.len);
5691	defer func.gpa.free(relocs);	5685	defer func.gpa.free(relocs);
5692		5686
5693	for (items, relocs, 0..) \|item, *reloc, i\| {	5687	for (case.items, relocs, 0..) \|item, *reloc, i\| {
5694	const item_mcv = try func.resolveInst(item);	5688	const item_mcv = try func.resolveInst(item);
5695		5689
5696	const cond_lock = switch (condition) {	5690	const cond_lock = switch (condition) {
...	@@ -5724,10 +5718,10 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -5724,10 +5718,10 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5724	reloc.* = try func.condBr(condition_ty, .{ .register = cmp_reg });	5718	reloc.* = try func.condBr(condition_ty, .{ .register = cmp_reg });
5725	}	5719	}
5726		5720
5727	for (liveness.deaths[case_i]) \|operand\| try func.processDeath(operand);	5721	for (liveness.deaths[case.idx]) \|operand\| try func.processDeath(operand);
5728		5722
5729	for (relocs[0 .. relocs.len - 1]) \|reloc\| func.performReloc(reloc);	5723	for (relocs[0 .. relocs.len - 1]) \|reloc\| func.performReloc(reloc);
5730	try func.genBody(case_body);	5724	try func.genBody(case.body);
5731	try func.restoreState(state, &.{}, .{	5725	try func.restoreState(state, &.{}, .{
5732	.emit_instructions = false,	5726	.emit_instructions = false,
5733	.update_tracking = true,	5727	.update_tracking = true,
...	@@ -5738,9 +5732,8 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -5738,9 +5732,8 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5738	func.performReloc(relocs[relocs.len - 1]);	5732	func.performReloc(relocs[relocs.len - 1]);
5739	}	5733	}
5740		5734
5741	if (switch_br.data.else_body_len > 0) {	5735	if (switch_br.else_body_len > 0) {
5742	const else_body: []const Air.Inst.Index =	5736	const else_body = it.elseBody();
5743	@ptrCast(func.air.extra[extra_index..][0..switch_br.data.else_body_len]);
5744		5737
5745	const else_deaths = liveness.deaths.len - 1;	5738	const else_deaths = liveness.deaths.len - 1;
5746	for (liveness.deaths[else_deaths]) \|operand\| try func.processDeath(operand);	5739	for (liveness.deaths[else_deaths]) \|operand\| try func.processDeath(operand);

src/arch/sparc64/CodeGen.zig+2

 @@ -637,7 +637,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .addrspace_cast  => @panic("TODO try self.airAddrSpaceCast(int)"),
             .@"try"          => try self.airTry(inst),
+            .try_cold        => try self.airTry(inst),
             .try_ptr         => @panic("TODO try self.airTryPtr(inst)"),
+            .try_ptr_cold    => @panic("TODO try self.airTryPtrCold(inst)"),
             .dbg_stmt         => try self.airDbgStmt(inst),
             .dbg_inline_block => try self.airDbgInlineBlock(inst),

src/arch/wasm/CodeGen.zig+16-20

...	@@ -1913,7 +1913,9 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -1913,7 +1913,9 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1913	.get_union_tag => func.airGetUnionTag(inst),	1913	.get_union_tag => func.airGetUnionTag(inst),
1914		1914
1915	.@"try" => func.airTry(inst),	1915	.@"try" => func.airTry(inst),
		1916	.try_cold => func.airTry(inst),
1916	.try_ptr => func.airTryPtr(inst),	1917	.try_ptr => func.airTryPtr(inst),
		1918	.try_ptr_cold => func.airTryPtr(inst),
1917		1919
1918	.dbg_stmt => func.airDbgStmt(inst),	1920	.dbg_stmt => func.airDbgStmt(inst),
1919	.dbg_inline_block => func.airDbgInlineBlock(inst),	1921	.dbg_inline_block => func.airDbgInlineBlock(inst),
...	@@ -4041,37 +4043,31 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -4041,37 +4043,31 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4041	const zcu = pt.zcu;	4043	const zcu = pt.zcu;
4042	// result type is always 'noreturn'	4044	// result type is always 'noreturn'
4043	const blocktype = wasm.block_empty;	4045	const blocktype = wasm.block_empty;
4044	const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	4046	const switch_br = func.air.unwrapSwitch(inst);
4045	const target = try func.resolveInst(pl_op.operand);	4047	const target = try func.resolveInst(switch_br.operand);
4046	const target_ty = func.typeOf(pl_op.operand);	4048	const target_ty = func.typeOf(switch_br.operand);
4047	const switch_br = func.air.extraData(Air.SwitchBr, pl_op.payload);	4049	const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.cases_len + 1);
4048	const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.data.cases_len + 1);
4049	defer func.gpa.free(liveness.deaths);	4050	defer func.gpa.free(liveness.deaths);
4050		4051
4051	var extra_index: usize = switch_br.end;
4052	var case_i: u32 = 0;
4053
4054	// a list that maps each value with its value and body based on the order inside the list.	4052	// a list that maps each value with its value and body based on the order inside the list.
4055	const CaseValue = struct { integer: i32, value: Value };	4053	const CaseValue = struct { integer: i32, value: Value };
4056	var case_list = try std.ArrayList(struct {	4054	var case_list = try std.ArrayList(struct {
4057	values: []const CaseValue,	4055	values: []const CaseValue,
4058	body: []const Air.Inst.Index,	4056	body: []const Air.Inst.Index,
4059	}).initCapacity(func.gpa, switch_br.data.cases_len);	4057	}).initCapacity(func.gpa, switch_br.cases_len);
4060	defer for (case_list.items) \|case\| {	4058	defer for (case_list.items) \|case\| {
4061	func.gpa.free(case.values);	4059	func.gpa.free(case.values);
4062	} else case_list.deinit();	4060	} else case_list.deinit();
4063		4061
4064	var lowest_maybe: ?i32 = null;	4062	var lowest_maybe: ?i32 = null;
4065	var highest_maybe: ?i32 = null;	4063	var highest_maybe: ?i32 = null;
4066	while (case_i < switch_br.data.cases_len) : (case_i += 1) {	4064
4067	const case = func.air.extraData(Air.SwitchBr.Case, extra_index);	4065	var it = switch_br.iterateCases();
4068	const items: []const Air.Inst.Ref = @ptrCast(func.air.extra[case.end..][0..case.data.items_len]);	4066	while (it.next()) \|case\| {
4069	const case_body: []const Air.Inst.Index = @ptrCast(func.air.extra[case.end + items.len ..][0..case.data.body_len]);	4067	const values = try func.gpa.alloc(CaseValue, case.items.len);
4070	extra_index = case.end + items.len + case_body.len;
4071	const values = try func.gpa.alloc(CaseValue, items.len);
4072	errdefer func.gpa.free(values);	4068	errdefer func.gpa.free(values);
4073		4069
4074	for (items, 0..) \|ref, i\| {	4070	for (case.items, 0..) \|ref, i\| {
4075	const item_val = (try func.air.value(ref, pt)).?;	4071	const item_val = (try func.air.value(ref, pt)).?;
4076	const int_val = func.valueAsI32(item_val);	4072	const int_val = func.valueAsI32(item_val);
4077	if (lowest_maybe == null or int_val < lowest_maybe.?) {	4073	if (lowest_maybe == null or int_val < lowest_maybe.?) {
...	@@ -4083,7 +4079,7 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -4083,7 +4079,7 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4083	values[i] = .{ .integer = int_val, .value = item_val };	4079	values[i] = .{ .integer = int_val, .value = item_val };
4084	}	4080	}
4085		4081
4086	case_list.appendAssumeCapacity(.{ .values = values, .body = case_body });	4082	case_list.appendAssumeCapacity(.{ .values = values, .body = case.body });
4087	try func.startBlock(.block, blocktype);	4083	try func.startBlock(.block, blocktype);
4088	}	4084	}
4089		4085
...	@@ -4097,7 +4093,7 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -4097,7 +4093,7 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4097	// TODO: Benchmark this to find a proper value, LLVM seems to draw the line at '40~45'.	4093	// TODO: Benchmark this to find a proper value, LLVM seems to draw the line at '40~45'.
4098	const is_sparse = highest - lowest > 50 or target_ty.bitSize(zcu) > 32;	4094	const is_sparse = highest - lowest > 50 or target_ty.bitSize(zcu) > 32;
4099		4095
4100	const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra_index..][0..switch_br.data.else_body_len]);	4096	const else_body = it.elseBody();
4101	const has_else_body = else_body.len != 0;	4097	const has_else_body = else_body.len != 0;
4102	if (has_else_body) {	4098	if (has_else_body) {
4103	try func.startBlock(.block, blocktype);	4099	try func.startBlock(.block, blocktype);
...	@@ -4140,11 +4136,11 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -4140,11 +4136,11 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4140	// for errors that are not present in any branch. This is fine as this default	4136	// for errors that are not present in any branch. This is fine as this default
4141	// case will never be hit for those cases but we do save runtime cost and size	4137	// case will never be hit for those cases but we do save runtime cost and size
4142	// by using a jump table for this instead of if-else chains.	4138	// by using a jump table for this instead of if-else chains.
4143	break :blk if (has_else_body or target_ty.zigTypeTag(zcu) == .ErrorSet) case_i else unreachable;	4139	break :blk if (has_else_body or target_ty.zigTypeTag(zcu) == .ErrorSet) switch_br.cases_len else unreachable;
4144	};	4140	};
4145	func.mir_extra.appendAssumeCapacity(idx);	4141	func.mir_extra.appendAssumeCapacity(idx);
4146	} else if (has_else_body) {	4142	} else if (has_else_body) {
4147	func.mir_extra.appendAssumeCapacity(case_i); // default branch	4143	func.mir_extra.appendAssumeCapacity(switch_br.cases_len); // default branch
4148	}	4144	}
4149	try func.endBlock();	4145	try func.endBlock();
4150	}	4146	}

src/arch/x86_64/CodeGen.zig+15-23

 @@ -2262,7 +2262,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .addrspace_cast  => return self.fail("TODO implement addrspace_cast", .{}),
             .@"try"          => try self.airTry(inst),
+            .try_cold        => try self.airTry(inst), // TODO
             .try_ptr         => try self.airTryPtr(inst),
+            .try_ptr_cold    => try self.airTryPtr(inst), // TODO
             .dbg_stmt         => try self.airDbgStmt(inst),
             .dbg_inline_block => try self.airDbgInlineBlock(inst),
 @@ -13631,38 +13633,29 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
 }
 fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
-    const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
+    const switch_br = self.air.unwrapSwitch(inst);
-    const condition = try self.resolveInst(pl_op.operand);
+    const condition = try self.resolveInst(switch_br.operand);
-    const condition_ty = self.typeOf(pl_op.operand);
+    const condition_ty = self.typeOf(switch_br.operand);
-    const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
+    const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.cases_len + 1);
-    var extra_index: usize = switch_br.end;
-    var case_i: u32 = 0;
-    const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.data.cases_len + 1);
     defer self.gpa.free(liveness.deaths);
     // If the condition dies here in this switch instruction, process
     // that death now instead of later as this has an effect on
     // whether it needs to be spilled in the branches
     if (self.liveness.operandDies(inst, 0)) {
-        if (pl_op.operand.toIndex()) |op_inst| try self.processDeath(op_inst);
+        if (switch_br.operand.toIndex()) |op_inst| try self.processDeath(op_inst);
     }
     self.scope_generation += 1;
     const state = try self.saveState();
-    while (case_i < switch_br.data.cases_len) : (case_i += 1) {
+    var it = switch_br.iterateCases();
-        const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
+    while (it.next()) |case| {
-        const items: []const Air.Inst.Ref =
+        var relocs = try self.gpa.alloc(Mir.Inst.Index, case.items.len);
-            @ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
-        const case_body: []const Air.Inst.Index =
-            @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
-        extra_index = case.end + items.len + case_body.len;
-        var relocs = try self.gpa.alloc(Mir.Inst.Index, items.len);
         defer self.gpa.free(relocs);
         try self.spillEflagsIfOccupied();
-        for (items, relocs, 0..) |item, *reloc, i| {
+        for (case.items, relocs, 0..) |item, *reloc, i| {
             const item_mcv = try self.resolveInst(item);
             const cc: Condition = switch (condition) {
                 .eflags => |cc| switch (item_mcv.immediate) {
 @@ -13678,10 +13671,10 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
             reloc.* = try self.asmJccReloc(if (i < relocs.len - 1) cc else cc.negate(), undefined);
         }
-        for (liveness.deaths[case_i]) |operand| try self.processDeath(operand);
+        for (liveness.deaths[case.idx]) |operand| try self.processDeath(operand);
         for (relocs[0 .. relocs.len - 1]) |reloc| self.performReloc(reloc);
-        try self.genBody(case_body);
+        try self.genBody(case.body);
         try self.restoreState(state, &.{}, .{
             .emit_instructions = false,
             .update_tracking = true,
 @@ -13692,9 +13685,8 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
         self.performReloc(relocs[relocs.len - 1]);
     }
-    if (switch_br.data.else_body_len > 0) {
+    if (switch_br.else_body_len > 0) {
-        const else_body: []const Air.Inst.Index =
+        const else_body = it.elseBody();
-            @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);
         const else_deaths = liveness.deaths.len - 1;
         for (liveness.deaths[else_deaths]) |operand| try self.processDeath(operand);

src/codegen/c.zig+19-24

 @@ -1786,7 +1786,7 @@ pub const DeclGen = struct {
                 else => unreachable,
             }
         }
-        if (fn_val.getFunction(zcu)) |func| if (func.analysisUnordered(ip).is_cold)
+        if (fn_val.getFunction(zcu)) |func| if (func.analysisUnordered(ip).branch_hint == .cold)
             try w.writeAll("zig_cold ");
         if (fn_info.return_type == .noreturn_type) try w.writeAll("zig_noreturn ");
 @@ -3290,8 +3290,10 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
             .prefetch         => try airPrefetch(f, inst),
             .addrspace_cast   => return f.fail("TODO: C backend: implement addrspace_cast", .{}),
-            .@"try"  => try airTry(f, inst),
+            .@"try"       => try airTry(f, inst),
-            .try_ptr => try airTryPtr(f, inst),
+            .try_cold     => try airTry(f, inst),
+            .try_ptr      => try airTryPtr(f, inst),
+            .try_ptr_cold => try airTryPtr(f, inst),
             .dbg_stmt => try airDbgStmt(f, inst),
             .dbg_inline_block => try airDbgInlineBlock(f, inst),
 @@ -4988,11 +4990,10 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {
 fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
     const pt = f.object.dg.pt;
     const zcu = pt.zcu;
-    const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
+    const switch_br = f.air.unwrapSwitch(inst);
-    const condition = try f.resolveInst(pl_op.operand);
+    const condition = try f.resolveInst(switch_br.operand);
-    try reap(f, inst, &.{pl_op.operand});
+    try reap(f, inst, &.{switch_br.operand});
-    const condition_ty = f.typeOf(pl_op.operand);
+    const condition_ty = f.typeOf(switch_br.operand);
-    const switch_br = f.air.extraData(Air.SwitchBr, pl_op.payload);
     const writer = f.object.writer();
     try writer.writeAll("switch (");
 @@ -5013,22 +5014,16 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
     f.object.indent_writer.pushIndent();
     const gpa = f.object.dg.gpa;
-    const liveness = try f.liveness.getSwitchBr(gpa, inst, switch_br.data.cases_len + 1);
+    const liveness = try f.liveness.getSwitchBr(gpa, inst, switch_br.cases_len + 1);
     defer gpa.free(liveness.deaths);
     // On the final iteration we do not need to fix any state. This is because, like in the `else`
     // branch of a `cond_br`, our parent has to do it for this entire body anyway.
-    const last_case_i = switch_br.data.cases_len - @intFromBool(switch_br.data.else_body_len == 0);
+    const last_case_i = switch_br.cases_len - @intFromBool(switch_br.else_body_len == 0);
-    var extra_index: usize = switch_br.end;
-    for (0..switch_br.data.cases_len) |case_i| {
-        const case = f.air.extraData(Air.SwitchBr.Case, extra_index);
-        const items = @as([]const Air.Inst.Ref, @ptrCast(f.air.extra[case.end..][0..case.data.items_len]));
-        const case_body: []const Air.Inst.Index =
-            @ptrCast(f.air.extra[case.end + items.len ..][0..case.data.body_len]);
-        extra_index = case.end + case.data.items_len + case_body.len;
-        for (items) |item| {
+    var it = switch_br.iterateCases();
+    while (it.next()) |case| {
+        for (case.items) |item| {
             try f.object.indent_writer.insertNewline();
             try writer.writeAll("case ");
             const item_value = try f.air.value(item, pt);
 @@ -5046,19 +5041,19 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
         }
         try writer.writeByte(' ');
-        if (case_i != last_case_i) {
+        if (case.idx != last_case_i) {
-            try genBodyResolveState(f, inst, liveness.deaths[case_i], case_body, false);
+            try genBodyResolveState(f, inst, liveness.deaths[case.idx], case.body, false);
         } else {
-            for (liveness.deaths[case_i]) |death| {
+            for (liveness.deaths[case.idx]) |death| {
                 try die(f, inst, death.toRef());
             }
-            try genBody(f, case_body);
+            try genBody(f, case.body);
         }
         // The case body must be noreturn so we don't need to insert a break.
     }
-    const else_body: []const Air.Inst.Index = @ptrCast(f.air.extra[extra_index..][0..switch_br.data.else_body_len]);
+    const else_body = it.elseBody();
     try f.object.indent_writer.insertNewline();
     if (else_body.len > 0) {
         // Note that this must be the last case (i.e. the `last_case_i` case was not hit above)

src/codegen/llvm.zig+158-73

...	@@ -898,9 +898,9 @@ pub const Object = struct {	...	@@ -898,9 +898,9 @@ pub const Object = struct {
898	const i32_2 = try builder.intConst(.i32, 2);	898	const i32_2 = try builder.intConst(.i32, 2);
899	const i32_3 = try builder.intConst(.i32, 3);	899	const i32_3 = try builder.intConst(.i32, 3);
900	const debug_info_version = try builder.debugModuleFlag(	900	const debug_info_version = try builder.debugModuleFlag(
901	try builder.debugConstant(i32_2),	901	try builder.metadataConstant(i32_2),
902	try builder.metadataString("Debug Info Version"),	902	try builder.metadataString("Debug Info Version"),
903	try builder.debugConstant(i32_3),	903	try builder.metadataConstant(i32_3),
904	);	904	);
905		905
906	switch (comp.config.debug_format) {	906	switch (comp.config.debug_format) {
...	@@ -908,9 +908,9 @@ pub const Object = struct {	...	@@ -908,9 +908,9 @@ pub const Object = struct {
908	.dwarf => \|f\| {	908	.dwarf => \|f\| {
909	const i32_4 = try builder.intConst(.i32, 4);	909	const i32_4 = try builder.intConst(.i32, 4);
910	const dwarf_version = try builder.debugModuleFlag(	910	const dwarf_version = try builder.debugModuleFlag(
911	try builder.debugConstant(i32_2),	911	try builder.metadataConstant(i32_2),
912	try builder.metadataString("Dwarf Version"),	912	try builder.metadataString("Dwarf Version"),
913	try builder.debugConstant(i32_4),	913	try builder.metadataConstant(i32_4),
914	);	914	);
915	switch (f) {	915	switch (f) {
916	.@"32" => {	916	.@"32" => {
...	@@ -921,9 +921,9 @@ pub const Object = struct {	...	@@ -921,9 +921,9 @@ pub const Object = struct {
921	},	921	},
922	.@"64" => {	922	.@"64" => {
923	const dwarf64 = try builder.debugModuleFlag(	923	const dwarf64 = try builder.debugModuleFlag(
924	try builder.debugConstant(i32_2),	924	try builder.metadataConstant(i32_2),
925	try builder.metadataString("DWARF64"),	925	try builder.metadataString("DWARF64"),
926	try builder.debugConstant(.@"1"),	926	try builder.metadataConstant(.@"1"),
927	);	927	);
928	try builder.debugNamed(try builder.metadataString("llvm.module.flags"), &.{	928	try builder.debugNamed(try builder.metadataString("llvm.module.flags"), &.{
929	debug_info_version,	929	debug_info_version,
...	@@ -935,9 +935,9 @@ pub const Object = struct {	...	@@ -935,9 +935,9 @@ pub const Object = struct {
935	},	935	},
936	.code_view => {	936	.code_view => {
937	const code_view = try builder.debugModuleFlag(	937	const code_view = try builder.debugModuleFlag(
938	try builder.debugConstant(i32_2),	938	try builder.metadataConstant(i32_2),
939	try builder.metadataString("CodeView"),	939	try builder.metadataString("CodeView"),
940	try builder.debugConstant(.@"1"),	940	try builder.metadataConstant(.@"1"),
941	);	941	);
942	try builder.debugNamed(try builder.metadataString("llvm.module.flags"), &.{	942	try builder.debugNamed(try builder.metadataString("llvm.module.flags"), &.{
943	debug_info_version,	943	debug_info_version,
...	@@ -1122,12 +1122,12 @@ pub const Object = struct {	...	@@ -1122,12 +1122,12 @@ pub const Object = struct {
1122		1122
1123	self.builder.debugForwardReferenceSetType(	1123	self.builder.debugForwardReferenceSetType(
1124	self.debug_enums_fwd_ref,	1124	self.debug_enums_fwd_ref,
1125	try self.builder.debugTuple(self.debug_enums.items),	1125	try self.builder.metadataTuple(self.debug_enums.items),
1126	);	1126	);
1127		1127
1128	self.builder.debugForwardReferenceSetType(	1128	self.builder.debugForwardReferenceSetType(
1129	self.debug_globals_fwd_ref,	1129	self.debug_globals_fwd_ref,
1130	try self.builder.debugTuple(self.debug_globals.items),	1130	try self.builder.metadataTuple(self.debug_globals.items),
1131	);	1131	);
1132	}	1132	}
1133	}	1133	}
...	@@ -1369,7 +1369,7 @@ pub const Object = struct {	...	@@ -1369,7 +1369,7 @@ pub const Object = struct {
1369	_ = try attributes.removeFnAttr(.alignstack);	1369	_ = try attributes.removeFnAttr(.alignstack);
1370	}	1370	}
1371		1371
1372	if (func_analysis.is_cold) {	1372	if (func_analysis.branch_hint == .cold) {
1373	try attributes.addFnAttr(.cold, &o.builder);	1373	try attributes.addFnAttr(.cold, &o.builder);
1374	} else {	1374	} else {
1375	_ = try attributes.removeFnAttr(.cold);	1375	_ = try attributes.removeFnAttr(.cold);
...	@@ -1978,7 +1978,7 @@ pub const Object = struct {	...	@@ -1978,7 +1978,7 @@ pub const Object = struct {
1978	try o.lowerDebugType(int_ty),	1978	try o.lowerDebugType(int_ty),
1979	ty.abiSize(zcu) * 8,	1979	ty.abiSize(zcu) * 8,
1980	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,	1980	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
1981	try o.builder.debugTuple(enumerators),	1981	try o.builder.metadataTuple(enumerators),
1982	);	1982	);
1983		1983
1984	try o.debug_type_map.put(gpa, ty, debug_enum_type);	1984	try o.debug_type_map.put(gpa, ty, debug_enum_type);
...	@@ -2087,7 +2087,7 @@ pub const Object = struct {	...	@@ -2087,7 +2087,7 @@ pub const Object = struct {
2087	.none, // Underlying type	2087	.none, // Underlying type
2088	ty.abiSize(zcu) * 8,	2088	ty.abiSize(zcu) * 8,
2089	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,	2089	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2090	try o.builder.debugTuple(&.{	2090	try o.builder.metadataTuple(&.{
2091	debug_ptr_type,	2091	debug_ptr_type,
2092	debug_len_type,	2092	debug_len_type,
2093	}),	2093	}),
...	@@ -2167,10 +2167,10 @@ pub const Object = struct {	...	@@ -2167,10 +2167,10 @@ pub const Object = struct {
2167	try o.lowerDebugType(ty.childType(zcu)),	2167	try o.lowerDebugType(ty.childType(zcu)),
2168	ty.abiSize(zcu) * 8,	2168	ty.abiSize(zcu) * 8,
2169	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,	2169	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2170	try o.builder.debugTuple(&.{	2170	try o.builder.metadataTuple(&.{
2171	try o.builder.debugSubrange(	2171	try o.builder.debugSubrange(
2172	try o.builder.debugConstant(try o.builder.intConst(.i64, 0)),	2172	try o.builder.metadataConstant(try o.builder.intConst(.i64, 0)),
2173	try o.builder.debugConstant(try o.builder.intConst(.i64, ty.arrayLen(zcu))),	2173	try o.builder.metadataConstant(try o.builder.intConst(.i64, ty.arrayLen(zcu))),
2174	),	2174	),
2175	}),	2175	}),
2176	);	2176	);
...	@@ -2210,10 +2210,10 @@ pub const Object = struct {	...	@@ -2210,10 +2210,10 @@ pub const Object = struct {
2210	debug_elem_type,	2210	debug_elem_type,
2211	ty.abiSize(zcu) * 8,	2211	ty.abiSize(zcu) * 8,
2212	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,	2212	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2213	try o.builder.debugTuple(&.{	2213	try o.builder.metadataTuple(&.{
2214	try o.builder.debugSubrange(	2214	try o.builder.debugSubrange(
2215	try o.builder.debugConstant(try o.builder.intConst(.i64, 0)),	2215	try o.builder.metadataConstant(try o.builder.intConst(.i64, 0)),
2216	try o.builder.debugConstant(try o.builder.intConst(.i64, ty.vectorLen(zcu))),	2216	try o.builder.metadataConstant(try o.builder.intConst(.i64, ty.vectorLen(zcu))),
2217	),	2217	),
2218	}),	2218	}),
2219	);	2219	);
...	@@ -2288,7 +2288,7 @@ pub const Object = struct {	...	@@ -2288,7 +2288,7 @@ pub const Object = struct {
2288	.none, // Underlying type	2288	.none, // Underlying type
2289	ty.abiSize(zcu) * 8,	2289	ty.abiSize(zcu) * 8,
2290	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,	2290	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2291	try o.builder.debugTuple(&.{	2291	try o.builder.metadataTuple(&.{
2292	debug_data_type,	2292	debug_data_type,
2293	debug_some_type,	2293	debug_some_type,
2294	}),	2294	}),
...	@@ -2367,7 +2367,7 @@ pub const Object = struct {	...	@@ -2367,7 +2367,7 @@ pub const Object = struct {
2367	.none, // Underlying type	2367	.none, // Underlying type
2368	ty.abiSize(zcu) * 8,	2368	ty.abiSize(zcu) * 8,
2369	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,	2369	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2370	try o.builder.debugTuple(&fields),	2370	try o.builder.metadataTuple(&fields),
2371	);	2371	);
2372		2372
2373	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_error_union_type);	2373	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_error_union_type);
...	@@ -2447,7 +2447,7 @@ pub const Object = struct {	...	@@ -2447,7 +2447,7 @@ pub const Object = struct {
2447	.none, // Underlying type	2447	.none, // Underlying type
2448	ty.abiSize(zcu) * 8,	2448	ty.abiSize(zcu) * 8,
2449	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,	2449	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2450	try o.builder.debugTuple(fields.items),	2450	try o.builder.metadataTuple(fields.items),
2451	);	2451	);
2452		2452
2453	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_struct_type);	2453	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_struct_type);
...	@@ -2520,7 +2520,7 @@ pub const Object = struct {	...	@@ -2520,7 +2520,7 @@ pub const Object = struct {
2520	.none, // Underlying type	2520	.none, // Underlying type
2521	ty.abiSize(zcu) * 8,	2521	ty.abiSize(zcu) * 8,
2522	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,	2522	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2523	try o.builder.debugTuple(fields.items),	2523	try o.builder.metadataTuple(fields.items),
2524	);	2524	);
2525		2525
2526	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_struct_type);	2526	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_struct_type);
...	@@ -2561,7 +2561,7 @@ pub const Object = struct {	...	@@ -2561,7 +2561,7 @@ pub const Object = struct {
2561	.none, // Underlying type	2561	.none, // Underlying type
2562	ty.abiSize(zcu) * 8,	2562	ty.abiSize(zcu) * 8,
2563	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,	2563	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2564	try o.builder.debugTuple(	2564	try o.builder.metadataTuple(
2565	&.{try o.lowerDebugType(Type.fromInterned(union_type.enum_tag_ty))},	2565	&.{try o.lowerDebugType(Type.fromInterned(union_type.enum_tag_ty))},
2566	),	2566	),
2567	);	2567	);
...	@@ -2623,7 +2623,7 @@ pub const Object = struct {	...	@@ -2623,7 +2623,7 @@ pub const Object = struct {
2623	.none, // Underlying type	2623	.none, // Underlying type
2624	ty.abiSize(zcu) * 8,	2624	ty.abiSize(zcu) * 8,
2625	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,	2625	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2626	try o.builder.debugTuple(fields.items),	2626	try o.builder.metadataTuple(fields.items),
2627	);	2627	);
2628		2628
2629	o.builder.debugForwardReferenceSetType(debug_union_fwd_ref, debug_union_type);	2629	o.builder.debugForwardReferenceSetType(debug_union_fwd_ref, debug_union_type);
...	@@ -2682,7 +2682,7 @@ pub const Object = struct {	...	@@ -2682,7 +2682,7 @@ pub const Object = struct {
2682	.none, // Underlying type	2682	.none, // Underlying type
2683	ty.abiSize(zcu) * 8,	2683	ty.abiSize(zcu) * 8,
2684	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,	2684	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2685	try o.builder.debugTuple(&full_fields),	2685	try o.builder.metadataTuple(&full_fields),
2686	);	2686	);
2687		2687
2688	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_tagged_union_type);	2688	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_tagged_union_type);
...	@@ -2735,7 +2735,7 @@ pub const Object = struct {	...	@@ -2735,7 +2735,7 @@ pub const Object = struct {
2735	}	2735	}
2736		2736
2737	const debug_function_type = try o.builder.debugSubroutineType(	2737	const debug_function_type = try o.builder.debugSubroutineType(
2738	try o.builder.debugTuple(debug_param_types.items),	2738	try o.builder.metadataTuple(debug_param_types.items),
2739	);	2739	);
2740		2740
2741	try o.debug_type_map.put(gpa, ty, debug_function_type);	2741	try o.debug_type_map.put(gpa, ty, debug_function_type);
...	@@ -4571,7 +4571,7 @@ pub const Object = struct {	...	@@ -4571,7 +4571,7 @@ pub const Object = struct {
4571	const bad_value_block = try wip.block(1, "BadValue");	4571	const bad_value_block = try wip.block(1, "BadValue");
4572	const tag_int_value = wip.arg(0);	4572	const tag_int_value = wip.arg(0);
4573	var wip_switch =	4573	var wip_switch =
4574	try wip.@"switch"(tag_int_value, bad_value_block, @intCast(enum_type.names.len));	4574	try wip.@"switch"(tag_int_value, bad_value_block, @intCast(enum_type.names.len), .none);
4575	defer wip_switch.finish(&wip);	4575	defer wip_switch.finish(&wip);
4576		4576
4577	for (0..enum_type.names.len) \|field_index\| {	4577	for (0..enum_type.names.len) \|field_index\| {
...	@@ -4958,8 +4958,10 @@ pub const FuncGen = struct {	...	@@ -4958,8 +4958,10 @@ pub const FuncGen = struct {
4958	.ret_addr => try self.airRetAddr(inst),	4958	.ret_addr => try self.airRetAddr(inst),
4959	.frame_addr => try self.airFrameAddress(inst),	4959	.frame_addr => try self.airFrameAddress(inst),
4960	.cond_br => try self.airCondBr(inst),	4960	.cond_br => try self.airCondBr(inst),
4961	.@"try" => try self.airTry(body[i..]),	4961	.@"try" => try self.airTry(body[i..], false),
4962	.try_ptr => try self.airTryPtr(inst),	4962	.try_cold => try self.airTry(body[i..], true),
		4963	.try_ptr => try self.airTryPtr(inst, false),
		4964	.try_ptr_cold => try self.airTryPtr(inst, true),
4963	.intcast => try self.airIntCast(inst),	4965	.intcast => try self.airIntCast(inst),
4964	.trunc => try self.airTrunc(inst),	4966	.trunc => try self.airTrunc(inst),
4965	.fptrunc => try self.airFptrunc(inst),	4967	.fptrunc => try self.airFptrunc(inst),
...	@@ -5506,6 +5508,7 @@ pub const FuncGen = struct {	...	@@ -5506,6 +5508,7 @@ pub const FuncGen = struct {
5506	const panic_nav = ip.getNav(panic_func.owner_nav);	5508	const panic_nav = ip.getNav(panic_func.owner_nav);
5507	const fn_info = zcu.typeToFunc(Type.fromInterned(panic_nav.typeOf(ip))).?;	5509	const fn_info = zcu.typeToFunc(Type.fromInterned(panic_nav.typeOf(ip))).?;
5508	const panic_global = try o.resolveLlvmFunction(panic_func.owner_nav);	5510	const panic_global = try o.resolveLlvmFunction(panic_func.owner_nav);
		5511	_ = try fg.wip.callIntrinsicAssumeCold();
5509	_ = try fg.wip.call(	5512	_ = try fg.wip.call(
5510	.normal,	5513	.normal,
5511	toLlvmCallConv(fn_info.cc, target),	5514	toLlvmCallConv(fn_info.cc, target),
...	@@ -5794,7 +5797,7 @@ pub const FuncGen = struct {	...	@@ -5794,7 +5797,7 @@ pub const FuncGen = struct {
5794	const mixed_block = try self.wip.block(1, "Mixed");	5797	const mixed_block = try self.wip.block(1, "Mixed");
5795	const both_pl_block = try self.wip.block(1, "BothNonNull");	5798	const both_pl_block = try self.wip.block(1, "BothNonNull");
5796	const end_block = try self.wip.block(3, "End");	5799	const end_block = try self.wip.block(3, "End");
5797	var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2);	5800	var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none);
5798	defer wip_switch.finish(&self.wip);	5801	defer wip_switch.finish(&self.wip);
5799	try wip_switch.addCase(	5802	try wip_switch.addCase(
5800	try o.builder.intConst(llvm_i2, 0b00),	5803	try o.builder.intConst(llvm_i2, 0b00),
...	@@ -5948,21 +5951,62 @@ pub const FuncGen = struct {	...	@@ -5948,21 +5951,62 @@ pub const FuncGen = struct {
5948	const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);	5951	const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);
5949	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);	5952	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
5950		5953
		5954	const Hint = enum {
		5955	none,
		5956	unpredictable,
		5957	then_likely,
		5958	else_likely,
		5959	then_cold,
		5960	else_cold,
		5961	};
		5962	const hint: Hint = switch (extra.data.branch_hints.true) {
		5963	.none => switch (extra.data.branch_hints.false) {
		5964	.none => .none,
		5965	.likely => .else_likely,
		5966	.unlikely => .then_likely,
		5967	.cold => .else_cold,
		5968	.unpredictable => .unpredictable,
		5969	},
		5970	.likely => switch (extra.data.branch_hints.false) {
		5971	.none => .then_likely,
		5972	.likely => .unpredictable,
		5973	.unlikely => .then_likely,
		5974	.cold => .else_cold,
		5975	.unpredictable => .unpredictable,
		5976	},
		5977	.unlikely => switch (extra.data.branch_hints.false) {
		5978	.none => .else_likely,
		5979	.likely => .else_likely,
		5980	.unlikely => .unpredictable,
		5981	.cold => .else_cold,
		5982	.unpredictable => .unpredictable,
		5983	},
		5984	.cold => .then_cold,
		5985	.unpredictable => .unpredictable,
		5986	};
		5987
5951	const then_block = try self.wip.block(1, "Then");	5988	const then_block = try self.wip.block(1, "Then");
5952	const else_block = try self.wip.block(1, "Else");	5989	const else_block = try self.wip.block(1, "Else");
5953	_ = try self.wip.brCond(cond, then_block, else_block);	5990	_ = try self.wip.brCond(cond, then_block, else_block, switch (hint) {
		5991	.none, .then_cold, .else_cold => .none,
		5992	.unpredictable => .unpredictable,
		5993	.then_likely => .then_likely,
		5994	.else_likely => .else_likely,
		5995	});
5954		5996
5955	self.wip.cursor = .{ .block = then_block };	5997	self.wip.cursor = .{ .block = then_block };
		5998	if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold();
5956	try self.genBodyDebugScope(null, then_body);	5999	try self.genBodyDebugScope(null, then_body);
5957		6000
5958	self.wip.cursor = .{ .block = else_block };	6001	self.wip.cursor = .{ .block = else_block };
		6002	if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold();
5959	try self.genBodyDebugScope(null, else_body);	6003	try self.genBodyDebugScope(null, else_body);
5960		6004
5961	// No need to reset the insert cursor since this instruction is noreturn.	6005	// No need to reset the insert cursor since this instruction is noreturn.
5962	return .none;	6006	return .none;
5963	}	6007	}
5964		6008
5965	fn airTry(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {	6009	fn airTry(self: *FuncGen, body_tail: []const Air.Inst.Index, err_cold: bool) !Builder.Value {
5966	const o = self.ng.object;	6010	const o = self.ng.object;
5967	const pt = o.pt;	6011	const pt = o.pt;
5968	const zcu = pt.zcu;	6012	const zcu = pt.zcu;
...	@@ -5975,10 +6019,10 @@ pub const FuncGen = struct {	...	@@ -5975,10 +6019,10 @@ pub const FuncGen = struct {
5975	const payload_ty = self.typeOfIndex(inst);	6019	const payload_ty = self.typeOfIndex(inst);
5976	const can_elide_load = if (isByRef(payload_ty, zcu)) self.canElideLoad(body_tail) else false;	6020	const can_elide_load = if (isByRef(payload_ty, zcu)) self.canElideLoad(body_tail) else false;
5977	const is_unused = self.liveness.isUnused(inst);	6021	const is_unused = self.liveness.isUnused(inst);
5978	return lowerTry(self, err_union, body, err_union_ty, false, can_elide_load, is_unused);	6022	return lowerTry(self, err_union, body, err_union_ty, false, can_elide_load, is_unused, err_cold);
5979	}	6023	}
5980		6024
5981	fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {	6025	fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
5982	const o = self.ng.object;	6026	const o = self.ng.object;
5983	const zcu = o.pt.zcu;	6027	const zcu = o.pt.zcu;
5984	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	6028	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...	@@ -5987,7 +6031,7 @@ pub const FuncGen = struct {	...	@@ -5987,7 +6031,7 @@ pub const FuncGen = struct {
5987	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);	6031	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
5988	const err_union_ty = self.typeOf(extra.data.ptr).childType(zcu);	6032	const err_union_ty = self.typeOf(extra.data.ptr).childType(zcu);
5989	const is_unused = self.liveness.isUnused(inst);	6033	const is_unused = self.liveness.isUnused(inst);
5990	return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, is_unused);	6034	return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, is_unused, err_cold);
5991	}	6035	}
5992		6036
5993	fn lowerTry(	6037	fn lowerTry(
...	@@ -5998,6 +6042,7 @@ pub const FuncGen = struct {	...	@@ -5998,6 +6042,7 @@ pub const FuncGen = struct {
5998	operand_is_ptr: bool,	6042	operand_is_ptr: bool,
5999	can_elide_load: bool,	6043	can_elide_load: bool,
6000	is_unused: bool,	6044	is_unused: bool,
		6045	err_cold: bool,
6001	) !Builder.Value {	6046	) !Builder.Value {
6002	const o = fg.ng.object;	6047	const o = fg.ng.object;
6003	const pt = o.pt;	6048	const pt = o.pt;
...	@@ -6036,9 +6081,10 @@ pub const FuncGen = struct {	...	@@ -6036,9 +6081,10 @@ pub const FuncGen = struct {
6036		6081
6037	const return_block = try fg.wip.block(1, "TryRet");	6082	const return_block = try fg.wip.block(1, "TryRet");
6038	const continue_block = try fg.wip.block(1, "TryCont");	6083	const continue_block = try fg.wip.block(1, "TryCont");
6039	_ = try fg.wip.brCond(is_err, return_block, continue_block);	6084	_ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely);
6040		6085
6041	fg.wip.cursor = .{ .block = return_block };	6086	fg.wip.cursor = .{ .block = return_block };
		6087	if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold();
6042	try fg.genBodyDebugScope(null, body);	6088	try fg.genBodyDebugScope(null, body);
6043		6089
6044	fg.wip.cursor = .{ .block = continue_block };	6090	fg.wip.cursor = .{ .block = continue_block };
...	@@ -6065,9 +6111,11 @@ pub const FuncGen = struct {	...	@@ -6065,9 +6111,11 @@ pub const FuncGen = struct {
6065		6111
6066	fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {	6112	fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6067	const o = self.ng.object;	6113	const o = self.ng.object;
6068	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	6114
6069	const cond = try self.resolveInst(pl_op.operand);	6115	const switch_br = self.air.unwrapSwitch(inst);
6070	const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);	6116
		6117	const cond = try self.resolveInst(switch_br.operand);
		6118
6071	const else_block = try self.wip.block(1, "Default");	6119	const else_block = try self.wip.block(1, "Default");
6072	const llvm_usize = try o.lowerType(Type.usize);	6120	const llvm_usize = try o.lowerType(Type.usize);
6073	const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder))	6121	const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder))
...	@@ -6075,34 +6123,70 @@ pub const FuncGen = struct {	...	@@ -6075,34 +6123,70 @@ pub const FuncGen = struct {
6075	else	6123	else
6076	cond;	6124	cond;
6077		6125
6078	var extra_index: usize = switch_br.end;	6126	const llvm_cases_len = llvm_cases_len: {
6079	var case_i: u32 = 0;	6127	var len: u32 = 0;
6080	var llvm_cases_len: u32 = 0;	6128	var it = switch_br.iterateCases();
6081	while (case_i < switch_br.data.cases_len) : (case_i += 1) {	6129	while (it.next()) \|case\| len += @intCast(case.items.len);
6082	const case = self.air.extraData(Air.SwitchBr.Case, extra_index);	6130	break :llvm_cases_len len;
6083	const items: []const Air.Inst.Ref =	6131	};
6084	@ptrCast(self.air.extra[case.end..][0..case.data.items_len]);	6132
6085	const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];	6133	const weights: Builder.Function.Instruction.BrCond.Weights = weights: {
6086	extra_index = case.end + case.data.items_len + case_body.len;	6134	// First pass. If any weights are `.unpredictable`, unpredictable.
		6135	// If all are `.none` or `.cold`, none.
		6136	var any_likely = false;
		6137	for (0..switch_br.cases_len) \|case_idx\| {
		6138	switch (switch_br.getHint(@intCast(case_idx))) {
		6139	.none, .cold => {},
		6140	.likely, .unlikely => any_likely = true,
		6141	.unpredictable => break :weights .unpredictable,
		6142	}
		6143	}
		6144	switch (switch_br.getElseHint()) {
		6145	.none, .cold => {},
		6146	.likely, .unlikely => any_likely = true,
		6147	.unpredictable => break :weights .unpredictable,
		6148	}
		6149	if (!any_likely) break :weights .none;
6087		6150
6088	llvm_cases_len += @intCast(items.len);	6151	var weights = try self.gpa.alloc(Builder.Metadata, llvm_cases_len + 1);
6089	}	6152	defer self.gpa.free(weights);
6090		6153
6091	var wip_switch = try self.wip.@"switch"(cond_int, else_block, llvm_cases_len);	6154	const else_weight: u32 = switch (switch_br.getElseHint()) {
6092	defer wip_switch.finish(&self.wip);	6155	.unpredictable => unreachable,
		6156	.none, .cold => 1000,
		6157	.likely => 2000,
		6158	.unlikely => 1,
		6159	};
		6160	weights[0] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight));
		6161
		6162	var weight_idx: usize = 1;
		6163	var it = switch_br.iterateCases();
		6164	while (it.next()) \|case\| {
		6165	const weight_val: u32 = switch (switch_br.getHint(case.idx)) {
		6166	.unpredictable => unreachable,
		6167	.none, .cold => 1000,
		6168	.likely => 2000,
		6169	.unlikely => 1,
		6170	};
		6171	const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val));
		6172	@memset(weights[weight_idx..][0..case.items.len], weight_meta);
		6173	weight_idx += case.items.len;
		6174	}
		6175
		6176	assert(weight_idx == weights.len);
6093		6177
6094	extra_index = switch_br.end;	6178	const branch_weights_str = try o.builder.metadataString("branch_weights");
6095	case_i = 0;	6179	const tuple = try o.builder.strTuple(branch_weights_str, weights);
6096	while (case_i < switch_br.data.cases_len) : (case_i += 1) {	6180	break :weights @enumFromInt(@intFromEnum(tuple));
6097	const case = self.air.extraData(Air.SwitchBr.Case, extra_index);	6181	};
6098	const items: []const Air.Inst.Ref =
6099	@ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
6100	const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
6101	extra_index = case.end + case.data.items_len + case_body.len;
6102		6182
6103	const case_block = try self.wip.block(@intCast(items.len), "Case");	6183	var wip_switch = try self.wip.@"switch"(cond_int, else_block, llvm_cases_len, weights);
		6184	defer wip_switch.finish(&self.wip);
6104		6185
6105	for (items) \|item\| {	6186	var it = switch_br.iterateCases();
		6187	while (it.next()) \|case\| {
		6188	const case_block = try self.wip.block(@intCast(case.items.len), "Case");
		6189	for (case.items) \|item\| {
6106	const llvm_item = (try self.resolveInst(item)).toConst().?;	6190	const llvm_item = (try self.resolveInst(item)).toConst().?;
6107	const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder))	6191	const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder))
6108	try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize)	6192	try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize)
...	@@ -6110,13 +6194,14 @@ pub const FuncGen = struct {	...	@@ -6110,13 +6194,14 @@ pub const FuncGen = struct {
6110	llvm_item;	6194	llvm_item;
6111	try wip_switch.addCase(llvm_int_item, case_block, &self.wip);	6195	try wip_switch.addCase(llvm_int_item, case_block, &self.wip);
6112	}	6196	}
6113
6114	self.wip.cursor = .{ .block = case_block };	6197	self.wip.cursor = .{ .block = case_block };
6115	try self.genBodyDebugScope(null, case_body);	6198	if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold();
		6199	try self.genBodyDebugScope(null, case.body);
6116	}	6200	}
6117		6201
		6202	const else_body = it.elseBody();
6118	self.wip.cursor = .{ .block = else_block };	6203	self.wip.cursor = .{ .block = else_block };
6119	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);	6204	if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold();
6120	if (else_body.len != 0) {	6205	if (else_body.len != 0) {
6121	try self.genBodyDebugScope(null, else_body);	6206	try self.genBodyDebugScope(null, else_body);
6122	} else {	6207	} else {
...	@@ -7748,7 +7833,7 @@ pub const FuncGen = struct {	...	@@ -7748,7 +7833,7 @@ pub const FuncGen = struct {
7748		7833
7749	const fail_block = try fg.wip.block(1, "OverflowFail");	7834	const fail_block = try fg.wip.block(1, "OverflowFail");
7750	const ok_block = try fg.wip.block(1, "OverflowOk");	7835	const ok_block = try fg.wip.block(1, "OverflowOk");
7751	_ = try fg.wip.brCond(overflow_bit, fail_block, ok_block);	7836	_ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none);
7752		7837
7753	fg.wip.cursor = .{ .block = fail_block };	7838	fg.wip.cursor = .{ .block = fail_block };
7754	try fg.buildSimplePanic(.integer_overflow);	7839	try fg.buildSimplePanic(.integer_overflow);
...	@@ -9389,7 +9474,7 @@ pub const FuncGen = struct {	...	@@ -9389,7 +9474,7 @@ pub const FuncGen = struct {
9389	self.wip.cursor = .{ .block = loop_block };	9474	self.wip.cursor = .{ .block = loop_block };
9390	const it_ptr = try self.wip.phi(.ptr, "");	9475	const it_ptr = try self.wip.phi(.ptr, "");
9391	const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, "");	9476	const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, "");
9392	_ = try self.wip.brCond(end, body_block, end_block);	9477	_ = try self.wip.brCond(end, body_block, end_block, .none);
9393		9478
9394	self.wip.cursor = .{ .block = body_block };	9479	self.wip.cursor = .{ .block = body_block };
9395	const elem_abi_align = elem_ty.abiAlignment(zcu);	9480	const elem_abi_align = elem_ty.abiAlignment(zcu);
...	@@ -9427,7 +9512,7 @@ pub const FuncGen = struct {	...	@@ -9427,7 +9512,7 @@ pub const FuncGen = struct {
9427	const cond = try self.cmp(.normal, .neq, Type.usize, len, usize_zero);	9512	const cond = try self.cmp(.normal, .neq, Type.usize, len, usize_zero);
9428	const memset_block = try self.wip.block(1, "MemsetTrapSkip");	9513	const memset_block = try self.wip.block(1, "MemsetTrapSkip");
9429	const end_block = try self.wip.block(2, "MemsetTrapEnd");	9514	const end_block = try self.wip.block(2, "MemsetTrapEnd");
9430	_ = try self.wip.brCond(cond, memset_block, end_block);	9515	_ = try self.wip.brCond(cond, memset_block, end_block, .none);
9431	self.wip.cursor = .{ .block = memset_block };	9516	self.wip.cursor = .{ .block = memset_block };
9432	_ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind);	9517	_ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind);
9433	_ = try self.wip.br(end_block);	9518	_ = try self.wip.br(end_block);
...	@@ -9462,7 +9547,7 @@ pub const FuncGen = struct {	...	@@ -9462,7 +9547,7 @@ pub const FuncGen = struct {
9462	const cond = try self.cmp(.normal, .neq, Type.usize, len, usize_zero);	9547	const cond = try self.cmp(.normal, .neq, Type.usize, len, usize_zero);
9463	const memcpy_block = try self.wip.block(1, "MemcpyTrapSkip");	9548	const memcpy_block = try self.wip.block(1, "MemcpyTrapSkip");
9464	const end_block = try self.wip.block(2, "MemcpyTrapEnd");	9549	const end_block = try self.wip.block(2, "MemcpyTrapEnd");
9465	_ = try self.wip.brCond(cond, memcpy_block, end_block);	9550	_ = try self.wip.brCond(cond, memcpy_block, end_block, .none);
9466	self.wip.cursor = .{ .block = memcpy_block };	9551	self.wip.cursor = .{ .block = memcpy_block };
9467	_ = try self.wip.callMemCpy(	9552	_ = try self.wip.callMemCpy(
9468	dest_ptr,	9553	dest_ptr,
...	@@ -9632,7 +9717,7 @@ pub const FuncGen = struct {	...	@@ -9632,7 +9717,7 @@ pub const FuncGen = struct {
9632	const valid_block = try self.wip.block(@intCast(names.len), "Valid");	9717	const valid_block = try self.wip.block(@intCast(names.len), "Valid");
9633	const invalid_block = try self.wip.block(1, "Invalid");	9718	const invalid_block = try self.wip.block(1, "Invalid");
9634	const end_block = try self.wip.block(2, "End");	9719	const end_block = try self.wip.block(2, "End");
9635	var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len));	9720	var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none);
9636	defer wip_switch.finish(&self.wip);	9721	defer wip_switch.finish(&self.wip);
9637		9722
9638	for (0..names.len) \|name_index\| {	9723	for (0..names.len) \|name_index\| {
...	@@ -9708,7 +9793,7 @@ pub const FuncGen = struct {	...	@@ -9708,7 +9793,7 @@ pub const FuncGen = struct {
9708	const named_block = try wip.block(@intCast(enum_type.names.len), "Named");	9793	const named_block = try wip.block(@intCast(enum_type.names.len), "Named");
9709	const unnamed_block = try wip.block(1, "Unnamed");	9794	const unnamed_block = try wip.block(1, "Unnamed");
9710	const tag_int_value = wip.arg(0);	9795	const tag_int_value = wip.arg(0);
9711	var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(enum_type.names.len));	9796	var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(enum_type.names.len), .none);
9712	defer wip_switch.finish(&wip);	9797	defer wip_switch.finish(&wip);
9713		9798
9714	for (0..enum_type.names.len) \|field_index\| {	9799	for (0..enum_type.names.len) \|field_index\| {
...	@@ -9858,7 +9943,7 @@ pub const FuncGen = struct {	...	@@ -9858,7 +9943,7 @@ pub const FuncGen = struct {
9858	const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");	9943	const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");
9859	const loop_then = try self.wip.block(1, "ReduceLoopThen");	9944	const loop_then = try self.wip.block(1, "ReduceLoopThen");
9860		9945
9861	_ = try self.wip.brCond(cond, loop_then, loop_exit);	9946	_ = try self.wip.brCond(cond, loop_then, loop_exit, .none);
9862		9947
9863	{	9948	{
9864	self.wip.cursor = .{ .block = loop_then };	9949	self.wip.cursor = .{ .block = loop_then };

src/codegen/llvm/Builder.zig+250-36

...	@@ -4817,12 +4817,22 @@ pub const Function = struct {	...	@@ -4817,12 +4817,22 @@ pub const Function = struct {
4817	cond: Value,	4817	cond: Value,
4818	then: Block.Index,	4818	then: Block.Index,
4819	@"else": Block.Index,	4819	@"else": Block.Index,
		4820	weights: Weights,
		4821	pub const Weights = enum(u32) {
		4822	// We can do this as metadata indices 0 and 1 are reserved.
		4823	none = 0,
		4824	unpredictable = 1,
		4825	/// These values should be converted to `Metadata` to be used
		4826	/// in a `prof` annotation providing branch weights.
		4827	_,
		4828	};
4820	};	4829	};
4821		4830
4822	pub const Switch = struct {	4831	pub const Switch = struct {
4823	val: Value,	4832	val: Value,
4824	default: Block.Index,	4833	default: Block.Index,
4825	cases_len: u32,	4834	cases_len: u32,
		4835	weights: BrCond.Weights,
4826	//case_vals: [cases_len]Constant,	4836	//case_vals: [cases_len]Constant,
4827	//case_blocks: [cases_len]Block.Index,	4837	//case_blocks: [cases_len]Block.Index,
4828	};	4838	};
...	@@ -4969,7 +4979,8 @@ pub const Function = struct {	...	@@ -4969,7 +4979,8 @@ pub const Function = struct {
4969	};	4979	};
4970	pub const Info = packed struct(u32) {	4980	pub const Info = packed struct(u32) {
4971	call_conv: CallConv,	4981	call_conv: CallConv,
4972	_: u22 = undefined,	4982	has_op_bundle_cold: bool,
		4983	_: u21 = undefined,
4973	};	4984	};
4974	};	4985	};
4975		4986
...	@@ -5036,6 +5047,7 @@ pub const Function = struct {	...	@@ -5036,6 +5047,7 @@ pub const Function = struct {
5036	FunctionAttributes,	5047	FunctionAttributes,
5037	Type,	5048	Type,
5038	Value,	5049	Value,
		5050	Instruction.BrCond.Weights,
5039	=> @enumFromInt(value),	5051	=> @enumFromInt(value),
5040	MemoryAccessInfo,	5052	MemoryAccessInfo,
5041	Instruction.Alloca.Info,	5053	Instruction.Alloca.Info,
...	@@ -5201,6 +5213,7 @@ pub const WipFunction = struct {	...	@@ -5201,6 +5213,7 @@ pub const WipFunction = struct {
5201	cond: Value,	5213	cond: Value,
5202	then: Block.Index,	5214	then: Block.Index,
5203	@"else": Block.Index,	5215	@"else": Block.Index,
		5216	weights: enum { none, unpredictable, then_likely, else_likely },
5204	) Allocator.Error!Instruction.Index {	5217	) Allocator.Error!Instruction.Index {
5205	assert(cond.typeOfWip(self) == .i1);	5218	assert(cond.typeOfWip(self) == .i1);
5206	try self.ensureUnusedExtraCapacity(1, Instruction.BrCond, 0);	5219	try self.ensureUnusedExtraCapacity(1, Instruction.BrCond, 0);
...	@@ -5210,6 +5223,22 @@ pub const WipFunction = struct {	...	@@ -5210,6 +5223,22 @@ pub const WipFunction = struct {
5210	.cond = cond,	5223	.cond = cond,
5211	.then = then,	5224	.then = then,
5212	.@"else" = @"else",	5225	.@"else" = @"else",
		5226	.weights = switch (weights) {
		5227	.none => .none,
		5228	.unpredictable => .unpredictable,
		5229	.then_likely, .else_likely => w: {
		5230	const branch_weights_str = try self.builder.metadataString("branch_weights");
		5231	const unlikely_const = try self.builder.metadataConstant(try self.builder.intConst(.i32, 1));
		5232	const likely_const = try self.builder.metadataConstant(try self.builder.intConst(.i32, 2000));
		5233	const weight_vals: [2]Metadata = switch (weights) {
		5234	.none, .unpredictable => unreachable,
		5235	.then_likely => .{ likely_const, unlikely_const },
		5236	.else_likely => .{ unlikely_const, likely_const },
		5237	};
		5238	const tuple = try self.builder.strTuple(branch_weights_str, &weight_vals);
		5239	break :w @enumFromInt(@intFromEnum(tuple));
		5240	},
		5241	},
5213	}),	5242	}),
5214	});	5243	});
5215	then.ptr(self).branches += 1;	5244	then.ptr(self).branches += 1;
...	@@ -5248,6 +5277,7 @@ pub const WipFunction = struct {	...	@@ -5248,6 +5277,7 @@ pub const WipFunction = struct {
5248	val: Value,	5277	val: Value,
5249	default: Block.Index,	5278	default: Block.Index,
5250	cases_len: u32,	5279	cases_len: u32,
		5280	weights: Instruction.BrCond.Weights,
5251	) Allocator.Error!WipSwitch {	5281	) Allocator.Error!WipSwitch {
5252	try self.ensureUnusedExtraCapacity(1, Instruction.Switch, cases_len * 2);	5282	try self.ensureUnusedExtraCapacity(1, Instruction.Switch, cases_len * 2);
5253	const instruction = try self.addInst(null, .{	5283	const instruction = try self.addInst(null, .{
...	@@ -5256,6 +5286,7 @@ pub const WipFunction = struct {	...	@@ -5256,6 +5286,7 @@ pub const WipFunction = struct {
5256	.val = val,	5286	.val = val,
5257	.default = default,	5287	.default = default,
5258	.cases_len = cases_len,	5288	.cases_len = cases_len,
		5289	.weights = weights,
5259	}),	5290	}),
5260	});	5291	});
5261	_ = self.extra.addManyAsSliceAssumeCapacity(cases_len * 2);	5292	_ = self.extra.addManyAsSliceAssumeCapacity(cases_len * 2);
...	@@ -5895,6 +5926,20 @@ pub const WipFunction = struct {	...	@@ -5895,6 +5926,20 @@ pub const WipFunction = struct {
5895	callee: Value,	5926	callee: Value,
5896	args: []const Value,	5927	args: []const Value,
5897	name: []const u8,	5928	name: []const u8,
		5929	) Allocator.Error!Value {
		5930	return self.callInner(kind, call_conv, function_attributes, ty, callee, args, name, false);
		5931	}
		5932
		5933	fn callInner(
		5934	self: *WipFunction,
		5935	kind: Instruction.Call.Kind,
		5936	call_conv: CallConv,
		5937	function_attributes: FunctionAttributes,
		5938	ty: Type,
		5939	callee: Value,
		5940	args: []const Value,
		5941	name: []const u8,
		5942	has_op_bundle_cold: bool,
5898	) Allocator.Error!Value {	5943	) Allocator.Error!Value {
5899	const ret_ty = ty.functionReturn(self.builder);	5944	const ret_ty = ty.functionReturn(self.builder);
5900	assert(ty.isFunction(self.builder));	5945	assert(ty.isFunction(self.builder));
...	@@ -5918,7 +5963,10 @@ pub const WipFunction = struct {	...	@@ -5918,7 +5963,10 @@ pub const WipFunction = struct {
5918	.tail_fast => .@"tail call fast",	5963	.tail_fast => .@"tail call fast",
5919	},	5964	},
5920	.data = self.addExtraAssumeCapacity(Instruction.Call{	5965	.data = self.addExtraAssumeCapacity(Instruction.Call{
5921	.info = .{ .call_conv = call_conv },	5966	.info = .{
		5967	.call_conv = call_conv,
		5968	.has_op_bundle_cold = has_op_bundle_cold,
		5969	},
5922	.attributes = function_attributes,	5970	.attributes = function_attributes,
5923	.ty = ty,	5971	.ty = ty,
5924	.callee = callee,	5972	.callee = callee,
...	@@ -5964,6 +6012,20 @@ pub const WipFunction = struct {	...	@@ -5964,6 +6012,20 @@ pub const WipFunction = struct {
5964	);	6012	);
5965	}	6013	}
5966		6014
		6015	pub fn callIntrinsicAssumeCold(self: *WipFunction) Allocator.Error!Value {
		6016	const intrinsic = try self.builder.getIntrinsic(.assume, &.{});
		6017	return self.callInner(
		6018	.normal,
		6019	CallConv.default,
		6020	.none,
		6021	intrinsic.typeOf(self.builder),
		6022	intrinsic.toValue(self.builder),
		6023	&.{try self.builder.intValue(.i1, 1)},
		6024	"",
		6025	true,
		6026	);
		6027	}
		6028
5967	pub fn callMemCpy(	6029	pub fn callMemCpy(
5968	self: *WipFunction,	6030	self: *WipFunction,
5969	dst: Value,	6031	dst: Value,
...	@@ -6040,7 +6102,7 @@ pub const WipFunction = struct {	...	@@ -6040,7 +6102,7 @@ pub const WipFunction = struct {
6040		6102
6041	break :blk metadata;	6103	break :blk metadata;
6042	},	6104	},
6043	.constant => \|constant\| try self.builder.debugConstant(constant),	6105	.constant => \|constant\| try self.builder.metadataConstant(constant),
6044	.metadata => \|metadata\| metadata,	6106	.metadata => \|metadata\| metadata,
6045	};	6107	};
6046	}	6108	}
...	@@ -6099,6 +6161,7 @@ pub const WipFunction = struct {	...	@@ -6099,6 +6161,7 @@ pub const WipFunction = struct {
6099	FunctionAttributes,	6161	FunctionAttributes,
6100	Type,	6162	Type,
6101	Value,	6163	Value,
		6164	Instruction.BrCond.Weights,
6102	=> @intFromEnum(value),	6165	=> @intFromEnum(value),
6103	MemoryAccessInfo,	6166	MemoryAccessInfo,
6104	Instruction.Alloca.Info,	6167	Instruction.Alloca.Info,
...	@@ -6380,6 +6443,7 @@ pub const WipFunction = struct {	...	@@ -6380,6 +6443,7 @@ pub const WipFunction = struct {
6380	.cond = instructions.map(extra.cond),	6443	.cond = instructions.map(extra.cond),
6381	.then = extra.then,	6444	.then = extra.then,
6382	.@"else" = extra.@"else",	6445	.@"else" = extra.@"else",
		6446	.weights = extra.weights,
6383	});	6447	});
6384	},	6448	},
6385	.call,	6449	.call,
...	@@ -6522,6 +6586,7 @@ pub const WipFunction = struct {	...	@@ -6522,6 +6586,7 @@ pub const WipFunction = struct {
6522	.val = instructions.map(extra.data.val),	6586	.val = instructions.map(extra.data.val),
6523	.default = extra.data.default,	6587	.default = extra.data.default,
6524	.cases_len = extra.data.cases_len,	6588	.cases_len = extra.data.cases_len,
		6589	.weights = extra.data.weights,
6525	});	6590	});
6526	wip_extra.appendSlice(case_vals);	6591	wip_extra.appendSlice(case_vals);
6527	wip_extra.appendSlice(case_blocks);	6592	wip_extra.appendSlice(case_blocks);
...	@@ -6744,6 +6809,7 @@ pub const WipFunction = struct {	...	@@ -6744,6 +6809,7 @@ pub const WipFunction = struct {
6744	FunctionAttributes,	6809	FunctionAttributes,
6745	Type,	6810	Type,
6746	Value,	6811	Value,
		6812	Instruction.BrCond.Weights,
6747	=> @intFromEnum(value),	6813	=> @intFromEnum(value),
6748	MemoryAccessInfo,	6814	MemoryAccessInfo,
6749	Instruction.Alloca.Info,	6815	Instruction.Alloca.Info,
...	@@ -6792,6 +6858,7 @@ pub const WipFunction = struct {	...	@@ -6792,6 +6858,7 @@ pub const WipFunction = struct {
6792	FunctionAttributes,	6858	FunctionAttributes,
6793	Type,	6859	Type,
6794	Value,	6860	Value,
		6861	Instruction.BrCond.Weights,
6795	=> @enumFromInt(value),	6862	=> @enumFromInt(value),
6796	MemoryAccessInfo,	6863	MemoryAccessInfo,
6797	Instruction.Alloca.Info,	6864	Instruction.Alloca.Info,
...	@@ -7735,6 +7802,7 @@ pub const Metadata = enum(u32) {	...	@@ -7735,6 +7802,7 @@ pub const Metadata = enum(u32) {
7735	enumerator_signed_negative,	7802	enumerator_signed_negative,
7736	subrange,	7803	subrange,
7737	tuple,	7804	tuple,
		7805	str_tuple,
7738	module_flag,	7806	module_flag,
7739	expression,	7807	expression,
7740	local_var,	7808	local_var,
...	@@ -7780,6 +7848,7 @@ pub const Metadata = enum(u32) {	...	@@ -7780,6 +7848,7 @@ pub const Metadata = enum(u32) {
7780	.enumerator_signed_negative,	7848	.enumerator_signed_negative,
7781	.subrange,	7849	.subrange,
7782	.tuple,	7850	.tuple,
		7851	.str_tuple,
7783	.module_flag,	7852	.module_flag,
7784	.local_var,	7853	.local_var,
7785	.parameter,	7854	.parameter,
...	@@ -8044,6 +8113,13 @@ pub const Metadata = enum(u32) {	...	@@ -8044,6 +8113,13 @@ pub const Metadata = enum(u32) {
8044	// elements: [elements_len]Metadata	8113	// elements: [elements_len]Metadata
8045	};	8114	};
8046		8115
		8116	pub const StrTuple = struct {
		8117	str: MetadataString,
		8118	elements_len: u32,
		8119
		8120	// elements: [elements_len]Metadata
		8121	};
		8122
8047	pub const ModuleFlag = struct {	8123	pub const ModuleFlag = struct {
8048	behavior: Metadata,	8124	behavior: Metadata,
8049	name: MetadataString,	8125	name: MetadataString,
...	@@ -8455,11 +8531,12 @@ pub fn init(options: Options) Allocator.Error!Builder {	...	@@ -8455,11 +8531,12 @@ pub fn init(options: Options) Allocator.Error!Builder {
8455	assert(try self.intConst(.i32, 0) == .@"0");	8531	assert(try self.intConst(.i32, 0) == .@"0");
8456	assert(try self.intConst(.i32, 1) == .@"1");	8532	assert(try self.intConst(.i32, 1) == .@"1");
8457	assert(try self.noneConst(.token) == .none);	8533	assert(try self.noneConst(.token) == .none);
8458	if (!self.strip) assert(try self.debugNone() == .none);	8534
		8535	assert(try self.metadataNone() == .none);
		8536	assert(try self.metadataTuple(&.{}) == .empty_tuple);
8459		8537
8460	try self.metadata_string_indices.append(self.gpa, 0);	8538	try self.metadata_string_indices.append(self.gpa, 0);
8461	assert(try self.metadataString("") == .none);	8539	assert(try self.metadataString("") == .none);
8462	assert(try self.debugTuple(&.{}) == .empty_tuple);
8463		8540
8464	return self;	8541	return self;
8465	}	8542	}
...	@@ -9685,6 +9762,13 @@ pub fn printUnbuffered(	...	@@ -9685,6 +9762,13 @@ pub fn printUnbuffered(
9685	extra.then.toInst(&function).fmt(function_index, self),	9762	extra.then.toInst(&function).fmt(function_index, self),
9686	extra.@"else".toInst(&function).fmt(function_index, self),	9763	extra.@"else".toInst(&function).fmt(function_index, self),
9687	});	9764	});
		9765	switch (extra.weights) {
		9766	.none => {},
		9767	.unpredictable => try writer.writeAll(", !unpredictable !{}"),
		9768	_ => try writer.print("{}", .{
		9769	try metadata_formatter.fmt(", !prof ", @as(Metadata, @enumFromInt(@intFromEnum(extra.weights)))),
		9770	}),
		9771	}
9688	},	9772	},
9689	.call,	9773	.call,
9690	.@"call fast",	9774	.@"call fast",
...	@@ -9729,6 +9813,9 @@ pub fn printUnbuffered(	...	@@ -9729,6 +9813,9 @@ pub fn printUnbuffered(
9729	});	9813	});
9730	}	9814	}
9731	try writer.writeByte(')');	9815	try writer.writeByte(')');
		9816	if (extra.data.info.has_op_bundle_cold) {
		9817	try writer.writeAll(" [ \"cold\"() ]");
		9818	}
9732	const call_function_attributes = extra.data.attributes.func(self);	9819	const call_function_attributes = extra.data.attributes.func(self);
9733	if (call_function_attributes != .none) try writer.print(" #{d}", .{	9820	if (call_function_attributes != .none) try writer.print(" #{d}", .{
9734	(try attribute_groups.getOrPutValue(	9821	(try attribute_groups.getOrPutValue(
...	@@ -9939,6 +10026,13 @@ pub fn printUnbuffered(	...	@@ -9939,6 +10026,13 @@ pub fn printUnbuffered(
9939	},	10026	},
9940	);	10027	);
9941	try writer.writeAll(" ]");	10028	try writer.writeAll(" ]");
		10029	switch (extra.data.weights) {
		10030	.none => {},
		10031	.unpredictable => try writer.writeAll(", !unpredictable !{}"),
		10032	_ => try writer.print("{}", .{
		10033	try metadata_formatter.fmt(", !prof ", @as(Metadata, @enumFromInt(@intFromEnum(extra.data.weights)))),
		10034	}),
		10035	}
9942	},	10036	},
9943	.va_arg => \|tag\| {	10037	.va_arg => \|tag\| {
9944	const extra = function.extraData(Function.Instruction.VaArg, instruction.data);	10038	const extra = function.extraData(Function.Instruction.VaArg, instruction.data);
...	@@ -10287,6 +10381,17 @@ pub fn printUnbuffered(	...	@@ -10287,6 +10381,17 @@ pub fn printUnbuffered(
10287	});	10381	});
10288	try writer.writeAll("}\n");	10382	try writer.writeAll("}\n");
10289	},	10383	},
		10384	.str_tuple => {
		10385	var extra = self.metadataExtraDataTrail(Metadata.StrTuple, metadata_item.data);
		10386	const elements = extra.trail.next(extra.data.elements_len, Metadata, self);
		10387	try writer.print("!{{{[str]%}", .{
		10388	.str = try metadata_formatter.fmt("", extra.data.str),
		10389	});
		10390	for (elements) \|element\| try writer.print("{[element]%}", .{
		10391	.element = try metadata_formatter.fmt("", element),
		10392	});
		10393	try writer.writeAll("}\n");
		10394	},
10290	.module_flag => {	10395	.module_flag => {
10291	const extra = self.metadataExtraData(Metadata.ModuleFlag, metadata_item.data);	10396	const extra = self.metadataExtraData(Metadata.ModuleFlag, metadata_item.data);
10292	try writer.print("!{{{[behavior]%}{[name]%}{[constant]%}}}\n", .{	10397	try writer.print("!{{{[behavior]%}{[name]%}{[constant]%}}}\n", .{
...	@@ -11799,9 +11904,9 @@ pub fn debugNamed(self: *Builder, name: MetadataString, operands: []const Metada	...	@@ -11799,9 +11904,9 @@ pub fn debugNamed(self: *Builder, name: MetadataString, operands: []const Metada
11799	self.debugNamedAssumeCapacity(name, operands);	11904	self.debugNamedAssumeCapacity(name, operands);
11800	}	11905	}
11801		11906
11802	fn debugNone(self: *Builder) Allocator.Error!Metadata {	11907	fn metadataNone(self: *Builder) Allocator.Error!Metadata {
11803	try self.ensureUnusedMetadataCapacity(1, NoExtra, 0);	11908	try self.ensureUnusedMetadataCapacity(1, NoExtra, 0);
11804	return self.debugNoneAssumeCapacity();	11909	return self.metadataNoneAssumeCapacity();
11805	}	11910	}
11806		11911
11807	pub fn debugFile(	11912	pub fn debugFile(
...	@@ -12090,12 +12195,21 @@ pub fn debugExpression(	...	@@ -12090,12 +12195,21 @@ pub fn debugExpression(
12090	return self.debugExpressionAssumeCapacity(elements);	12195	return self.debugExpressionAssumeCapacity(elements);
12091	}	12196	}
12092		12197
12093	pub fn debugTuple(	12198	pub fn metadataTuple(
12094	self: *Builder,	12199	self: *Builder,
12095	elements: []const Metadata,	12200	elements: []const Metadata,
12096	) Allocator.Error!Metadata {	12201	) Allocator.Error!Metadata {
12097	try self.ensureUnusedMetadataCapacity(1, Metadata.Tuple, elements.len);	12202	try self.ensureUnusedMetadataCapacity(1, Metadata.Tuple, elements.len);
12098	return self.debugTupleAssumeCapacity(elements);	12203	return self.metadataTupleAssumeCapacity(elements);
		12204	}
		12205
		12206	pub fn strTuple(
		12207	self: *Builder,
		12208	str: MetadataString,
		12209	elements: []const Metadata,
		12210	) Allocator.Error!Metadata {
		12211	try self.ensureUnusedMetadataCapacity(1, Metadata.StrTuple, elements.len);
		12212	return self.strTupleAssumeCapacity(str, elements);
12099	}	12213	}
12100		12214
12101	pub fn debugModuleFlag(	12215	pub fn debugModuleFlag(
...	@@ -12166,9 +12280,9 @@ pub fn debugGlobalVarExpression(	...	@@ -12166,9 +12280,9 @@ pub fn debugGlobalVarExpression(
12166	return self.debugGlobalVarExpressionAssumeCapacity(variable, expression);	12280	return self.debugGlobalVarExpressionAssumeCapacity(variable, expression);
12167	}	12281	}
12168		12282
12169	pub fn debugConstant(self: *Builder, value: Constant) Allocator.Error!Metadata {	12283	pub fn metadataConstant(self: *Builder, value: Constant) Allocator.Error!Metadata {
12170	try self.ensureUnusedMetadataCapacity(1, NoExtra, 0);	12284	try self.ensureUnusedMetadataCapacity(1, NoExtra, 0);
12171	return self.debugConstantAssumeCapacity(value);	12285	return self.metadataConstantAssumeCapacity(value);
12172	}	12286	}
12173		12287
12174	pub fn debugForwardReferenceSetType(self: *Builder, fwd_ref: Metadata, ty: Metadata) void {	12288	pub fn debugForwardReferenceSetType(self: *Builder, fwd_ref: Metadata, ty: Metadata) void {
...	@@ -12263,8 +12377,7 @@ fn debugNamedAssumeCapacity(self: *Builder, name: MetadataString, operands: []co	...	@@ -12263,8 +12377,7 @@ fn debugNamedAssumeCapacity(self: *Builder, name: MetadataString, operands: []co
12263	};	12377	};
12264	}	12378	}
12265		12379
12266	pub fn debugNoneAssumeCapacity(self: *Builder) Metadata {	12380	pub fn metadataNoneAssumeCapacity(self: *Builder) Metadata {
12267	assert(!self.strip);
12268	return self.metadataSimpleAssumeCapacity(.none, .{});	12381	return self.metadataSimpleAssumeCapacity(.none, .{});
12269	}	12382	}
12270		12383
...	@@ -12740,11 +12853,10 @@ fn debugExpressionAssumeCapacity(	...	@@ -12740,11 +12853,10 @@ fn debugExpressionAssumeCapacity(
12740	return @enumFromInt(gop.index);	12853	return @enumFromInt(gop.index);
12741	}	12854	}
12742		12855
12743	fn debugTupleAssumeCapacity(	12856	fn metadataTupleAssumeCapacity(
12744	self: *Builder,	12857	self: *Builder,
12745	elements: []const Metadata,	12858	elements: []const Metadata,
12746	) Metadata {	12859	) Metadata {
12747	assert(!self.strip);
12748	const Key = struct {	12860	const Key = struct {
12749	elements: []const Metadata,	12861	elements: []const Metadata,
12750	};	12862	};
...	@@ -12787,6 +12899,55 @@ fn debugTupleAssumeCapacity(	...	@@ -12787,6 +12899,55 @@ fn debugTupleAssumeCapacity(
12787	return @enumFromInt(gop.index);	12899	return @enumFromInt(gop.index);
12788	}	12900	}
12789		12901
		12902	fn strTupleAssumeCapacity(
		12903	self: *Builder,
		12904	str: MetadataString,
		12905	elements: []const Metadata,
		12906	) Metadata {
		12907	const Key = struct {
		12908	str: MetadataString,
		12909	elements: []const Metadata,
		12910	};
		12911	const Adapter = struct {
		12912	builder: *const Builder,
		12913	pub fn hash(_: @This(), key: Key) u32 {
		12914	var hasher = comptime std.hash.Wyhash.init(std.hash.uint32(@intFromEnum(Metadata.Tag.tuple)));
		12915	hasher.update(std.mem.sliceAsBytes(key.elements));
		12916	return @truncate(hasher.final());
		12917	}
		12918
		12919	pub fn eql(ctx: @This(), lhs_key: Key, _: void, rhs_index: usize) bool {
		12920	if (.str_tuple != ctx.builder.metadata_items.items(.tag)[rhs_index]) return false;
		12921	const rhs_data = ctx.builder.metadata_items.items(.data)[rhs_index];
		12922	var rhs_extra = ctx.builder.metadataExtraDataTrail(Metadata.StrTuple, rhs_data);
		12923	return rhs_extra.data.str == lhs_key.str and std.mem.eql(
		12924	Metadata,
		12925	lhs_key.elements,
		12926	rhs_extra.trail.next(rhs_extra.data.elements_len, Metadata, ctx.builder),
		12927	);
		12928	}
		12929	};
		12930
		12931	const gop = self.metadata_map.getOrPutAssumeCapacityAdapted(
		12932	Key{ .str = str, .elements = elements },
		12933	Adapter{ .builder = self },
		12934	);
		12935
		12936	if (!gop.found_existing) {
		12937	gop.key_ptr.* = {};
		12938	gop.value_ptr.* = {};
		12939	self.metadata_items.appendAssumeCapacity(.{
		12940	.tag = .str_tuple,
		12941	.data = self.addMetadataExtraAssumeCapacity(Metadata.StrTuple{
		12942	.str = str,
		12943	.elements_len = @intCast(elements.len),
		12944	}),
		12945	});
		12946	self.metadata_extra.appendSliceAssumeCapacity(@ptrCast(elements));
		12947	}
		12948	return @enumFromInt(gop.index);
		12949	}
		12950
12790	fn debugModuleFlagAssumeCapacity(	12951	fn debugModuleFlagAssumeCapacity(
12791	self: *Builder,	12952	self: *Builder,
12792	behavior: Metadata,	12953	behavior: Metadata,
...	@@ -12877,8 +13038,7 @@ fn debugGlobalVarExpressionAssumeCapacity(	...	@@ -12877,8 +13038,7 @@ fn debugGlobalVarExpressionAssumeCapacity(
12877	});	13038	});
12878	}	13039	}
12879		13040
12880	fn debugConstantAssumeCapacity(self: *Builder, constant: Constant) Metadata {	13041	fn metadataConstantAssumeCapacity(self: *Builder, constant: Constant) Metadata {
12881	assert(!self.strip);
12882	const Adapter = struct {	13042	const Adapter = struct {
12883	builder: *const Builder,	13043	builder: *const Builder,
12884	pub fn hash(_: @This(), key: Constant) u32 {	13044	pub fn hash(_: @This(), key: Constant) u32 {
...	@@ -13757,15 +13917,18 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co	...	@@ -13757,15 +13917,18 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
13757	}	13917	}
13758		13918
13759	// METADATA_KIND_BLOCK	13919	// METADATA_KIND_BLOCK
13760	if (!self.strip) {	13920	{
13761	const MetadataKindBlock = ir.MetadataKindBlock;	13921	const MetadataKindBlock = ir.MetadataKindBlock;
13762	var metadata_kind_block = try module_block.enterSubBlock(MetadataKindBlock, true);	13922	var metadata_kind_block = try module_block.enterSubBlock(MetadataKindBlock, true);
13763		13923
13764	inline for (@typeInfo(ir.FixedMetadataKind).Enum.fields) \|field\| {	13924	inline for (@typeInfo(ir.FixedMetadataKind).Enum.fields) \|field\| {
13765	try metadata_kind_block.writeAbbrev(MetadataKindBlock.Kind{	13925	// don't include `dbg` in stripped functions
13766	.id = field.value,	13926	if (!(self.strip and std.mem.eql(u8, field.name, "dbg"))) {
13767	.name = field.name,	13927	try metadata_kind_block.writeAbbrev(MetadataKindBlock.Kind{
13768	});	13928	.id = field.value,
		13929	.name = field.name,
		13930	});
		13931	}
13769	}	13932	}
13770		13933
13771	try metadata_kind_block.end();	13934	try metadata_kind_block.end();
...	@@ -13810,14 +13973,14 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co	...	@@ -13810,14 +13973,14 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
13810	const metadata_adapter = MetadataAdapter.init(self, constant_adapter);	13973	const metadata_adapter = MetadataAdapter.init(self, constant_adapter);
13811		13974
13812	// METADATA_BLOCK	13975	// METADATA_BLOCK
13813	if (!self.strip) {	13976	{
13814	const MetadataBlock = ir.MetadataBlock;	13977	const MetadataBlock = ir.MetadataBlock;
13815	var metadata_block = try module_block.enterSubBlock(MetadataBlock, true);	13978	var metadata_block = try module_block.enterSubBlock(MetadataBlock, true);
13816		13979
13817	const MetadataBlockWriter = @TypeOf(metadata_block);	13980	const MetadataBlockWriter = @TypeOf(metadata_block);
13818		13981
13819	// Emit all MetadataStrings	13982	// Emit all MetadataStrings
13820	{	13983	if (self.metadata_string_map.count() > 1) {
13821	const strings_offset, const strings_size = blk: {	13984	const strings_offset, const strings_size = blk: {
13822	var strings_offset: u32 = 0;	13985	var strings_offset: u32 = 0;
13823	var strings_size: u32 = 0;	13986	var strings_size: u32 = 0;
...	@@ -14087,6 +14250,22 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co	...	@@ -14087,6 +14250,22 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14087	.elements = elements,	14250	.elements = elements,
14088	}, metadata_adapter);	14251	}, metadata_adapter);
14089	},	14252	},
		14253	.str_tuple => {
		14254	var extra = self.metadataExtraDataTrail(Metadata.StrTuple, data);
		14255
		14256	const elements = extra.trail.next(extra.data.elements_len, Metadata, self);
		14257
		14258	const all_elems = try self.gpa.alloc(Metadata, elements.len + 1);
		14259	defer self.gpa.free(all_elems);
		14260	all_elems[0] = @enumFromInt(metadata_adapter.getMetadataStringIndex(extra.data.str));
		14261	for (elements, all_elems[1..]) \|elem, *out_elem\| {
		14262	out_elem.* = @enumFromInt(metadata_adapter.getMetadataIndex(elem));
		14263	}
		14264
		14265	try metadata_block.writeAbbrev(MetadataBlock.Node{
		14266	.elements = all_elems,
		14267	});
		14268	},
14090	.module_flag => {	14269	.module_flag => {
14091	const extra = self.metadataExtraData(Metadata.ModuleFlag, data);	14270	const extra = self.metadataExtraData(Metadata.ModuleFlag, data);
14092	try metadata_block.writeAbbrev(MetadataBlock.Node{	14271	try metadata_block.writeAbbrev(MetadataBlock.Node{
...	@@ -14188,6 +14367,18 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co	...	@@ -14188,6 +14367,18 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14188	try metadata_block.end();	14367	try metadata_block.end();
14189	}	14368	}
14190		14369
		14370	// OPERAND_BUNDLE_TAGS_BLOCK
		14371	{
		14372	const OperandBundleTags = ir.OperandBundleTags;
		14373	var operand_bundle_tags_block = try module_block.enterSubBlock(OperandBundleTags, true);
		14374
		14375	try operand_bundle_tags_block.writeAbbrev(OperandBundleTags.OperandBundleTag{
		14376	.tag = "cold",
		14377	});
		14378
		14379	try operand_bundle_tags_block.end();
		14380	}
		14381
14191	// Block info	14382	// Block info
14192	{	14383	{
14193	const BlockInfo = ir.BlockInfo;	14384	const BlockInfo = ir.BlockInfo;
...	@@ -14243,7 +14434,6 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co	...	@@ -14243,7 +14434,6 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14243	.instruction => \|instruction\| instruction.valueIndex(adapter.func) + adapter.firstInstr(),	14434	.instruction => \|instruction\| instruction.valueIndex(adapter.func) + adapter.firstInstr(),
14244	.constant => \|constant\| adapter.constant_adapter.getConstantIndex(constant),	14435	.constant => \|constant\| adapter.constant_adapter.getConstantIndex(constant),
14245	.metadata => \|metadata\| {	14436	.metadata => \|metadata\| {
14246	assert(!adapter.func.strip);
14247	const real_metadata = metadata.unwrap(adapter.metadata_adapter.builder);	14437	const real_metadata = metadata.unwrap(adapter.metadata_adapter.builder);
14248	if (@intFromEnum(real_metadata) < Metadata.first_local_metadata)	14438	if (@intFromEnum(real_metadata) < Metadata.first_local_metadata)
14249	return adapter.metadata_adapter.getMetadataIndex(real_metadata) - 1;	14439	return adapter.metadata_adapter.getMetadataIndex(real_metadata) - 1;
...	@@ -14335,6 +14525,10 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co	...	@@ -14335,6 +14525,10 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14335	=> \|kind\| {	14525	=> \|kind\| {
14336	var extra = func.extraDataTrail(Function.Instruction.Call, data);	14526	var extra = func.extraDataTrail(Function.Instruction.Call, data);
14337		14527
		14528	if (extra.data.info.has_op_bundle_cold) {
		14529	try function_block.writeAbbrev(FunctionBlock.ColdOperandBundle{});
		14530	}
		14531
14338	const call_conv = extra.data.info.call_conv;	14532	const call_conv = extra.data.info.call_conv;
14339	const args = extra.trail.next(extra.data.args_len, Value, &func);	14533	const args = extra.trail.next(extra.data.args_len, Value, &func);
14340	try function_block.writeAbbrevAdapted(FunctionBlock.Call{	14534	try function_block.writeAbbrevAdapted(FunctionBlock.Call{
...	@@ -14358,6 +14552,10 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co	...	@@ -14358,6 +14552,10 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14358	=> \|kind\| {	14552	=> \|kind\| {
14359	var extra = func.extraDataTrail(Function.Instruction.Call, data);	14553	var extra = func.extraDataTrail(Function.Instruction.Call, data);
14360		14554
		14555	if (extra.data.info.has_op_bundle_cold) {
		14556	try function_block.writeAbbrev(FunctionBlock.ColdOperandBundle{});
		14557	}
		14558
14361	const call_conv = extra.data.info.call_conv;	14559	const call_conv = extra.data.info.call_conv;
14362	const args = extra.trail.next(extra.data.args_len, Value, &func);	14560	const args = extra.trail.next(extra.data.args_len, Value, &func);
14363	try function_block.writeAbbrevAdapted(FunctionBlock.CallFast{	14561	try function_block.writeAbbrevAdapted(FunctionBlock.CallFast{
...	@@ -14837,14 +15035,14 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co	...	@@ -14837,14 +15035,14 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14837	}	15035	}
14838		15036
14839	// METADATA_ATTACHMENT_BLOCK	15037	// METADATA_ATTACHMENT_BLOCK
14840	const any_nosanitize = true;	15038	{
14841	if (!func.strip or any_nosanitize) {
14842	const MetadataAttachmentBlock = ir.MetadataAttachmentBlock;	15039	const MetadataAttachmentBlock = ir.MetadataAttachmentBlock;
14843	var metadata_attach_block = try function_block.enterSubBlock(MetadataAttachmentBlock, false);	15040	var metadata_attach_block = try function_block.enterSubBlock(MetadataAttachmentBlock, false);
14844		15041
14845	if (!func.strip) blk: {	15042	dbg: {
		15043	if (func.strip) break :dbg;
14846	const dbg = func.global.ptrConst(self).dbg;	15044	const dbg = func.global.ptrConst(self).dbg;
14847	if (dbg == .none) break :blk;	15045	if (dbg == .none) break :dbg;
14848	try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentGlobalSingle{	15046	try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentGlobalSingle{
14849	.kind = .dbg,	15047	.kind = .dbg,
14850	.metadata = @enumFromInt(metadata_adapter.getMetadataIndex(dbg) - 1),	15048	.metadata = @enumFromInt(metadata_adapter.getMetadataIndex(dbg) - 1),
...	@@ -14852,14 +15050,30 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co	...	@@ -14852,14 +15050,30 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14852	}	15050	}
14853		15051
14854	var instr_index: u32 = 0;	15052	var instr_index: u32 = 0;
14855	for (func.instructions.items(.tag)) \|instr_tag\| switch (instr_tag) {	15053	for (func.instructions.items(.tag), func.instructions.items(.data)) \|instr_tag, data\| switch (instr_tag) {
14856	.arg, .block => {},	15054	.arg, .block => {}, // not an actual instruction
14857	else => {	15055	else => {
14858	try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentInstructionSingle{	15056	instr_index += 1;
14859	.inst = instr_index,	15057	},
14860	.kind = .nosanitize,	15058	.br_cond, .@"switch" => {
14861	.metadata = @enumFromInt(metadata_adapter.getMetadataIndex(.empty_tuple) - 1),	15059	const weights = switch (instr_tag) {
14862	});	15060	.br_cond => func.extraData(Function.Instruction.BrCond, data).weights,
		15061	.@"switch" => func.extraData(Function.Instruction.Switch, data).weights,
		15062	else => unreachable,
		15063	};
		15064	switch (weights) {
		15065	.none => {},
		15066	.unpredictable => try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentInstructionSingle{
		15067	.inst = instr_index,
		15068	.kind = .unpredictable,
		15069	.metadata = @enumFromInt(metadata_adapter.getMetadataIndex(.empty_tuple) - 1),
		15070	}),
		15071	_ => try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentInstructionSingle{
		15072	.inst = instr_index,
		15073	.kind = .prof,
		15074	.metadata = @enumFromInt(metadata_adapter.getMetadataIndex(@enumFromInt(@intFromEnum(weights))) - 1),
		15075	}),
		15076	}
14863	instr_index += 1;	15077	instr_index += 1;
14864	},	15078	},
14865	};	15079	};

src/codegen/llvm/ir.zig+25-3

 @@ -25,7 +25,7 @@ const BlockAbbrev = AbbrevOp{ .vbr = 6 };
 pub const FixedMetadataKind = enum(u8) {
     dbg = 0,
     //tbaa = 1,
-    //prof = 2,
+    prof = 2,
     //fpmath = 3,
     //range = 4,
     //@"tbaa.struct" = 5,
 @@ -38,7 +38,7 @@ pub const FixedMetadataKind = enum(u8) {
     //dereferenceable = 12,
     //dereferenceable_or_null = 13,
     //@"make.implicit" = 14,
-    //unpredictable = 15,
+    unpredictable = 15,
     //@"invariant.group" = 16,
     //@"align" = 17,
     //@"llvm.loop" = 18,
 @@ -54,7 +54,7 @@ pub const FixedMetadataKind = enum(u8) {
     //vcall_visibility = 28,
     //noundef = 29,
     //annotation = 30,
-    nosanitize = 31,
+    //nosanitize = 31,
     //func_sanitize = 32,
     //exclude = 33,
     //memprof = 34,
 @@ -1220,6 +1220,20 @@ pub const MetadataBlock = struct {
     };
 };
+pub const OperandBundleTags = struct {
+    pub const id = 21;
+    pub const abbrevs = [_]type{OperandBundleTag};
+    pub const OperandBundleTag = struct {
+        pub const ops = [_]AbbrevOp{
+            .{ .literal = 1 },
+            .array_char6,
+        };
+        tag: []const u8,
+    };
+};
 pub const FunctionMetadataBlock = struct {
     pub const id = 15;
 @@ -1279,6 +1293,7 @@ pub const FunctionBlock = struct {
         Fence,
         DebugLoc,
         DebugLocAgain,
+        ColdOperandBundle,
     };
     pub const DeclareBlocks = struct {
 @@ -1791,6 +1806,13 @@ pub const FunctionBlock = struct {
             .{ .literal = 33 },
         };
     };
+    pub const ColdOperandBundle = struct {
+        pub const ops = [_]AbbrevOp{
+            .{ .literal = 55 },
+            .{ .literal = 0 },
+        };
+    };
 };
 pub const FunctionValueSymbolTable = struct {

src/codegen/spirv.zig+17-31

 @@ -6173,11 +6173,10 @@ const NavGen = struct {
         const pt = self.pt;
         const zcu = pt.zcu;
         const target = self.getTarget();
-        const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
+        const switch_br = self.air.unwrapSwitch(inst);
-        const cond_ty = self.typeOf(pl_op.operand);
+        const cond_ty = self.typeOf(switch_br.operand);
-        const cond = try self.resolve(pl_op.operand);
+        const cond = try self.resolve(switch_br.operand);
         var cond_indirect = try self.convertToIndirect(cond_ty, cond);
-        const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
         const cond_words: u32 = switch (cond_ty.zigTypeTag(zcu)) {
             .Bool, .ErrorSet => 1,
 @@ -6204,18 +6203,15 @@ const NavGen = struct {
             else => return self.todo("implement switch for type {s}", .{@tagName(cond_ty.zigTypeTag(zcu))}),
         };
-        const num_cases = switch_br.data.cases_len;
+        const num_cases = switch_br.cases_len;
         // Compute the total number of arms that we need.
         // Zig switches are grouped by condition, so we need to loop through all of them
         const num_conditions = blk: {
-            var extra_index: usize = switch_br.end;
             var num_conditions: u32 = 0;
-            for (0..num_cases) |_| {
+            var it = switch_br.iterateCases();
-                const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
+            while (it.next()) |case| {
-                const case_body = self.air.extra[case.end + case.data.items_len ..][0..case.data.body_len];
+                num_conditions += @intCast(case.items.len);
-                extra_index = case.end + case.data.items_len + case_body.len;
-                num_conditions += case.data.items_len;
             }
             break :blk num_conditions;
         };
 @@ -6244,17 +6240,12 @@ const NavGen = struct {
         // Emit each of the cases
         {
-            var extra_index: usize = switch_br.end;
+            var it = switch_br.iterateCases();
-            for (0..num_cases) |case_i| {
+            while (it.next()) |case| {
                 // SPIR-V needs a literal here, which' width depends on the case condition.
-                const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
+                const label = case_labels.at(case.idx);
-                const items: []const Air.Inst.Ref = @ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
-                const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];
-                extra_index = case.end + case.data.items_len + case_body.len;
-                const label = case_labels.at(case_i);
-                for (items) |item| {
+                for (case.items) |item| {
                     const value = (try self.air.value(item, pt)) orelse unreachable;
                     const int_val: u64 = switch (cond_ty.zigTypeTag(zcu)) {
                         .Bool, .Int => if (cond_ty.isSignedInt(zcu)) @bitCast(value.toSignedInt(zcu)) else value.toUnsignedInt(zcu),
 @@ -6285,20 +6276,15 @@ const NavGen = struct {
         }
         // Now, finally, we can start emitting each of the cases.
-        var extra_index: usize = switch_br.end;
+        var it = switch_br.iterateCases();
-        for (0..num_cases) |case_i| {
+        while (it.next()) |case| {
-            const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
+            const label = case_labels.at(case.idx);
-            const items: []const Air.Inst.Ref = @ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
-            const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
-            extra_index = case.end + case.data.items_len + case_body.len;
-            const label = case_labels.at(case_i);
             try self.beginSpvBlock(label);
             switch (self.control_flow) {
                 .structured => {
-                    const next_block = try self.genStructuredBody(.selection, case_body);
+                    const next_block = try self.genStructuredBody(.selection, case.body);
                     incoming_structured_blocks.appendAssumeCapacity(.{
                         .src_label = self.current_block_label,
                         .next_block = next_block,
 @@ -6306,12 +6292,12 @@ const NavGen = struct {
                     try self.func.body.emitBranch(self.spv.gpa, merge_label.?);
                 },
                 .unstructured => {
-                    try self.genBody(case_body);
+                    try self.genBody(case.body);
                 },
             }
         }
-        const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);
+        const else_body = it.elseBody();
         try self.beginSpvBlock(default);
         if (else_body.len != 0) {
             switch (self.control_flow) {

src/print_air.zig+25-21

...	@@ -297,8 +297,8 @@ const Writer = struct {	...	@@ -297,8 +297,8 @@ const Writer = struct {
297	.union_init => try w.writeUnionInit(s, inst),	297	.union_init => try w.writeUnionInit(s, inst),
298	.br => try w.writeBr(s, inst),	298	.br => try w.writeBr(s, inst),
299	.cond_br => try w.writeCondBr(s, inst),	299	.cond_br => try w.writeCondBr(s, inst),
300	.@"try" => try w.writeTry(s, inst),	300	.@"try", .try_cold => try w.writeTry(s, inst),
301	.try_ptr => try w.writeTryPtr(s, inst),	301	.try_ptr, .try_ptr_cold => try w.writeTryPtr(s, inst),
302	.switch_br => try w.writeSwitchBr(s, inst),	302	.switch_br => try w.writeSwitchBr(s, inst),
303	.cmpxchg_weak, .cmpxchg_strong => try w.writeCmpxchg(s, inst),	303	.cmpxchg_weak, .cmpxchg_strong => try w.writeCmpxchg(s, inst),
304	.fence => try w.writeFence(s, inst),	304	.fence => try w.writeFence(s, inst),
...	@@ -825,41 +825,40 @@ const Writer = struct {	...	@@ -825,41 +825,40 @@ const Writer = struct {
825	}	825	}
826		826
827	fn writeSwitchBr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {	827	fn writeSwitchBr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
828	const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	828	const switch_br = w.air.unwrapSwitch(inst);
829	const switch_br = w.air.extraData(Air.SwitchBr, pl_op.payload);	829
830	const liveness = if (w.liveness) \|liveness\|	830	const liveness = if (w.liveness) \|liveness\|
831	liveness.getSwitchBr(w.gpa, inst, switch_br.data.cases_len + 1) catch	831	liveness.getSwitchBr(w.gpa, inst, switch_br.cases_len + 1) catch
832	@panic("out of memory")	832	@panic("out of memory")
833	else blk: {	833	else blk: {
834	const slice = w.gpa.alloc([]const Air.Inst.Index, switch_br.data.cases_len + 1) catch	834	const slice = w.gpa.alloc([]const Air.Inst.Index, switch_br.cases_len + 1) catch
835	@panic("out of memory");	835	@panic("out of memory");
836	@memset(slice, &.{});	836	@memset(slice, &.{});
837	break :blk Liveness.SwitchBrTable{ .deaths = slice };	837	break :blk Liveness.SwitchBrTable{ .deaths = slice };
838	};	838	};
839	defer w.gpa.free(liveness.deaths);	839	defer w.gpa.free(liveness.deaths);
840	var extra_index: usize = switch_br.end;
841	var case_i: u32 = 0;
842		840
843	try w.writeOperand(s, inst, 0, pl_op.operand);	841	try w.writeOperand(s, inst, 0, switch_br.operand);
844	if (w.skip_body) return s.writeAll(", ...");	842	if (w.skip_body) return s.writeAll(", ...");
845	const old_indent = w.indent;	843	const old_indent = w.indent;
846	w.indent += 2;	844	w.indent += 2;
847		845
848	while (case_i < switch_br.data.cases_len) : (case_i += 1) {	846	var it = switch_br.iterateCases();
849	const case = w.air.extraData(Air.SwitchBr.Case, extra_index);	847	while (it.next()) \|case\| {
850	const items = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[case.end..][0..case.data.items_len]));
851	const case_body: []const Air.Inst.Index = @ptrCast(w.air.extra[case.end + items.len ..][0..case.data.body_len]);
852	extra_index = case.end + case.data.items_len + case_body.len;
853
854	try s.writeAll(", [");	848	try s.writeAll(", [");
855	for (items, 0..) \|item, item_i\| {	849	for (case.items, 0..) \|item, item_i\| {
856	if (item_i != 0) try s.writeAll(", ");	850	if (item_i != 0) try s.writeAll(", ");
857	try w.writeInstRef(s, item, false);	851	try w.writeInstRef(s, item, false);
858	}	852	}
859	try s.writeAll("] => {\n");	853	try s.writeAll("] ");
		854	const hint = switch_br.getHint(case.idx);
		855	if (hint != .none) {
		856	try s.print(".{s} ", .{@tagName(hint)});
		857	}
		858	try s.writeAll("=> {\n");
860	w.indent += 2;	859	w.indent += 2;
861		860
862	const deaths = liveness.deaths[case_i];	861	const deaths = liveness.deaths[case.idx];
863	if (deaths.len != 0) {	862	if (deaths.len != 0) {
864	try s.writeByteNTimes(' ', w.indent);	863	try s.writeByteNTimes(' ', w.indent);
865	for (deaths, 0..) \|operand, i\| {	864	for (deaths, 0..) \|operand, i\| {
...	@@ -869,15 +868,20 @@ const Writer = struct {	...	@@ -869,15 +868,20 @@ const Writer = struct {
869	try s.writeAll("\n");	868	try s.writeAll("\n");
870	}	869	}
871		870
872	try w.writeBody(s, case_body);	871	try w.writeBody(s, case.body);
873	w.indent -= 2;	872	w.indent -= 2;
874	try s.writeByteNTimes(' ', w.indent);	873	try s.writeByteNTimes(' ', w.indent);
875	try s.writeAll("}");	874	try s.writeAll("}");
876	}	875	}
877		876
878	const else_body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra_index..][0..switch_br.data.else_body_len]);	877	const else_body = it.elseBody();
879	if (else_body.len != 0) {	878	if (else_body.len != 0) {
880	try s.writeAll(", else => {\n");	879	try s.writeAll(", else ");
		880	const hint = switch_br.getElseHint();
		881	if (hint != .none) {
		882	try s.print(".{s} ", .{@tagName(hint)});
		883	}
		884	try s.writeAll("=> {\n");
881	w.indent += 2;	885	w.indent += 2;
882		886
883	const deaths = liveness.deaths[liveness.deaths.len - 1];	887	const deaths = liveness.deaths[liveness.deaths.len - 1];

src/print_zir.zig+1-1

...	@@ -564,7 +564,6 @@ const Writer = struct {	...	@@ -564,7 +564,6 @@ const Writer = struct {
564	.fence,	564	.fence,
565	.set_float_mode,	565	.set_float_mode,
566	.set_align_stack,	566	.set_align_stack,
567	.set_cold,
568	.wasm_memory_size,	567	.wasm_memory_size,
569	.int_from_error,	568	.int_from_error,
570	.error_from_int,	569	.error_from_int,
...	@@ -573,6 +572,7 @@ const Writer = struct {	...	@@ -573,6 +572,7 @@ const Writer = struct {
573	.work_item_id,	572	.work_item_id,
574	.work_group_size,	573	.work_group_size,
575	.work_group_id,	574	.work_group_id,
		575	.branch_hint,
576	=> {	576	=> {
577	const inst_data = self.code.extraData(Zir.Inst.UnNode, extended.operand).data;	577	const inst_data = self.code.extraData(Zir.Inst.UnNode, extended.operand).data;
578	try self.writeInstRef(stream, inst_data.operand);	578	try self.writeInstRef(stream, inst_data.operand);