authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2024-04-25 03:46:10+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2024-09-01 18:30:31+01:00
log5fb4a7df38deb705f77088d7788f0acc09da613d
tree5616328dadfb98993775ae1758ce631a3a216334
parent1b000b90c9a7abde3aeacf29cef73a877da237e1
signaturelock-open Commit is signed but in an unrecognized format.

Air: add explicit `repeat` instruction to repeat loops

This commit introduces a new AIR instruction, `repeat`, which causes control flow to move back to the start of a given AIR loop. `loop` instructions will no longer automatically perform this operation after control flow reaches the end of the body. The motivation for making this change now was really just consistency with the upcoming implementation of #8220: it wouldn't make sense to have this feature work significantly differently. However, there were already some TODOs kicking around which wanted this feature. It's useful for two key reasons: * It allows loops over AIR instruction bodies to loop precisely until they reach a `noreturn` instruction. This allows for tail calling a few things, and avoiding a range check on each iteration of a hot path, plus gives a nice assertion that validates AIR structure a little. This is a very minor benefit, which this commit does apply to the LLVM and C backends. * It should allow for more compact ZIR and AIR to be emitted by having AstGen emit `repeat` instructions more often rather than having `continue` statements `break` to a `block` which is *followed* by a `repeat`. This is done in status quo because `repeat` instructions only ever cause the direct parent block to repeat. Now that AIR is more flexible, this flexibility can be pretty trivially extended to ZIR, and we can then emit better ZIR. This commit does not implement this. Support for this feature is currently regressed on all self-hosted native backends, including x86_64. This support will be added where necessary before this branch is merged.

15 files changed, 257 insertions(+), 114 deletions(-)

src/Air.zig+11-4
......@@ -274,13 +274,15 @@ pub const Inst = struct {
274274 /// is to encounter a `br` that targets this `block`. If the `block` type is `noreturn`,
275275 /// then there do not exist any `br` instructions targeting this `block`.
276276 block,
277 /// A labeled block of code that loops forever. At the end of the body it is implied
278 /// to repeat; no explicit "repeat" instruction terminates loop bodies.
277 /// A labeled block of code that loops forever. The body must be `noreturn`: loops
278 /// occur through an explicit `repeat` instruction pointing back to this one.
279279 /// Result type is always `noreturn`; no instructions in a block follow this one.
280 /// The body never ends with a `noreturn` instruction, so the "repeat" operation
281 /// is always statically reachable.
280 /// There is always at least one `repeat` instruction referencing the loop.
282281 /// Uses the `ty_pl` field. Payload is `Block`.
283282 loop,
283 /// Sends control flow back to the beginning of a parent `loop` body.
284 /// Uses the `repeat` field.
285 repeat,
284286 /// Return from a block with a result.
285287 /// Result type is always noreturn; no instructions in a block follow this one.
286288 /// Uses the `br` field.
......@@ -1045,6 +1047,9 @@ pub const Inst = struct {
10451047 block_inst: Index,
10461048 operand: Ref,
10471049 },
1050 repeat: struct {
1051 loop_inst: Index,
1052 },
10481053 pl_op: struct {
10491054 operand: Ref,
10501055 payload: u32,
......@@ -1445,6 +1450,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
14451450 => return datas[@intFromEnum(inst)].ty_op.ty.toType(),
14461451
14471452 .loop,
1453 .repeat,
14481454 .br,
14491455 .cond_br,
14501456 .switch_br,
......@@ -1602,6 +1608,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
16021608 .arg,
16031609 .block,
16041610 .loop,
1611 .repeat,
16051612 .br,
16061613 .trap,
16071614 .breakpoint,
src/Air/types_resolved.zig+1
......@@ -420,6 +420,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
420420 .dbg_stmt,
421421 .err_return_trace,
422422 .save_err_return_trace_index,
423 .repeat,
423424 => {},
424425 }
425426 }
src/Liveness.zig+56-6
......@@ -70,7 +70,8 @@ pub const Block = struct {
7070const LivenessPass = enum {
7171 /// In this pass, we perform some basic analysis of loops to gain information the main pass
7272 /// needs. In particular, for every `loop`, we track the following information:
73 /// * Every block which the loop body contains a `br` to.
73 /// * Every outer block which the loop body contains a `br` to.
74 /// * Every outer loop which the loop body contains a `repeat` to.
7475 /// * Every operand referenced within the loop body but created outside the loop.
7576 /// This gives the main analysis pass enough information to determine the full set of
7677 /// instructions which need to be alive when a loop repeats. This data is TEMPORARILY stored in
......@@ -89,7 +90,8 @@ fn LivenessPassData(comptime pass: LivenessPass) type {
8990 return switch (pass) {
9091 .loop_analysis => struct {
9192 /// The set of blocks which are exited with a `br` instruction at some point within this
92 /// body and which we are currently within.
93 /// body and which we are currently within. Also includes `loop`s which are the target
94 /// of a `repeat` instruction.
9395 breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .{},
9496
9597 /// The set of operands for which we have seen at least one usage but not their birth.
......@@ -102,7 +104,7 @@ fn LivenessPassData(comptime pass: LivenessPass) type {
102104 },
103105
104106 .main_analysis => struct {
105 /// Every `block` currently under analysis.
107 /// Every `block` and `loop` currently under analysis.
106108 block_scopes: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockScope) = .{},
107109
108110 /// The set of instructions currently alive in the current control
......@@ -114,7 +116,8 @@ fn LivenessPassData(comptime pass: LivenessPass) type {
114116 old_extra: std.ArrayListUnmanaged(u32) = .{},
115117
116118 const BlockScope = struct {
117 /// The set of instructions which are alive upon a `br` to this block.
119 /// If this is a `block`, these instructions are alive upon a `br` to this block.
120 /// If this is a `loop`, these instructions are alive upon a `repeat` to this block.
118121 live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
119122 };
120123
......@@ -326,6 +329,7 @@ pub fn categorizeOperand(
326329 .ret_ptr,
327330 .trap,
328331 .breakpoint,
332 .repeat,
329333 .dbg_stmt,
330334 .unreach,
331335 .ret_addr,
......@@ -1201,6 +1205,7 @@ fn analyzeInst(
12011205 },
12021206
12031207 .br => return analyzeInstBr(a, pass, data, inst),
1208 .repeat => return analyzeInstRepeat(a, pass, data, inst),
12041209
12051210 .assembly => {
12061211 const extra = a.air.extraData(Air.Asm, inst_datas[@intFromEnum(inst)].ty_pl.payload);
......@@ -1380,6 +1385,33 @@ fn analyzeInstBr(
13801385 return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none });
13811386}
13821387
1388fn analyzeInstRepeat(
1389 a: *Analysis,
1390 comptime pass: LivenessPass,
1391 data: *LivenessPassData(pass),
1392 inst: Air.Inst.Index,
1393) !void {
1394 const inst_datas = a.air.instructions.items(.data);
1395 const repeat = inst_datas[@intFromEnum(inst)].repeat;
1396 const gpa = a.gpa;
1397
1398 switch (pass) {
1399 .loop_analysis => {
1400 try data.breaks.put(gpa, repeat.loop_inst, {});
1401 },
1402
1403 .main_analysis => {
1404 const block_scope = data.block_scopes.get(repeat.loop_inst).?; // we should always be repeating an enclosing loop
1405
1406 const new_live_set = try block_scope.live_set.clone(gpa);
1407 data.live_set.deinit(gpa);
1408 data.live_set = new_live_set;
1409 },
1410 }
1411
1412 return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1413}
1414
13831415fn analyzeInstBlock(
13841416 a: *Analysis,
13851417 comptime pass: LivenessPass,
......@@ -1402,8 +1434,10 @@ fn analyzeInstBlock(
14021434
14031435 .main_analysis => {
14041436 log.debug("[{}] %{}: block live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1437 // We can move the live set because the body should have a noreturn
1438 // instruction which overrides the set.
14051439 try data.block_scopes.put(gpa, inst, .{
1406 .live_set = try data.live_set.clone(gpa),
1440 .live_set = data.live_set.move(),
14071441 });
14081442 defer {
14091443 log.debug("[{}] %{}: popped block scope", .{ pass, inst });
......@@ -1471,10 +1505,15 @@ fn analyzeInstLoop(
14711505
14721506 try analyzeBody(a, pass, data, body);
14731507
1508 // `loop`s are guaranteed to have at least one matching `repeat`.
1509 // However, we no longer care about repeats of this loop itself.
1510 assert(data.breaks.remove(inst));
1511
1512 const extra_index: u32 = @intCast(a.extra.items.len);
1513
14741514 const num_breaks = data.breaks.count();
14751515 try a.extra.ensureUnusedCapacity(gpa, 1 + num_breaks);
14761516
1477 const extra_index = @as(u32, @intCast(a.extra.items.len));
14781517 a.extra.appendAssumeCapacity(num_breaks);
14791518
14801519 var it = data.breaks.keyIterator();
......@@ -1543,6 +1582,17 @@ fn analyzeInstLoop(
15431582 }
15441583 }
15451584
1585 // Now, `data.live_set` is the operands which must be alive when the loop repeats.
1586 // Move them into a block scope for corresponding `repeat` instructions to notice.
1587 log.debug("[{}] %{}: loop live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1588 try data.block_scopes.putNoClobber(gpa, inst, .{
1589 .live_set = data.live_set.move(),
1590 });
1591 defer {
1592 log.debug("[{}] %{}: popped loop block scop", .{ pass, inst });
1593 var scope = data.block_scopes.fetchRemove(inst).?.value;
1594 scope.live_set.deinit(gpa);
1595 }
15461596 try analyzeBody(a, pass, data, body);
15471597 },
15481598 }
src/Liveness/Verify.zig+29-9
......@@ -1,28 +1,38 @@
1//! Verifies that liveness information is valid.
1//! Verifies that Liveness information is valid.
22
33gpa: std.mem.Allocator,
44air: Air,
55liveness: Liveness,
66live: LiveMap = .{},
77blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .{},
8loops: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .{},
89intern_pool: *const InternPool,
910
1011pub const Error = error{ LivenessInvalid, OutOfMemory };
1112
1213pub fn deinit(self: *Verify) void {
1314 self.live.deinit(self.gpa);
14 var block_it = self.blocks.valueIterator();
15 while (block_it.next()) |block| block.deinit(self.gpa);
16 self.blocks.deinit(self.gpa);
15 {
16 var it = self.blocks.valueIterator();
17 while (it.next()) |block| block.deinit(self.gpa);
18 self.blocks.deinit(self.gpa);
19 }
20 {
21 var it = self.loops.valueIterator();
22 while (it.next()) |block| block.deinit(self.gpa);
23 self.loops.deinit(self.gpa);
24 }
1725 self.* = undefined;
1826}
1927
2028pub fn verify(self: *Verify) Error!void {
2129 self.live.clearRetainingCapacity();
2230 self.blocks.clearRetainingCapacity();
31 self.loops.clearRetainingCapacity();
2332 try self.verifyBody(self.air.getMainBody());
2433 // We don't care about `self.live` now, because the loop body was noreturn - everything being dead was checked on `ret` etc
2534 assert(self.blocks.count() == 0);
35 assert(self.loops.count() == 0);
2636}
2737
2838const LiveMap = std.AutoHashMapUnmanaged(Air.Inst.Index, void);
......@@ -430,6 +440,13 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
430440 }
431441 try self.verifyInst(inst);
432442 },
443 .repeat => {
444 const repeat = data[@intFromEnum(inst)].repeat;
445 const expected_live = self.loops.get(repeat.loop_inst) orelse
446 return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(repeat.loop_inst) });
447
448 try self.verifyMatchingLiveness(repeat.loop_inst, expected_live);
449 },
433450 .block, .dbg_inline_block => |tag| {
434451 const ty_pl = data[@intFromEnum(inst)].ty_pl;
435452 const block_ty = ty_pl.ty.toType();
......@@ -475,14 +492,17 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
475492 const extra = self.air.extraData(Air.Block, ty_pl.payload);
476493 const loop_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
477494
478 var live = try self.live.clone(self.gpa);
479 defer live.deinit(self.gpa);
495 // The same stuff should be alive after the loop as before it.
496 const gop = try self.loops.getOrPut(self.gpa, inst);
497 defer {
498 var live = self.loops.fetchRemove(inst).?;
499 live.value.deinit(self.gpa);
500 }
501 if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)});
502 gop.value_ptr.* = try self.live.clone(self.gpa);
480503
481504 try self.verifyBody(loop_body);
482505
483 // The same stuff should be alive after the loop as before it
484 try self.verifyMatchingLiveness(inst, live);
485
486506 try self.verifyInstOperands(inst, .{ .none, .none, .none });
487507 },
488508 .cond_br => {
src/Sema.zig+17-2
......@@ -1559,6 +1559,8 @@ fn analyzeBodyInner(
15591559 // We are definitely called by `zirLoop`, which will treat the
15601560 // fact that this body does not terminate `noreturn` as an
15611561 // implicit repeat.
1562 // TODO: since AIR has `repeat` now, we could change ZIR to generate
1563 // more optimal code utilizing `repeat` instructions across blocks!
15621564 break;
15631565 }
15641566 },
......@@ -5811,17 +5813,30 @@ fn zirLoop(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileError
58115813 // Use `analyzeBodyInner` directly to push any comptime control flow up the stack.
58125814 try sema.analyzeBodyInner(&loop_block, body);
58135815
5816 // TODO: since AIR has `repeat` now, we could change ZIR to generate
5817 // more optimal code utilizing `repeat` instructions across blocks!
5818 // For now, if the generated loop body does not terminate `noreturn`,
5819 // then `analyzeBodyInner` is signalling that it ended with `repeat`.
5820
58145821 const loop_block_len = loop_block.instructions.items.len;
58155822 if (loop_block_len > 0 and sema.typeOf(loop_block.instructions.items[loop_block_len - 1].toRef()).isNoReturn(zcu)) {
58165823 // If the loop ended with a noreturn terminator, then there is no way for it to loop,
58175824 // so we can just use the block instead.
58185825 try child_block.instructions.appendSlice(gpa, loop_block.instructions.items);
58195826 } else {
5827 _ = try loop_block.addInst(.{
5828 .tag = .repeat,
5829 .data = .{ .repeat = .{
5830 .loop_inst = loop_inst,
5831 } },
5832 });
5833 // Note that `loop_block_len` is now off by one.
5834
58205835 try child_block.instructions.append(gpa, loop_inst);
58215836
5822 try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.Block).@"struct".fields.len + loop_block_len);
5837 try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.Block).@"struct".fields.len + loop_block_len + 1);
58235838 sema.air_instructions.items(.data)[@intFromEnum(loop_inst)].ty_pl.payload = sema.addExtraAssumeCapacity(
5824 Air.Block{ .body_len = @intCast(loop_block_len) },
5839 Air.Block{ .body_len = @intCast(loop_block_len + 1) },
58255840 );
58265841 sema.air_extra.appendSliceAssumeCapacity(@ptrCast(loop_block.instructions.items));
58275842 }
src/arch/aarch64/CodeGen.zig+1
......@@ -734,6 +734,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
734734 .bitcast => try self.airBitCast(inst),
735735 .block => try self.airBlock(inst),
736736 .br => try self.airBr(inst),
737 .repeat => return self.fail("TODO implement `repeat`", .{}),
737738 .trap => try self.airTrap(),
738739 .breakpoint => try self.airBreakpoint(),
739740 .ret_addr => try self.airRetAddr(inst),
src/arch/arm/CodeGen.zig+1
......@@ -721,6 +721,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
721721 .bitcast => try self.airBitCast(inst),
722722 .block => try self.airBlock(inst),
723723 .br => try self.airBr(inst),
724 .repeat => return self.fail("TODO implement `repeat`", .{}),
724725 .trap => try self.airTrap(),
725726 .breakpoint => try self.airBreakpoint(),
726727 .ret_addr => try self.airRetAddr(inst),
src/arch/riscv64/CodeGen.zig+1
......@@ -1579,6 +1579,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
15791579 .bitcast => try func.airBitCast(inst),
15801580 .block => try func.airBlock(inst),
15811581 .br => try func.airBr(inst),
1582 .repeat => return func.fail("TODO implement `repeat`", .{}),
15821583 .trap => try func.airTrap(),
15831584 .breakpoint => try func.airBreakpoint(),
15841585 .ret_addr => try func.airRetAddr(inst),
src/arch/sparc64/CodeGen.zig+1
......@@ -576,6 +576,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
576576 .bitcast => try self.airBitCast(inst),
577577 .block => try self.airBlock(inst),
578578 .br => try self.airBr(inst),
579 .repeat => return self.fail("TODO implement `repeat`", .{}),
579580 .trap => try self.airTrap(),
580581 .breakpoint => try self.airBreakpoint(),
581582 .ret_addr => @panic("TODO try self.airRetAddr(inst)"),
src/arch/wasm/CodeGen.zig+1
......@@ -1903,6 +1903,7 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
19031903 .trap => func.airTrap(inst),
19041904 .breakpoint => func.airBreakpoint(inst),
19051905 .br => func.airBr(inst),
1906 .repeat => return func.fail("TODO implement `repeat`", .{}),
19061907 .int_from_bool => func.airIntFromBool(inst),
19071908 .cond_br => func.airCondBr(inst),
19081909 .intcast => func.airIntcast(inst),
src/arch/x86_64/CodeGen.zig+1
......@@ -2247,6 +2247,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
22472247 .bitcast => try self.airBitCast(inst),
22482248 .block => try self.airBlock(inst),
22492249 .br => try self.airBr(inst),
2250 .repeat => return self.fail("TODO implement `repeat`", .{}),
22502251 .trap => try self.airTrap(),
22512252 .breakpoint => try self.airBreakpoint(),
22522253 .ret_addr => try self.airRetAddr(inst),
src/codegen/c.zig+55-39
......@@ -3137,11 +3137,9 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
31373137
31383138 .arg => try airArg(f, inst),
31393139
3140 .trap => try airTrap(f, f.object.writer()),
31413140 .breakpoint => try airBreakpoint(f.object.writer()),
31423141 .ret_addr => try airRetAddr(f, inst),
31433142 .frame_addr => try airFrameAddress(f, inst),
3144 .unreach => try airUnreach(f),
31453143 .fence => try airFence(f, inst),
31463144
31473145 .ptr_add => try airPtrAddSub(f, inst, '+'),
......@@ -3248,21 +3246,13 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
32483246 .alloc => try airAlloc(f, inst),
32493247 .ret_ptr => try airRetPtr(f, inst),
32503248 .assembly => try airAsm(f, inst),
3251 .block => try airBlock(f, inst),
32523249 .bitcast => try airBitcast(f, inst),
32533250 .intcast => try airIntCast(f, inst),
32543251 .trunc => try airTrunc(f, inst),
32553252 .int_from_bool => try airIntFromBool(f, inst),
32563253 .load => try airLoad(f, inst),
3257 .ret => try airRet(f, inst, false),
3258 .ret_safe => try airRet(f, inst, false), // TODO
3259 .ret_load => try airRet(f, inst, true),
32603254 .store => try airStore(f, inst, false),
32613255 .store_safe => try airStore(f, inst, true),
3262 .loop => try airLoop(f, inst),
3263 .cond_br => try airCondBr(f, inst),
3264 .br => try airBr(f, inst),
3265 .switch_br => try airSwitchBr(f, inst),
32663256 .struct_field_ptr => try airStructFieldPtr(f, inst),
32673257 .array_to_slice => try airArrayToSlice(f, inst),
32683258 .cmpxchg_weak => try airCmpxchg(f, inst, "weak"),
......@@ -3296,14 +3286,8 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
32963286 .try_ptr_cold => try airTryPtr(f, inst),
32973287
32983288 .dbg_stmt => try airDbgStmt(f, inst),
3299 .dbg_inline_block => try airDbgInlineBlock(f, inst),
33003289 .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => try airDbgVar(f, inst),
33013290
3302 .call => try airCall(f, inst, .auto),
3303 .call_always_tail => .none,
3304 .call_never_tail => try airCall(f, inst, .never_tail),
3305 .call_never_inline => try airCall(f, inst, .never_inline),
3306
33073291 .float_from_int,
33083292 .int_from_float,
33093293 .fptrunc,
......@@ -3390,6 +3374,39 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
33903374 .work_group_size,
33913375 .work_group_id,
33923376 => unreachable,
3377
3378 // Instructions that are known to always be `noreturn` based on their tag.
3379 .br => return airBr(f, inst),
3380 .repeat => return airRepeat(f, inst),
3381 .cond_br => return airCondBr(f, inst),
3382 .switch_br => return airSwitchBr(f, inst),
3383 .loop => return airLoop(f, inst),
3384 .ret => return airRet(f, inst, false),
3385 .ret_safe => return airRet(f, inst, false), // TODO
3386 .ret_load => return airRet(f, inst, true),
3387 .trap => return airTrap(f, f.object.writer()),
3388 .unreach => return airUnreach(f),
3389
3390 // Instructions which may be `noreturn`.
3391 .block => res: {
3392 const res = try airBlock(f, inst);
3393 if (f.typeOfIndex(inst).isNoReturn(zcu)) return;
3394 break :res res;
3395 },
3396 .dbg_inline_block => res: {
3397 const res = try airDbgInlineBlock(f, inst);
3398 if (f.typeOfIndex(inst).isNoReturn(zcu)) return;
3399 break :res res;
3400 },
3401 // TODO: calls should be in this category! The AIR we emit for them is a bit weird.
3402 // The instruction has type `noreturn`, but there are instructions (and maybe a safety
3403 // check) following nonetheless. The `unreachable` or safety check should be emitted by
3404 // backends instead.
3405 .call => try airCall(f, inst, .auto),
3406 .call_always_tail => .none,
3407 .call_never_tail => try airCall(f, inst, .never_tail),
3408 .call_never_inline => try airCall(f, inst, .never_inline),
3409
33933410 // zig fmt: on
33943411 };
33953412 if (result_value == .new_local) {
......@@ -3401,6 +3418,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
34013418 else => result_value,
34023419 });
34033420 }
3421 unreachable;
34043422}
34053423
34063424fn airSliceField(f: *Function, inst: Air.Inst.Index, is_ptr: bool, field_name: []const u8) !CValue {
......@@ -3718,7 +3736,7 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
37183736 return local;
37193737}
37203738
3721fn airRet(f: *Function, inst: Air.Inst.Index, is_ptr: bool) !CValue {
3739fn airRet(f: *Function, inst: Air.Inst.Index, is_ptr: bool) !void {
37223740 const pt = f.object.dg.pt;
37233741 const zcu = pt.zcu;
37243742 const un_op = f.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
......@@ -3769,7 +3787,6 @@ fn airRet(f: *Function, inst: Air.Inst.Index, is_ptr: bool) !CValue {
37693787 // Not even allowed to return void in a naked function.
37703788 if (!f.object.dg.is_naked_fn) try writer.writeAll("return;\n");
37713789 }
3772 return .none;
37733790}
37743791
37753792fn airIntCast(f: *Function, inst: Air.Inst.Index) !CValue {
......@@ -4741,7 +4758,7 @@ fn lowerTry(
47414758 return local;
47424759}
47434760
4744fn airBr(f: *Function, inst: Air.Inst.Index) !CValue {
4761fn airBr(f: *Function, inst: Air.Inst.Index) !void {
47454762 const branch = f.air.instructions.items(.data)[@intFromEnum(inst)].br;
47464763 const block = f.blocks.get(branch.block_inst).?;
47474764 const result = block.result;
......@@ -4761,7 +4778,12 @@ fn airBr(f: *Function, inst: Air.Inst.Index) !CValue {
47614778 }
47624779
47634780 try writer.print("goto zig_block_{d};\n", .{block.block_id});
4764 return .none;
4781}
4782
4783fn airRepeat(f: *Function, inst: Air.Inst.Index) !void {
4784 const repeat = f.air.instructions.items(.data)[@intFromEnum(inst)].repeat;
4785 const writer = f.object.writer();
4786 try writer.print("goto zig_loop_{d};\n", .{@intFromEnum(repeat.loop_inst)});
47654787}
47664788
47674789fn airBitcast(f: *Function, inst: Air.Inst.Index) !CValue {
......@@ -4889,12 +4911,10 @@ fn bitcast(f: *Function, dest_ty: Type, operand: CValue, operand_ty: Type) !CVal
48894911 return local;
48904912}
48914913
4892fn airTrap(f: *Function, writer: anytype) !CValue {
4914fn airTrap(f: *Function, writer: anytype) !void {
48934915 // Not even allowed to call trap in a naked function.
4894 if (f.object.dg.is_naked_fn) return .none;
4895
4916 if (f.object.dg.is_naked_fn) return;
48964917 try writer.writeAll("zig_trap();\n");
4897 return .none;
48984918}
48994919
49004920fn airBreakpoint(writer: anytype) !CValue {
......@@ -4933,28 +4953,27 @@ fn airFence(f: *Function, inst: Air.Inst.Index) !CValue {
49334953 return .none;
49344954}
49354955
4936fn airUnreach(f: *Function) !CValue {
4956fn airUnreach(f: *Function) !void {
49374957 // Not even allowed to call unreachable in a naked function.
4938 if (f.object.dg.is_naked_fn) return .none;
4939
4958 if (f.object.dg.is_naked_fn) return;
49404959 try f.object.writer().writeAll("zig_unreachable();\n");
4941 return .none;
49424960}
49434961
4944fn airLoop(f: *Function, inst: Air.Inst.Index) !CValue {
4962fn airLoop(f: *Function, inst: Air.Inst.Index) !void {
49454963 const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
49464964 const loop = f.air.extraData(Air.Block, ty_pl.payload);
49474965 const body: []const Air.Inst.Index = @ptrCast(f.air.extra[loop.end..][0..loop.data.body_len]);
49484966 const writer = f.object.writer();
49494967
4950 try writer.writeAll("for (;;) ");
4951 try genBody(f, body); // no need to restore state, we're noreturn
4952 try writer.writeByte('\n');
4953
4954 return .none;
4968 // `repeat` instructions matching this loop will branch to
4969 // this label. Since we need a label for arbitrary `repeat`
4970 // anyway, there's actually no need to use a "real" looping
4971 // construct at all!
4972 try writer.print("zig_loop_{d}:\n", .{@intFromEnum(inst)});
4973 try genBodyInner(f, body); // no need to restore state, we're noreturn
49554974}
49564975
4957fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {
4976fn airCondBr(f: *Function, inst: Air.Inst.Index) !void {
49584977 const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
49594978 const cond = try f.resolveInst(pl_op.operand);
49604979 try reap(f, inst, &.{pl_op.operand});
......@@ -4983,11 +5002,9 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {
49835002 // instance) `br` to a block (label).
49845003
49855004 try genBodyInner(f, else_body);
4986
4987 return .none;
49885005}
49895006
4990fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
5007fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !void {
49915008 const pt = f.object.dg.pt;
49925009 const zcu = pt.zcu;
49935010 const switch_br = f.air.unwrapSwitch(inst);
......@@ -5097,7 +5114,6 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
50975114
50985115 f.object.indent_writer.popIndent();
50995116 try writer.writeAll("}\n");
5100 return .none;
51015117}
51025118
51035119fn asmInputNeedsLocal(f: *Function, constraint: []const u8, value: CValue) bool {
src/codegen/llvm.zig+75-54
......@@ -1720,6 +1720,7 @@ pub const Object = struct {
17201720 .arg_inline_index = 0,
17211721 .func_inst_table = .{},
17221722 .blocks = .{},
1723 .loops = .{},
17231724 .sync_scope = if (owner_mod.single_threaded) .singlethread else .system,
17241725 .file = file,
17251726 .scope = subprogram,
......@@ -4841,6 +4842,9 @@ pub const FuncGen = struct {
48414842 breaks: *BreakList,
48424843 }),
48434844
4845 /// Maps `loop` instructions to the bb to branch to to repeat the loop.
4846 loops: std.AutoHashMapUnmanaged(Air.Inst.Index, Builder.Function.Block.Index),
4847
48444848 sync_scope: Builder.SyncScope,
48454849
48464850 const Fuzz = struct {
......@@ -4867,6 +4871,7 @@ pub const FuncGen = struct {
48674871 self.wip.deinit();
48684872 self.func_inst_table.deinit(gpa);
48694873 self.blocks.deinit(gpa);
4874 self.loops.deinit(gpa);
48704875 }
48714876
48724877 fn todo(self: *FuncGen, comptime format: []const u8, args: anytype) Error {
......@@ -5058,14 +5063,9 @@ pub const FuncGen = struct {
50585063 .arg => try self.airArg(inst),
50595064 .bitcast => try self.airBitCast(inst),
50605065 .int_from_bool => try self.airIntFromBool(inst),
5061 .block => try self.airBlock(inst),
5062 .br => try self.airBr(inst),
5063 .switch_br => try self.airSwitchBr(inst),
5064 .trap => try self.airTrap(inst),
50655066 .breakpoint => try self.airBreakpoint(inst),
50665067 .ret_addr => try self.airRetAddr(inst),
50675068 .frame_addr => try self.airFrameAddress(inst),
5068 .cond_br => try self.airCondBr(inst),
50695069 .@"try" => try self.airTry(body[i..], false),
50705070 .try_cold => try self.airTry(body[i..], true),
50715071 .try_ptr => try self.airTryPtr(inst, false),
......@@ -5076,22 +5076,13 @@ pub const FuncGen = struct {
50765076 .fpext => try self.airFpext(inst),
50775077 .int_from_ptr => try self.airIntFromPtr(inst),
50785078 .load => try self.airLoad(body[i..]),
5079 .loop => try self.airLoop(inst),
50805079 .not => try self.airNot(inst),
5081 .ret => try self.airRet(inst, false),
5082 .ret_safe => try self.airRet(inst, true),
5083 .ret_load => try self.airRetLoad(inst),
50845080 .store => try self.airStore(inst, false),
50855081 .store_safe => try self.airStore(inst, true),
50865082 .assembly => try self.airAssembly(inst),
50875083 .slice_ptr => try self.airSliceField(inst, 0),
50885084 .slice_len => try self.airSliceField(inst, 1),
50895085
5090 .call => try self.airCall(inst, .auto),
5091 .call_always_tail => try self.airCall(inst, .always_tail),
5092 .call_never_tail => try self.airCall(inst, .never_tail),
5093 .call_never_inline => try self.airCall(inst, .never_inline),
5094
50955086 .ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),
50965087 .ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),
50975088
......@@ -5176,9 +5167,7 @@ pub const FuncGen = struct {
51765167
51775168 .inferred_alloc, .inferred_alloc_comptime => unreachable,
51785169
5179 .unreach => try self.airUnreach(inst),
51805170 .dbg_stmt => try self.airDbgStmt(inst),
5181 .dbg_inline_block => try self.airDbgInlineBlock(inst),
51825171 .dbg_var_ptr => try self.airDbgVarPtr(inst),
51835172 .dbg_var_val => try self.airDbgVarVal(inst, false),
51845173 .dbg_arg_inline => try self.airDbgVarVal(inst, true),
......@@ -5191,10 +5180,50 @@ pub const FuncGen = struct {
51915180 .work_item_id => try self.airWorkItemId(inst),
51925181 .work_group_size => try self.airWorkGroupSize(inst),
51935182 .work_group_id => try self.airWorkGroupId(inst),
5183
5184 // Instructions that are known to always be `noreturn` based on their tag.
5185 .br => return self.airBr(inst),
5186 .repeat => return self.airRepeat(inst),
5187 .cond_br => return self.airCondBr(inst),
5188 .switch_br => return self.airSwitchBr(inst),
5189 .loop => return self.airLoop(inst),
5190 .ret => return self.airRet(inst, false),
5191 .ret_safe => return self.airRet(inst, true),
5192 .ret_load => return self.airRetLoad(inst),
5193 .trap => return self.airTrap(inst),
5194 .unreach => return self.airUnreach(inst),
5195
5196 // Instructions which may be `noreturn`.
5197 .block => res: {
5198 const res = try self.airBlock(inst);
5199 if (self.typeOfIndex(inst).isNoReturn(zcu)) return;
5200 break :res res;
5201 },
5202 .dbg_inline_block => res: {
5203 const res = try self.airDbgInlineBlock(inst);
5204 if (self.typeOfIndex(inst).isNoReturn(zcu)) return;
5205 break :res res;
5206 },
5207 .call, .call_always_tail, .call_never_tail, .call_never_inline => |tag| res: {
5208 const res = try self.airCall(inst, switch (tag) {
5209 .call => .auto,
5210 .call_always_tail => .always_tail,
5211 .call_never_tail => .never_tail,
5212 .call_never_inline => .never_inline,
5213 else => unreachable,
5214 });
5215 // TODO: the AIR we emit for calls is a bit weird - the instruction has
5216 // type `noreturn`, but there are instructions (and maybe a safety check) following
5217 // nonetheless. The `unreachable` or safety check should be emitted by backends instead.
5218 //if (self.typeOfIndex(inst).isNoReturn(mod)) return;
5219 break :res res;
5220 },
5221
51945222 // zig fmt: on
51955223 };
51965224 if (val != .none) try self.func_inst_table.putNoClobber(self.gpa, inst.toRef(), val);
51975225 }
5226 unreachable;
51985227 }
51995228
52005229 fn genBodyDebugScope(
......@@ -5640,7 +5669,7 @@ pub const FuncGen = struct {
56405669 _ = try fg.wip.@"unreachable"();
56415670 }
56425671
5643 fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
5672 fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void {
56445673 const o = self.ng.object;
56455674 const pt = o.pt;
56465675 const zcu = pt.zcu;
......@@ -5675,7 +5704,7 @@ pub const FuncGen = struct {
56755704 try self.valgrindMarkUndef(self.ret_ptr, len);
56765705 }
56775706 _ = try self.wip.retVoid();
5678 return .none;
5707 return;
56795708 }
56805709
56815710 const unwrapped_operand = operand.unwrap();
......@@ -5684,12 +5713,12 @@ pub const FuncGen = struct {
56845713 // Return value was stored previously
56855714 if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) {
56865715 _ = try self.wip.retVoid();
5687 return .none;
5716 return;
56885717 }
56895718
56905719 try self.store(self.ret_ptr, ptr_ty, operand, .none);
56915720 _ = try self.wip.retVoid();
5692 return .none;
5721 return;
56935722 }
56945723 const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.ng.nav_index).typeOf(ip))).?;
56955724 if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
......@@ -5701,7 +5730,7 @@ pub const FuncGen = struct {
57015730 } else {
57025731 _ = try self.wip.retVoid();
57035732 }
5704 return .none;
5733 return;
57055734 }
57065735
57075736 const abi_ret_ty = try lowerFnRetTy(o, fn_info);
......@@ -5725,29 +5754,29 @@ pub const FuncGen = struct {
57255754 try self.valgrindMarkUndef(rp, len);
57265755 }
57275756 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
5728 return .none;
5757 return;
57295758 }
57305759
57315760 if (isByRef(ret_ty, zcu)) {
57325761 // operand is a pointer however self.ret_ptr is null so that means
57335762 // we need to return a value.
57345763 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, ""));
5735 return .none;
5764 return;
57365765 }
57375766
57385767 const llvm_ret_ty = operand.typeOfWip(&self.wip);
57395768 if (abi_ret_ty == llvm_ret_ty) {
57405769 _ = try self.wip.ret(operand);
5741 return .none;
5770 return;
57425771 }
57435772
57445773 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
57455774 _ = try self.wip.store(.normal, operand, rp, alignment);
57465775 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
5747 return .none;
5776 return;
57485777 }
57495778
5750 fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5779 fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !void {
57515780 const o = self.ng.object;
57525781 const pt = o.pt;
57535782 const zcu = pt.zcu;
......@@ -5765,17 +5794,17 @@ pub const FuncGen = struct {
57655794 } else {
57665795 _ = try self.wip.retVoid();
57675796 }
5768 return .none;
5797 return;
57695798 }
57705799 if (self.ret_ptr != .none) {
57715800 _ = try self.wip.retVoid();
5772 return .none;
5801 return;
57735802 }
57745803 const ptr = try self.resolveInst(un_op);
57755804 const abi_ret_ty = try lowerFnRetTy(o, fn_info);
57765805 const alignment = ret_ty.abiAlignment(zcu).toLlvm();
57775806 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, ""));
5778 return .none;
5807 return;
57795808 }
57805809
57815810 fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -6039,7 +6068,7 @@ pub const FuncGen = struct {
60396068 }
60406069 }
60416070
6042 fn airBr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6071 fn airBr(self: *FuncGen, inst: Air.Inst.Index) !void {
60436072 const o = self.ng.object;
60446073 const zcu = o.pt.zcu;
60456074 const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
......@@ -6055,10 +6084,16 @@ pub const FuncGen = struct {
60556084 try block.breaks.list.append(self.gpa, .{ .bb = self.wip.cursor.block, .val = val });
60566085 } else block.breaks.len += 1;
60576086 _ = try self.wip.br(block.parent_bb);
6058 return .none;
60596087 }
60606088
6061 fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6089 fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) !void {
6090 const repeat = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat;
6091 const loop_bb = self.loops.get(repeat.loop_inst).?;
6092 loop_bb.ptr(&self.wip).incoming += 1;
6093 _ = try self.wip.br(loop_bb);
6094 }
6095
6096 fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !void {
60626097 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
60636098 const cond = try self.resolveInst(pl_op.operand);
60646099 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
......@@ -6117,7 +6152,6 @@ pub const FuncGen = struct {
61176152 try self.genBodyDebugScope(null, else_body, extra.data.branch_hints.else_cov);
61186153
61196154 // No need to reset the insert cursor since this instruction is noreturn.
6120 return .none;
61216155 }
61226156
61236157 fn airTry(self: *FuncGen, body_tail: []const Air.Inst.Index, err_cold: bool) !Builder.Value {
......@@ -6223,7 +6257,7 @@ pub const FuncGen = struct {
62236257 return fg.wip.extractValue(err_union, &.{offset}, "");
62246258 }
62256259
6226 fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6260 fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index) !void {
62276261 const o = self.ng.object;
62286262
62296263 const switch_br = self.air.unwrapSwitch(inst);
......@@ -6371,31 +6405,20 @@ pub const FuncGen = struct {
63716405 }
63726406
63736407 // No need to reset the insert cursor since this instruction is noreturn.
6374 return .none;
63756408 }
63766409
6377 fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6378 const o = self.ng.object;
6379 const zcu = o.pt.zcu;
6410 fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void {
63806411 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
63816412 const loop = self.air.extraData(Air.Block, ty_pl.payload);
63826413 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);
6383 const loop_block = try self.wip.block(2, "Loop");
6414 const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time
63846415 _ = try self.wip.br(loop_block);
63856416
6417 try self.loops.putNoClobber(self.gpa, inst, loop_block);
6418 defer assert(self.loops.remove(inst));
6419
63866420 self.wip.cursor = .{ .block = loop_block };
63876421 try self.genBodyDebugScope(null, body, .none);
6388
6389 // TODO instead of this logic, change AIR to have the property that
6390 // every block is guaranteed to end with a noreturn instruction.
6391 // Then we can simply rely on the fact that a repeat or break instruction
6392 // would have been emitted already. Also the main loop in genBody can
6393 // be while(true) instead of for(body), which will eliminate 1 branch on
6394 // a hot path.
6395 if (body.len == 0 or !self.typeOfIndex(body[body.len - 1]).isNoReturn(zcu)) {
6396 _ = try self.wip.br(loop_block);
6397 }
6398 return .none;
63996422 }
64006423
64016424 fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -6890,10 +6913,9 @@ pub const FuncGen = struct {
68906913 return self.wip.not(operand, "");
68916914 }
68926915
6893 fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6916 fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) !void {
68946917 _ = inst;
68956918 _ = try self.wip.@"unreachable"();
6896 return .none;
68976919 }
68986920
68996921 fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -9296,11 +9318,10 @@ pub const FuncGen = struct {
92969318 return fg.load(ptr, ptr_ty);
92979319 }
92989320
9299 fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9321 fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !void {
93009322 _ = inst;
93019323 _ = try self.wip.callIntrinsic(.normal, .none, .trap, &.{}, &.{}, "");
93029324 _ = try self.wip.@"unreachable"();
9303 return .none;
93049325 }
93059326
93069327 fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
src/codegen/spirv.zig+1
......@@ -3340,6 +3340,7 @@ const NavGen = struct {
33403340 .store, .store_safe => return self.airStore(inst),
33413341
33423342 .br => return self.airBr(inst),
3343 .repeat => return self.fail("TODO implement `repeat`", .{}),
33433344 .breakpoint => return,
33443345 .cond_br => return self.airCondBr(inst),
33453346 .loop => return self.airLoop(inst),
src/print_air.zig+6
......@@ -296,6 +296,7 @@ const Writer = struct {
296296 .aggregate_init => try w.writeAggregateInit(s, inst),
297297 .union_init => try w.writeUnionInit(s, inst),
298298 .br => try w.writeBr(s, inst),
299 .repeat => try w.writeRepeat(s, inst),
299300 .cond_br => try w.writeCondBr(s, inst),
300301 .@"try", .try_cold => try w.writeTry(s, inst),
301302 .try_ptr, .try_ptr_cold => try w.writeTryPtr(s, inst),
......@@ -708,6 +709,11 @@ const Writer = struct {
708709 try w.writeOperand(s, inst, 0, br.operand);
709710 }
710711
712 fn writeRepeat(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
713 const repeat = w.air.instructions.items(.data)[@intFromEnum(inst)].repeat;
714 try w.writeInstIndex(s, repeat.loop_inst, false);
715 }
716
711717 fn writeTry(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
712718 const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
713719 const extra = w.air.extraData(Air.Try, pl_op.payload);