authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-09-02 23:10:18-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-09-20 18:33:00-07:00
log14bda4130a9a7f8b529b12bc74a1d6caa71b9545
treea60bcdc73299b869caf643d29d9b146b2ae09a03
parent3fbb88c4bd146ca7bd9e7ab5da9c4b05298f3b34

llvm backend: remove canElideLoad mechanism

2 files changed, 21 insertions(+), 576 deletions(-)

src/Air/Liveness.zig-495
......@@ -207,501 +207,6 @@ pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool
207207 return (l.tomb_bits[usize_index] & mask) != 0;
208208}
209209
210const OperandCategory = enum {
211 /// The operand lives on, but this instruction cannot possibly mutate memory.
212 none,
213 /// The operand lives on and this instruction can mutate memory.
214 write,
215 /// The operand dies at this instruction.
216 tomb,
217 /// The operand lives on, and this instruction is noreturn.
218 noret,
219 /// This instruction is too complicated for analysis, no information is available.
220 complex,
221};
222
223/// Given an instruction that we are examining, and an operand that we are looking for,
224/// returns a classification.
225pub fn categorizeOperand(
226 l: Liveness,
227 air: Air,
228 zcu: *Zcu,
229 inst: Air.Inst.Index,
230 operand: Air.Inst.Index,
231 ip: *const InternPool,
232) OperandCategory {
233 const air_tags = air.instructions.items(.tag);
234 const air_datas = air.instructions.items(.data);
235 const operand_ref = operand.toRef();
236 switch (air_tags[@intFromEnum(inst)]) {
237 .add,
238 .add_safe,
239 .add_wrap,
240 .add_sat,
241 .add_optimized,
242 .sub,
243 .sub_safe,
244 .sub_wrap,
245 .sub_sat,
246 .sub_optimized,
247 .mul,
248 .mul_safe,
249 .mul_wrap,
250 .mul_sat,
251 .mul_optimized,
252 .div_float,
253 .div_trunc,
254 .div_floor,
255 .div_exact,
256 .rem,
257 .mod,
258 .bit_and,
259 .bit_or,
260 .xor,
261 .cmp_lt,
262 .cmp_lte,
263 .cmp_eq,
264 .cmp_gte,
265 .cmp_gt,
266 .cmp_neq,
267 .bool_and,
268 .bool_or,
269 .array_elem_val,
270 .slice_elem_val,
271 .ptr_elem_val,
272 .shl,
273 .shl_exact,
274 .shl_sat,
275 .shr,
276 .shr_exact,
277 .min,
278 .max,
279 .div_float_optimized,
280 .div_trunc_optimized,
281 .div_floor_optimized,
282 .div_exact_optimized,
283 .rem_optimized,
284 .mod_optimized,
285 .neg_optimized,
286 .cmp_lt_optimized,
287 .cmp_lte_optimized,
288 .cmp_eq_optimized,
289 .cmp_gte_optimized,
290 .cmp_gt_optimized,
291 .cmp_neq_optimized,
292 => {
293 const o = air_datas[@intFromEnum(inst)].bin_op;
294 if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
295 if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
296 return .none;
297 },
298
299 .store,
300 .store_safe,
301 .atomic_store_unordered,
302 .atomic_store_monotonic,
303 .atomic_store_release,
304 .atomic_store_seq_cst,
305 .set_union_tag,
306 .memset,
307 .memset_safe,
308 .memcpy,
309 .memmove,
310 => {
311 const o = air_datas[@intFromEnum(inst)].bin_op;
312 if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
313 if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
314 return .write;
315 },
316
317 .vector_store_elem => {
318 const o = air_datas[@intFromEnum(inst)].vector_store_elem;
319 const extra = air.extraData(Air.Bin, o.payload).data;
320 if (o.vector_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
321 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
322 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
323 return .write;
324 },
325
326 .arg,
327 .alloc,
328 .inferred_alloc,
329 .inferred_alloc_comptime,
330 .ret_ptr,
331 .trap,
332 .breakpoint,
333 .repeat,
334 .switch_dispatch,
335 .dbg_stmt,
336 .dbg_empty_stmt,
337 .unreach,
338 .ret_addr,
339 .frame_addr,
340 .wasm_memory_size,
341 .err_return_trace,
342 .save_err_return_trace_index,
343 .runtime_nav_ptr,
344 .c_va_start,
345 .work_item_id,
346 .work_group_size,
347 .work_group_id,
348 => return .none,
349
350 .not,
351 .bitcast,
352 .load,
353 .fpext,
354 .fptrunc,
355 .intcast,
356 .intcast_safe,
357 .trunc,
358 .optional_payload,
359 .optional_payload_ptr,
360 .wrap_optional,
361 .unwrap_errunion_payload,
362 .unwrap_errunion_err,
363 .unwrap_errunion_payload_ptr,
364 .unwrap_errunion_err_ptr,
365 .wrap_errunion_payload,
366 .wrap_errunion_err,
367 .slice_ptr,
368 .slice_len,
369 .ptr_slice_len_ptr,
370 .ptr_slice_ptr_ptr,
371 .struct_field_ptr_index_0,
372 .struct_field_ptr_index_1,
373 .struct_field_ptr_index_2,
374 .struct_field_ptr_index_3,
375 .array_to_slice,
376 .int_from_float,
377 .int_from_float_optimized,
378 .int_from_float_safe,
379 .int_from_float_optimized_safe,
380 .float_from_int,
381 .get_union_tag,
382 .clz,
383 .ctz,
384 .popcount,
385 .byte_swap,
386 .bit_reverse,
387 .splat,
388 .error_set_has_value,
389 .addrspace_cast,
390 .c_va_arg,
391 .c_va_copy,
392 .abs,
393 => {
394 const o = air_datas[@intFromEnum(inst)].ty_op;
395 if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
396 return .none;
397 },
398
399 .optional_payload_ptr_set,
400 .errunion_payload_ptr_set,
401 => {
402 const o = air_datas[@intFromEnum(inst)].ty_op;
403 if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
404 return .write;
405 },
406
407 .is_null,
408 .is_non_null,
409 .is_null_ptr,
410 .is_non_null_ptr,
411 .is_err,
412 .is_non_err,
413 .is_err_ptr,
414 .is_non_err_ptr,
415 .is_named_enum_value,
416 .tag_name,
417 .error_name,
418 .sqrt,
419 .sin,
420 .cos,
421 .tan,
422 .exp,
423 .exp2,
424 .log,
425 .log2,
426 .log10,
427 .floor,
428 .ceil,
429 .round,
430 .trunc_float,
431 .neg,
432 .cmp_lt_errors_len,
433 .c_va_end,
434 => {
435 const o = air_datas[@intFromEnum(inst)].un_op;
436 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
437 return .none;
438 },
439
440 .ret,
441 .ret_safe,
442 .ret_load,
443 => {
444 const o = air_datas[@intFromEnum(inst)].un_op;
445 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .noret);
446 return .noret;
447 },
448
449 .set_err_return_trace => {
450 const o = air_datas[@intFromEnum(inst)].un_op;
451 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
452 return .write;
453 },
454
455 .add_with_overflow,
456 .sub_with_overflow,
457 .mul_with_overflow,
458 .shl_with_overflow,
459 .ptr_add,
460 .ptr_sub,
461 .ptr_elem_ptr,
462 .slice_elem_ptr,
463 .slice,
464 => {
465 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
466 const extra = air.extraData(Air.Bin, ty_pl.payload).data;
467 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
468 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
469 return .none;
470 },
471
472 .dbg_var_ptr,
473 .dbg_var_val,
474 .dbg_arg_inline,
475 => {
476 const o = air_datas[@intFromEnum(inst)].pl_op.operand;
477 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
478 return .none;
479 },
480
481 .prefetch => {
482 const prefetch = air_datas[@intFromEnum(inst)].prefetch;
483 if (prefetch.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
484 return .none;
485 },
486
487 .call, .call_always_tail, .call_never_tail, .call_never_inline => {
488 const inst_data = air_datas[@intFromEnum(inst)].pl_op;
489 const callee = inst_data.operand;
490 const extra = air.extraData(Air.Call, inst_data.payload);
491 const args = @as([]const Air.Inst.Ref, @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]));
492 if (args.len + 1 <= bpi - 1) {
493 if (callee == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
494 for (args, 0..) |arg, i| {
495 if (arg == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i + 1)), .write);
496 }
497 return .write;
498 }
499 var bt = l.iterateBigTomb(inst);
500 if (bt.feed()) {
501 if (callee == operand_ref) return .tomb;
502 } else {
503 if (callee == operand_ref) return .write;
504 }
505 for (args) |arg| {
506 if (bt.feed()) {
507 if (arg == operand_ref) return .tomb;
508 } else {
509 if (arg == operand_ref) return .write;
510 }
511 }
512 return .write;
513 },
514 .select => {
515 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
516 const extra = air.extraData(Air.Bin, pl_op.payload).data;
517 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
518 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
519 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
520 return .none;
521 },
522 .shuffle_one => {
523 const unwrapped = air.unwrapShuffleOne(zcu, inst);
524 if (unwrapped.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
525 return .none;
526 },
527 .shuffle_two => {
528 const unwrapped = air.unwrapShuffleTwo(zcu, inst);
529 if (unwrapped.operand_a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
530 if (unwrapped.operand_b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
531 return .none;
532 },
533 .reduce, .reduce_optimized => {
534 const reduce = air_datas[@intFromEnum(inst)].reduce;
535 if (reduce.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
536 return .none;
537 },
538 .cmp_vector, .cmp_vector_optimized => {
539 const extra = air.extraData(Air.VectorCmp, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
540 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
541 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
542 return .none;
543 },
544 .aggregate_init => {
545 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
546 const aggregate_ty = ty_pl.ty.toType();
547 const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
548 const elements = @as([]const Air.Inst.Ref, @ptrCast(air.extra.items[ty_pl.payload..][0..len]));
549
550 if (elements.len <= bpi - 1) {
551 for (elements, 0..) |elem, i| {
552 if (elem == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i)), .none);
553 }
554 return .none;
555 }
556
557 var bt = l.iterateBigTomb(inst);
558 for (elements) |elem| {
559 if (bt.feed()) {
560 if (elem == operand_ref) return .tomb;
561 } else {
562 if (elem == operand_ref) return .write;
563 }
564 }
565 return .write;
566 },
567 .union_init => {
568 const extra = air.extraData(Air.UnionInit, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
569 if (extra.init == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
570 return .none;
571 },
572 .struct_field_ptr, .struct_field_val => {
573 const extra = air.extraData(Air.StructField, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
574 if (extra.struct_operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
575 return .none;
576 },
577 .field_parent_ptr => {
578 const extra = air.extraData(Air.FieldParentPtr, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
579 if (extra.field_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
580 return .none;
581 },
582 .cmpxchg_strong, .cmpxchg_weak => {
583 const extra = air.extraData(Air.Cmpxchg, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
584 if (extra.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
585 if (extra.expected_value == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
586 if (extra.new_value == operand_ref) return matchOperandSmallIndex(l, inst, 2, .write);
587 return .write;
588 },
589 .mul_add => {
590 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
591 const extra = air.extraData(Air.Bin, pl_op.payload).data;
592 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
593 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
594 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
595 return .none;
596 },
597 .atomic_load => {
598 const ptr = air_datas[@intFromEnum(inst)].atomic_load.ptr;
599 if (ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
600 return .none;
601 },
602 .atomic_rmw => {
603 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
604 const extra = air.extraData(Air.AtomicRmw, pl_op.payload).data;
605 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
606 if (extra.operand == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
607 return .write;
608 },
609
610 .br => {
611 const br = air_datas[@intFromEnum(inst)].br;
612 if (br.operand == operand_ref) return matchOperandSmallIndex(l, operand, 0, .noret);
613 return .noret;
614 },
615 .assembly => {
616 return .complex;
617 },
618 .block, .dbg_inline_block => |tag| {
619 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
620 const body: []const Air.Inst.Index = @ptrCast(switch (tag) {
621 inline .block, .dbg_inline_block => |comptime_tag| body: {
622 const extra = air.extraData(switch (comptime_tag) {
623 .block => Air.Block,
624 .dbg_inline_block => Air.DbgInlineBlock,
625 else => unreachable,
626 }, ty_pl.payload);
627 break :body air.extra.items[extra.end..][0..extra.data.body_len];
628 },
629 else => unreachable,
630 });
631
632 if (body.len == 1 and air_tags[@intFromEnum(body[0])] == .cond_br) {
633 // Peephole optimization for "panic-like" conditionals, which have
634 // one empty branch and another which calls a `noreturn` function.
635 // This allows us to infer that safety checks do not modify memory,
636 // as far as control flow successors are concerned.
637
638 const inst_data = air_datas[@intFromEnum(body[0])].pl_op;
639 const cond_extra = air.extraData(Air.CondBr, inst_data.payload);
640 if (inst_data.operand == operand_ref and operandDies(l, body[0], 0))
641 return .tomb;
642
643 if (cond_extra.data.then_body_len > 2 or cond_extra.data.else_body_len > 2)
644 return .complex;
645
646 const then_body: []const Air.Inst.Index = @ptrCast(air.extra.items[cond_extra.end..][0..cond_extra.data.then_body_len]);
647 const else_body: []const Air.Inst.Index = @ptrCast(air.extra.items[cond_extra.end + cond_extra.data.then_body_len ..][0..cond_extra.data.else_body_len]);
648 if (then_body.len > 1 and air_tags[@intFromEnum(then_body[1])] != .unreach)
649 return .complex;
650 if (else_body.len > 1 and air_tags[@intFromEnum(else_body[1])] != .unreach)
651 return .complex;
652
653 var operand_live: bool = true;
654 for (&[_]Air.Inst.Index{ then_body[0], else_body[0] }) |cond_inst| {
655 if (l.categorizeOperand(air, zcu, cond_inst, operand, ip) == .tomb)
656 operand_live = false;
657
658 switch (air_tags[@intFromEnum(cond_inst)]) {
659 .br => { // Breaks immediately back to block
660 const br = air_datas[@intFromEnum(cond_inst)].br;
661 if (br.block_inst != inst)
662 return .complex;
663 },
664 .call => {}, // Calls a noreturn function
665 else => return .complex,
666 }
667 }
668 return if (operand_live) .none else .tomb;
669 }
670
671 return .complex;
672 },
673
674 .@"try",
675 .try_cold,
676 .try_ptr,
677 .try_ptr_cold,
678 .loop,
679 .cond_br,
680 .switch_br,
681 .loop_switch_br,
682 => return .complex,
683
684 .wasm_memory_grow => {
685 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
686 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
687 return .none;
688 },
689 }
690}
691
692fn matchOperandSmallIndex(
693 l: Liveness,
694 inst: Air.Inst.Index,
695 operand: OperandInt,
696 default: OperandCategory,
697) OperandCategory {
698 if (operandDies(l, inst, operand)) {
699 return .tomb;
700 } else {
701 return default;
702 }
703}
704
705210/// Higher level API.
706211pub const CondBrSlices = struct {
707212 then_deaths: []const Air.Inst.Index,
src/codegen/llvm.zig+21-81
......@@ -4980,8 +4980,8 @@ pub const FuncGen = struct {
49804980 .breakpoint => try self.airBreakpoint(inst),
49814981 .ret_addr => try self.airRetAddr(inst),
49824982 .frame_addr => try self.airFrameAddress(inst),
4983 .@"try" => try self.airTry(body[i..], false),
4984 .try_cold => try self.airTry(body[i..], true),
4983 .@"try" => try self.airTry(inst, false),
4984 .try_cold => try self.airTry(inst, true),
49854985 .try_ptr => try self.airTryPtr(inst, false),
49864986 .try_ptr_cold => try self.airTryPtr(inst, true),
49874987 .intcast => try self.airIntCast(inst, false),
......@@ -4989,7 +4989,7 @@ pub const FuncGen = struct {
49894989 .trunc => try self.airTrunc(inst),
49904990 .fptrunc => try self.airFptrunc(inst),
49914991 .fpext => try self.airFpext(inst),
4992 .load => try self.airLoad(body[i..]),
4992 .load => try self.airLoad(inst),
49934993 .not => try self.airNot(inst),
49944994 .store => try self.airStore(inst, false),
49954995 .store_safe => try self.airStore(inst, true),
......@@ -5045,7 +5045,7 @@ pub const FuncGen = struct {
50455045 .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
50465046
50475047 .struct_field_ptr => try self.airStructFieldPtr(inst),
5048 .struct_field_val => try self.airStructFieldVal(body[i..]),
5048 .struct_field_val => try self.airStructFieldVal(inst),
50495049
50505050 .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
50515051 .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
......@@ -5054,18 +5054,18 @@ pub const FuncGen = struct {
50545054
50555055 .field_parent_ptr => try self.airFieldParentPtr(inst),
50565056
5057 .array_elem_val => try self.airArrayElemVal(body[i..]),
5058 .slice_elem_val => try self.airSliceElemVal(body[i..]),
5057 .array_elem_val => try self.airArrayElemVal(inst),
5058 .slice_elem_val => try self.airSliceElemVal(inst),
50595059 .slice_elem_ptr => try self.airSliceElemPtr(inst),
5060 .ptr_elem_val => try self.airPtrElemVal(body[i..]),
5060 .ptr_elem_val => try self.airPtrElemVal(inst),
50615061 .ptr_elem_ptr => try self.airPtrElemPtr(inst),
50625062
5063 .optional_payload => try self.airOptionalPayload(body[i..]),
5063 .optional_payload => try self.airOptionalPayload(inst),
50645064 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
50655065 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
50665066
5067 .unwrap_errunion_payload => try self.airErrUnionPayload(body[i..], false),
5068 .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(body[i..], true),
5067 .unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),
5068 .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),
50695069 .unwrap_errunion_err => try self.airErrUnionErr(inst, false),
50705070 .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
50715071 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
......@@ -6266,19 +6266,14 @@ pub const FuncGen = struct {
62666266 // No need to reset the insert cursor since this instruction is noreturn.
62676267 }
62686268
6269 fn airTry(self: *FuncGen, body_tail: []const Air.Inst.Index, err_cold: bool) !Builder.Value {
6270 const pt = self.ng.pt;
6271 const zcu = pt.zcu;
6272 const inst = body_tail[0];
6269 fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
62736270 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
62746271 const err_union = try self.resolveInst(pl_op.operand);
62756272 const extra = self.air.extraData(Air.Try, pl_op.payload);
62766273 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
62776274 const err_union_ty = self.typeOf(pl_op.operand);
6278 const payload_ty = self.typeOfIndex(inst);
6279 const can_elide_load = if (isByRef(payload_ty, zcu)) self.canElideLoad(body_tail) else false;
62806275 const is_unused = self.liveness.isUnused(inst);
6281 return lowerTry(self, err_union, body, err_union_ty, false, can_elide_load, is_unused, err_cold);
6276 return lowerTry(self, err_union, body, err_union_ty, false, false, is_unused, err_cold);
62826277 }
62836278
62846279 fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
......@@ -6824,11 +6819,10 @@ pub const FuncGen = struct {
68246819 return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, "");
68256820 }
68266821
6827 fn airSliceElemVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
6822 fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
68286823 const o = self.ng.object;
68296824 const pt = self.ng.pt;
68306825 const zcu = pt.zcu;
6831 const inst = body_tail[0];
68326826 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
68336827 const slice_ty = self.typeOf(bin_op.lhs);
68346828 const slice = try self.resolveInst(bin_op.lhs);
......@@ -6838,9 +6832,6 @@ pub const FuncGen = struct {
68386832 const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
68396833 const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
68406834 if (isByRef(elem_ty, zcu)) {
6841 if (self.canElideLoad(body_tail))
6842 return ptr;
6843
68446835 self.maybeMarkAllowZeroAccess(slice_ty.ptrInfo(zcu));
68456836
68466837 const slice_align = (slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();
......@@ -6867,11 +6858,10 @@ pub const FuncGen = struct {
68676858 return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
68686859 }
68696860
6870 fn airArrayElemVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
6861 fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
68716862 const o = self.ng.object;
68726863 const pt = self.ng.pt;
68736864 const zcu = pt.zcu;
6874 const inst = body_tail[0];
68756865
68766866 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
68776867 const array_ty = self.typeOf(bin_op.lhs);
......@@ -6884,9 +6874,7 @@ pub const FuncGen = struct {
68846874 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), rhs,
68856875 };
68866876 if (isByRef(elem_ty, zcu)) {
6887 const elem_ptr =
6888 try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, "");
6889 if (canElideLoad(self, body_tail)) return elem_ptr;
6877 const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, "");
68906878 const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();
68916879 return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);
68926880 } else {
......@@ -6900,11 +6888,10 @@ pub const FuncGen = struct {
69006888 return self.wip.extractElement(array_llvm_val, rhs, "");
69016889 }
69026890
6903 fn airPtrElemVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
6891 fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
69046892 const o = self.ng.object;
69056893 const pt = self.ng.pt;
69066894 const zcu = pt.zcu;
6907 const inst = body_tail[0];
69086895 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
69096896 const ptr_ty = self.typeOf(bin_op.lhs);
69106897 const elem_ty = ptr_ty.childType(zcu);
......@@ -6918,10 +6905,7 @@ pub const FuncGen = struct {
69186905 else
69196906 &.{rhs}, "");
69206907 if (isByRef(elem_ty, zcu)) {
6921 if (self.canElideLoad(body_tail)) return ptr;
6922
69236908 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
6924
69256909 const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();
69266910 return self.loadByRef(ptr, elem_ty, ptr_align, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal);
69276911 }
......@@ -6974,11 +6958,10 @@ pub const FuncGen = struct {
69746958 return self.fieldPtr(inst, struct_ptr, struct_ptr_ty, field_index);
69756959 }
69766960
6977 fn airStructFieldVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
6961 fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
69786962 const o = self.ng.object;
69796963 const pt = self.ng.pt;
69806964 const zcu = pt.zcu;
6981 const inst = body_tail[0];
69826965 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
69836966 const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
69846967 const struct_ty = self.typeOf(struct_field.struct_operand);
......@@ -7052,9 +7035,6 @@ pub const FuncGen = struct {
70527035 .flags = .{ .alignment = alignment },
70537036 });
70547037 if (isByRef(field_ty, zcu)) {
7055 if (canElideLoad(self, body_tail))
7056 return field_ptr;
7057
70587038 assert(alignment != .none);
70597039 const field_alignment = alignment.toLlvm();
70607040 return self.loadByRef(field_ptr, field_ty, field_alignment, .normal);
......@@ -7070,7 +7050,6 @@ pub const FuncGen = struct {
70707050 try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, "");
70717051 const payload_alignment = layout.payload_align.toLlvm();
70727052 if (isByRef(field_ty, zcu)) {
7073 if (canElideLoad(self, body_tail)) return field_ptr;
70747053 return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal);
70757054 } else {
70767055 return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment);
......@@ -7829,11 +7808,10 @@ pub const FuncGen = struct {
78297808 return self.wip.gepStruct(optional_llvm_ty, operand, 0, "");
78307809 }
78317810
7832 fn airOptionalPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
7811 fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
78337812 const o = self.ng.object;
78347813 const pt = self.ng.pt;
78357814 const zcu = pt.zcu;
7836 const inst = body_tail[0];
78377815 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
78387816 const operand = try self.resolveInst(ty_op.operand);
78397817 const optional_ty = self.typeOf(ty_op.operand);
......@@ -7846,19 +7824,13 @@ pub const FuncGen = struct {
78467824 }
78477825
78487826 const opt_llvm_ty = try o.lowerType(pt, optional_ty);
7849 const can_elide_load = if (isByRef(payload_ty, zcu)) self.canElideLoad(body_tail) else false;
7850 return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, can_elide_load);
7827 return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false);
78517828 }
78527829
7853 fn airErrUnionPayload(
7854 self: *FuncGen,
7855 body_tail: []const Air.Inst.Index,
7856 operand_is_ptr: bool,
7857 ) !Builder.Value {
7830 fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value {
78587831 const o = self.ng.object;
78597832 const pt = self.ng.pt;
78607833 const zcu = pt.zcu;
7861 const inst = body_tail[0];
78627834 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
78637835 const operand = try self.resolveInst(ty_op.operand);
78647836 const operand_ty = self.typeOf(ty_op.operand);
......@@ -7877,7 +7849,6 @@ pub const FuncGen = struct {
78777849 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
78787850 const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
78797851 if (isByRef(payload_ty, zcu)) {
7880 if (self.canElideLoad(body_tail)) return payload_ptr;
78817852 return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
78827853 }
78837854 const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
......@@ -9740,45 +9711,14 @@ pub const FuncGen = struct {
97409711 return .none;
97419712 }
97429713
9743 /// As an optimization, we want to avoid unnecessary copies of isByRef=true
9744 /// types. Here, we scan forward in the current block, looking to see if
9745 /// this load dies before any side effects occur. In such case, we can
9746 /// safely return the operand without making a copy.
9747 ///
9748 /// The first instruction of `body_tail` is the one whose copy we want to elide.
9749 fn canElideLoad(fg: *FuncGen, body_tail: []const Air.Inst.Index) bool {
9750 const zcu = fg.ng.pt.zcu;
9751 const ip = &zcu.intern_pool;
9752 for (body_tail[1..]) |body_inst| {
9753 switch (fg.liveness.categorizeOperand(fg.air, zcu, body_inst, body_tail[0], ip)) {
9754 .none => continue,
9755 .write, .noret, .complex => return false,
9756 .tomb => return true,
9757 }
9758 }
9759 // The only way to get here is to hit the end of a loop instruction
9760 // (implicit repeat).
9761 return false;
9762 }
9763
9764 fn airLoad(fg: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
9714 fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
97659715 const pt = fg.ng.pt;
97669716 const zcu = pt.zcu;
9767 const inst = body_tail[0];
97689717 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
97699718 const ptr_ty = fg.typeOf(ty_op.operand);
97709719 const ptr_info = ptr_ty.ptrInfo(zcu);
97719720 const ptr = try fg.resolveInst(ty_op.operand);
9772
9773 elide: {
9774 if (ptr_info.flags.alignment != .none) break :elide;
9775 if (!isByRef(Type.fromInterned(ptr_info.child), zcu)) break :elide;
9776 if (!canElideLoad(fg, body_tail)) break :elide;
9777 return ptr;
9778 }
9779
97809721 fg.maybeMarkAllowZeroAccess(ptr_info);
9781
97829722 return fg.load(ptr, ptr_ty);
97839723 }
97849724