authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-09-02 23:10:18-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-09-20 18:33:00-07:00
log14bda4130a9a7f8b529b12bc74a1d6caa71b9545
treea60bcdc73299b869caf643d29d9b146b2ae09a03
parent3fbb88c4bd146ca7bd9e7ab5da9c4b05298f3b34

llvm backend: remove canElideLoad mechanism

2 files changed, 21 insertions(+), 576 deletions(-)

src/Air/Liveness.zig-495
...@@ -207,501 +207,6 @@ pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool...@@ -207,501 +207,6 @@ pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool
207 return (l.tomb_bits[usize_index] & mask) != 0;207 return (l.tomb_bits[usize_index] & mask) != 0;
208}208}
209209
210const OperandCategory = enum {
211 /// The operand lives on, but this instruction cannot possibly mutate memory.
212 none,
213 /// The operand lives on and this instruction can mutate memory.
214 write,
215 /// The operand dies at this instruction.
216 tomb,
217 /// The operand lives on, and this instruction is noreturn.
218 noret,
219 /// This instruction is too complicated for analysis, no information is available.
220 complex,
221};
222
223/// Given an instruction that we are examining, and an operand that we are looking for,
224/// returns a classification.
225pub fn categorizeOperand(
226 l: Liveness,
227 air: Air,
228 zcu: *Zcu,
229 inst: Air.Inst.Index,
230 operand: Air.Inst.Index,
231 ip: *const InternPool,
232) OperandCategory {
233 const air_tags = air.instructions.items(.tag);
234 const air_datas = air.instructions.items(.data);
235 const operand_ref = operand.toRef();
236 switch (air_tags[@intFromEnum(inst)]) {
237 .add,
238 .add_safe,
239 .add_wrap,
240 .add_sat,
241 .add_optimized,
242 .sub,
243 .sub_safe,
244 .sub_wrap,
245 .sub_sat,
246 .sub_optimized,
247 .mul,
248 .mul_safe,
249 .mul_wrap,
250 .mul_sat,
251 .mul_optimized,
252 .div_float,
253 .div_trunc,
254 .div_floor,
255 .div_exact,
256 .rem,
257 .mod,
258 .bit_and,
259 .bit_or,
260 .xor,
261 .cmp_lt,
262 .cmp_lte,
263 .cmp_eq,
264 .cmp_gte,
265 .cmp_gt,
266 .cmp_neq,
267 .bool_and,
268 .bool_or,
269 .array_elem_val,
270 .slice_elem_val,
271 .ptr_elem_val,
272 .shl,
273 .shl_exact,
274 .shl_sat,
275 .shr,
276 .shr_exact,
277 .min,
278 .max,
279 .div_float_optimized,
280 .div_trunc_optimized,
281 .div_floor_optimized,
282 .div_exact_optimized,
283 .rem_optimized,
284 .mod_optimized,
285 .neg_optimized,
286 .cmp_lt_optimized,
287 .cmp_lte_optimized,
288 .cmp_eq_optimized,
289 .cmp_gte_optimized,
290 .cmp_gt_optimized,
291 .cmp_neq_optimized,
292 => {
293 const o = air_datas[@intFromEnum(inst)].bin_op;
294 if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
295 if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
296 return .none;
297 },
298
299 .store,
300 .store_safe,
301 .atomic_store_unordered,
302 .atomic_store_monotonic,
303 .atomic_store_release,
304 .atomic_store_seq_cst,
305 .set_union_tag,
306 .memset,
307 .memset_safe,
308 .memcpy,
309 .memmove,
310 => {
311 const o = air_datas[@intFromEnum(inst)].bin_op;
312 if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
313 if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
314 return .write;
315 },
316
317 .vector_store_elem => {
318 const o = air_datas[@intFromEnum(inst)].vector_store_elem;
319 const extra = air.extraData(Air.Bin, o.payload).data;
320 if (o.vector_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
321 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
322 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
323 return .write;
324 },
325
326 .arg,
327 .alloc,
328 .inferred_alloc,
329 .inferred_alloc_comptime,
330 .ret_ptr,
331 .trap,
332 .breakpoint,
333 .repeat,
334 .switch_dispatch,
335 .dbg_stmt,
336 .dbg_empty_stmt,
337 .unreach,
338 .ret_addr,
339 .frame_addr,
340 .wasm_memory_size,
341 .err_return_trace,
342 .save_err_return_trace_index,
343 .runtime_nav_ptr,
344 .c_va_start,
345 .work_item_id,
346 .work_group_size,
347 .work_group_id,
348 => return .none,
349
350 .not,
351 .bitcast,
352 .load,
353 .fpext,
354 .fptrunc,
355 .intcast,
356 .intcast_safe,
357 .trunc,
358 .optional_payload,
359 .optional_payload_ptr,
360 .wrap_optional,
361 .unwrap_errunion_payload,
362 .unwrap_errunion_err,
363 .unwrap_errunion_payload_ptr,
364 .unwrap_errunion_err_ptr,
365 .wrap_errunion_payload,
366 .wrap_errunion_err,
367 .slice_ptr,
368 .slice_len,
369 .ptr_slice_len_ptr,
370 .ptr_slice_ptr_ptr,
371 .struct_field_ptr_index_0,
372 .struct_field_ptr_index_1,
373 .struct_field_ptr_index_2,
374 .struct_field_ptr_index_3,
375 .array_to_slice,
376 .int_from_float,
377 .int_from_float_optimized,
378 .int_from_float_safe,
379 .int_from_float_optimized_safe,
380 .float_from_int,
381 .get_union_tag,
382 .clz,
383 .ctz,
384 .popcount,
385 .byte_swap,
386 .bit_reverse,
387 .splat,
388 .error_set_has_value,
389 .addrspace_cast,
390 .c_va_arg,
391 .c_va_copy,
392 .abs,
393 => {
394 const o = air_datas[@intFromEnum(inst)].ty_op;
395 if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
396 return .none;
397 },
398
399 .optional_payload_ptr_set,
400 .errunion_payload_ptr_set,
401 => {
402 const o = air_datas[@intFromEnum(inst)].ty_op;
403 if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
404 return .write;
405 },
406
407 .is_null,
408 .is_non_null,
409 .is_null_ptr,
410 .is_non_null_ptr,
411 .is_err,
412 .is_non_err,
413 .is_err_ptr,
414 .is_non_err_ptr,
415 .is_named_enum_value,
416 .tag_name,
417 .error_name,
418 .sqrt,
419 .sin,
420 .cos,
421 .tan,
422 .exp,
423 .exp2,
424 .log,
425 .log2,
426 .log10,
427 .floor,
428 .ceil,
429 .round,
430 .trunc_float,
431 .neg,
432 .cmp_lt_errors_len,
433 .c_va_end,
434 => {
435 const o = air_datas[@intFromEnum(inst)].un_op;
436 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
437 return .none;
438 },
439
440 .ret,
441 .ret_safe,
442 .ret_load,
443 => {
444 const o = air_datas[@intFromEnum(inst)].un_op;
445 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .noret);
446 return .noret;
447 },
448
449 .set_err_return_trace => {
450 const o = air_datas[@intFromEnum(inst)].un_op;
451 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
452 return .write;
453 },
454
455 .add_with_overflow,
456 .sub_with_overflow,
457 .mul_with_overflow,
458 .shl_with_overflow,
459 .ptr_add,
460 .ptr_sub,
461 .ptr_elem_ptr,
462 .slice_elem_ptr,
463 .slice,
464 => {
465 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
466 const extra = air.extraData(Air.Bin, ty_pl.payload).data;
467 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
468 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
469 return .none;
470 },
471
472 .dbg_var_ptr,
473 .dbg_var_val,
474 .dbg_arg_inline,
475 => {
476 const o = air_datas[@intFromEnum(inst)].pl_op.operand;
477 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
478 return .none;
479 },
480
481 .prefetch => {
482 const prefetch = air_datas[@intFromEnum(inst)].prefetch;
483 if (prefetch.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
484 return .none;
485 },
486
487 .call, .call_always_tail, .call_never_tail, .call_never_inline => {
488 const inst_data = air_datas[@intFromEnum(inst)].pl_op;
489 const callee = inst_data.operand;
490 const extra = air.extraData(Air.Call, inst_data.payload);
491 const args = @as([]const Air.Inst.Ref, @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]));
492 if (args.len + 1 <= bpi - 1) {
493 if (callee == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
494 for (args, 0..) |arg, i| {
495 if (arg == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i + 1)), .write);
496 }
497 return .write;
498 }
499 var bt = l.iterateBigTomb(inst);
500 if (bt.feed()) {
501 if (callee == operand_ref) return .tomb;
502 } else {
503 if (callee == operand_ref) return .write;
504 }
505 for (args) |arg| {
506 if (bt.feed()) {
507 if (arg == operand_ref) return .tomb;
508 } else {
509 if (arg == operand_ref) return .write;
510 }
511 }
512 return .write;
513 },
514 .select => {
515 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
516 const extra = air.extraData(Air.Bin, pl_op.payload).data;
517 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
518 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
519 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
520 return .none;
521 },
522 .shuffle_one => {
523 const unwrapped = air.unwrapShuffleOne(zcu, inst);
524 if (unwrapped.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
525 return .none;
526 },
527 .shuffle_two => {
528 const unwrapped = air.unwrapShuffleTwo(zcu, inst);
529 if (unwrapped.operand_a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
530 if (unwrapped.operand_b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
531 return .none;
532 },
533 .reduce, .reduce_optimized => {
534 const reduce = air_datas[@intFromEnum(inst)].reduce;
535 if (reduce.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
536 return .none;
537 },
538 .cmp_vector, .cmp_vector_optimized => {
539 const extra = air.extraData(Air.VectorCmp, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
540 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
541 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
542 return .none;
543 },
544 .aggregate_init => {
545 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
546 const aggregate_ty = ty_pl.ty.toType();
547 const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
548 const elements = @as([]const Air.Inst.Ref, @ptrCast(air.extra.items[ty_pl.payload..][0..len]));
549
550 if (elements.len <= bpi - 1) {
551 for (elements, 0..) |elem, i| {
552 if (elem == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i)), .none);
553 }
554 return .none;
555 }
556
557 var bt = l.iterateBigTomb(inst);
558 for (elements) |elem| {
559 if (bt.feed()) {
560 if (elem == operand_ref) return .tomb;
561 } else {
562 if (elem == operand_ref) return .write;
563 }
564 }
565 return .write;
566 },
567 .union_init => {
568 const extra = air.extraData(Air.UnionInit, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
569 if (extra.init == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
570 return .none;
571 },
572 .struct_field_ptr, .struct_field_val => {
573 const extra = air.extraData(Air.StructField, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
574 if (extra.struct_operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
575 return .none;
576 },
577 .field_parent_ptr => {
578 const extra = air.extraData(Air.FieldParentPtr, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
579 if (extra.field_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
580 return .none;
581 },
582 .cmpxchg_strong, .cmpxchg_weak => {
583 const extra = air.extraData(Air.Cmpxchg, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
584 if (extra.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
585 if (extra.expected_value == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
586 if (extra.new_value == operand_ref) return matchOperandSmallIndex(l, inst, 2, .write);
587 return .write;
588 },
589 .mul_add => {
590 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
591 const extra = air.extraData(Air.Bin, pl_op.payload).data;
592 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
593 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
594 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
595 return .none;
596 },
597 .atomic_load => {
598 const ptr = air_datas[@intFromEnum(inst)].atomic_load.ptr;
599 if (ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
600 return .none;
601 },
602 .atomic_rmw => {
603 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
604 const extra = air.extraData(Air.AtomicRmw, pl_op.payload).data;
605 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
606 if (extra.operand == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
607 return .write;
608 },
609
610 .br => {
611 const br = air_datas[@intFromEnum(inst)].br;
612 if (br.operand == operand_ref) return matchOperandSmallIndex(l, operand, 0, .noret);
613 return .noret;
614 },
615 .assembly => {
616 return .complex;
617 },
618 .block, .dbg_inline_block => |tag| {
619 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
620 const body: []const Air.Inst.Index = @ptrCast(switch (tag) {
621 inline .block, .dbg_inline_block => |comptime_tag| body: {
622 const extra = air.extraData(switch (comptime_tag) {
623 .block => Air.Block,
624 .dbg_inline_block => Air.DbgInlineBlock,
625 else => unreachable,
626 }, ty_pl.payload);
627 break :body air.extra.items[extra.end..][0..extra.data.body_len];
628 },
629 else => unreachable,
630 });
631
632 if (body.len == 1 and air_tags[@intFromEnum(body[0])] == .cond_br) {
633 // Peephole optimization for "panic-like" conditionals, which have
634 // one empty branch and another which calls a `noreturn` function.
635 // This allows us to infer that safety checks do not modify memory,
636 // as far as control flow successors are concerned.
637
638 const inst_data = air_datas[@intFromEnum(body[0])].pl_op;
639 const cond_extra = air.extraData(Air.CondBr, inst_data.payload);
640 if (inst_data.operand == operand_ref and operandDies(l, body[0], 0))
641 return .tomb;
642
643 if (cond_extra.data.then_body_len > 2 or cond_extra.data.else_body_len > 2)
644 return .complex;
645
646 const then_body: []const Air.Inst.Index = @ptrCast(air.extra.items[cond_extra.end..][0..cond_extra.data.then_body_len]);
647 const else_body: []const Air.Inst.Index = @ptrCast(air.extra.items[cond_extra.end + cond_extra.data.then_body_len ..][0..cond_extra.data.else_body_len]);
648 if (then_body.len > 1 and air_tags[@intFromEnum(then_body[1])] != .unreach)
649 return .complex;
650 if (else_body.len > 1 and air_tags[@intFromEnum(else_body[1])] != .unreach)
651 return .complex;
652
653 var operand_live: bool = true;
654 for (&[_]Air.Inst.Index{ then_body[0], else_body[0] }) |cond_inst| {
655 if (l.categorizeOperand(air, zcu, cond_inst, operand, ip) == .tomb)
656 operand_live = false;
657
658 switch (air_tags[@intFromEnum(cond_inst)]) {
659 .br => { // Breaks immediately back to block
660 const br = air_datas[@intFromEnum(cond_inst)].br;
661 if (br.block_inst != inst)
662 return .complex;
663 },
664 .call => {}, // Calls a noreturn function
665 else => return .complex,
666 }
667 }
668 return if (operand_live) .none else .tomb;
669 }
670
671 return .complex;
672 },
673
674 .@"try",
675 .try_cold,
676 .try_ptr,
677 .try_ptr_cold,
678 .loop,
679 .cond_br,
680 .switch_br,
681 .loop_switch_br,
682 => return .complex,
683
684 .wasm_memory_grow => {
685 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
686 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
687 return .none;
688 },
689 }
690}
691
692fn matchOperandSmallIndex(
693 l: Liveness,
694 inst: Air.Inst.Index,
695 operand: OperandInt,
696 default: OperandCategory,
697) OperandCategory {
698 if (operandDies(l, inst, operand)) {
699 return .tomb;
700 } else {
701 return default;
702 }
703}
704
705/// Higher level API.210/// Higher level API.
706pub const CondBrSlices = struct {211pub const CondBrSlices = struct {
707 then_deaths: []const Air.Inst.Index,212 then_deaths: []const Air.Inst.Index,
src/codegen/llvm.zig+21-81
...@@ -4980,8 +4980,8 @@ pub const FuncGen = struct {...@@ -4980,8 +4980,8 @@ pub const FuncGen = struct {
4980 .breakpoint => try self.airBreakpoint(inst),4980 .breakpoint => try self.airBreakpoint(inst),
4981 .ret_addr => try self.airRetAddr(inst),4981 .ret_addr => try self.airRetAddr(inst),
4982 .frame_addr => try self.airFrameAddress(inst),4982 .frame_addr => try self.airFrameAddress(inst),
4983 .@"try" => try self.airTry(body[i..], false),4983 .@"try" => try self.airTry(inst, false),
4984 .try_cold => try self.airTry(body[i..], true),4984 .try_cold => try self.airTry(inst, true),
4985 .try_ptr => try self.airTryPtr(inst, false),4985 .try_ptr => try self.airTryPtr(inst, false),
4986 .try_ptr_cold => try self.airTryPtr(inst, true),4986 .try_ptr_cold => try self.airTryPtr(inst, true),
4987 .intcast => try self.airIntCast(inst, false),4987 .intcast => try self.airIntCast(inst, false),
...@@ -4989,7 +4989,7 @@ pub const FuncGen = struct {...@@ -4989,7 +4989,7 @@ pub const FuncGen = struct {
4989 .trunc => try self.airTrunc(inst),4989 .trunc => try self.airTrunc(inst),
4990 .fptrunc => try self.airFptrunc(inst),4990 .fptrunc => try self.airFptrunc(inst),
4991 .fpext => try self.airFpext(inst),4991 .fpext => try self.airFpext(inst),
4992 .load => try self.airLoad(body[i..]),4992 .load => try self.airLoad(inst),
4993 .not => try self.airNot(inst),4993 .not => try self.airNot(inst),
4994 .store => try self.airStore(inst, false),4994 .store => try self.airStore(inst, false),
4995 .store_safe => try self.airStore(inst, true),4995 .store_safe => try self.airStore(inst, true),
...@@ -5045,7 +5045,7 @@ pub const FuncGen = struct {...@@ -5045,7 +5045,7 @@ pub const FuncGen = struct {
5045 .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),5045 .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
50465046
5047 .struct_field_ptr => try self.airStructFieldPtr(inst),5047 .struct_field_ptr => try self.airStructFieldPtr(inst),
5048 .struct_field_val => try self.airStructFieldVal(body[i..]),5048 .struct_field_val => try self.airStructFieldVal(inst),
50495049
5050 .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),5050 .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
5051 .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),5051 .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
...@@ -5054,18 +5054,18 @@ pub const FuncGen = struct {...@@ -5054,18 +5054,18 @@ pub const FuncGen = struct {
50545054
5055 .field_parent_ptr => try self.airFieldParentPtr(inst),5055 .field_parent_ptr => try self.airFieldParentPtr(inst),
50565056
5057 .array_elem_val => try self.airArrayElemVal(body[i..]),5057 .array_elem_val => try self.airArrayElemVal(inst),
5058 .slice_elem_val => try self.airSliceElemVal(body[i..]),5058 .slice_elem_val => try self.airSliceElemVal(inst),
5059 .slice_elem_ptr => try self.airSliceElemPtr(inst),5059 .slice_elem_ptr => try self.airSliceElemPtr(inst),
5060 .ptr_elem_val => try self.airPtrElemVal(body[i..]),5060 .ptr_elem_val => try self.airPtrElemVal(inst),
5061 .ptr_elem_ptr => try self.airPtrElemPtr(inst),5061 .ptr_elem_ptr => try self.airPtrElemPtr(inst),
50625062
5063 .optional_payload => try self.airOptionalPayload(body[i..]),5063 .optional_payload => try self.airOptionalPayload(inst),
5064 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),5064 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
5065 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),5065 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
50665066
5067 .unwrap_errunion_payload => try self.airErrUnionPayload(body[i..], false),5067 .unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),
5068 .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(body[i..], true),5068 .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),
5069 .unwrap_errunion_err => try self.airErrUnionErr(inst, false),5069 .unwrap_errunion_err => try self.airErrUnionErr(inst, false),
5070 .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),5070 .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
5071 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),5071 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
...@@ -6266,19 +6266,14 @@ pub const FuncGen = struct {...@@ -6266,19 +6266,14 @@ pub const FuncGen = struct {
6266 // No need to reset the insert cursor since this instruction is noreturn.6266 // No need to reset the insert cursor since this instruction is noreturn.
6267 }6267 }
62686268
6269 fn airTry(self: *FuncGen, body_tail: []const Air.Inst.Index, err_cold: bool) !Builder.Value {6269 fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
6270 const pt = self.ng.pt;
6271 const zcu = pt.zcu;
6272 const inst = body_tail[0];
6273 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;6270 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6274 const err_union = try self.resolveInst(pl_op.operand);6271 const err_union = try self.resolveInst(pl_op.operand);
6275 const extra = self.air.extraData(Air.Try, pl_op.payload);6272 const extra = self.air.extraData(Air.Try, pl_op.payload);
6276 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);6273 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
6277 const err_union_ty = self.typeOf(pl_op.operand);6274 const err_union_ty = self.typeOf(pl_op.operand);
6278 const payload_ty = self.typeOfIndex(inst);
6279 const can_elide_load = if (isByRef(payload_ty, zcu)) self.canElideLoad(body_tail) else false;
6280 const is_unused = self.liveness.isUnused(inst);6275 const is_unused = self.liveness.isUnused(inst);
6281 return lowerTry(self, err_union, body, err_union_ty, false, can_elide_load, is_unused, err_cold);6276 return lowerTry(self, err_union, body, err_union_ty, false, false, is_unused, err_cold);
6282 }6277 }
62836278
6284 fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {6279 fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
...@@ -6824,11 +6819,10 @@ pub const FuncGen = struct {...@@ -6824,11 +6819,10 @@ pub const FuncGen = struct {
6824 return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, "");6819 return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, "");
6825 }6820 }
68266821
6827 fn airSliceElemVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {6822 fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6828 const o = self.ng.object;6823 const o = self.ng.object;
6829 const pt = self.ng.pt;6824 const pt = self.ng.pt;
6830 const zcu = pt.zcu;6825 const zcu = pt.zcu;
6831 const inst = body_tail[0];
6832 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6826 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6833 const slice_ty = self.typeOf(bin_op.lhs);6827 const slice_ty = self.typeOf(bin_op.lhs);
6834 const slice = try self.resolveInst(bin_op.lhs);6828 const slice = try self.resolveInst(bin_op.lhs);
...@@ -6838,9 +6832,6 @@ pub const FuncGen = struct {...@@ -6838,9 +6832,6 @@ pub const FuncGen = struct {
6838 const base_ptr = try self.wip.extractValue(slice, &.{0}, "");6832 const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
6839 const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");6833 const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
6840 if (isByRef(elem_ty, zcu)) {6834 if (isByRef(elem_ty, zcu)) {
6841 if (self.canElideLoad(body_tail))
6842 return ptr;
6843
6844 self.maybeMarkAllowZeroAccess(slice_ty.ptrInfo(zcu));6835 self.maybeMarkAllowZeroAccess(slice_ty.ptrInfo(zcu));
68456836
6846 const slice_align = (slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();6837 const slice_align = (slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();
...@@ -6867,11 +6858,10 @@ pub const FuncGen = struct {...@@ -6867,11 +6858,10 @@ pub const FuncGen = struct {
6867 return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");6858 return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
6868 }6859 }
68696860
6870 fn airArrayElemVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {6861 fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6871 const o = self.ng.object;6862 const o = self.ng.object;
6872 const pt = self.ng.pt;6863 const pt = self.ng.pt;
6873 const zcu = pt.zcu;6864 const zcu = pt.zcu;
6874 const inst = body_tail[0];
68756865
6876 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6866 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6877 const array_ty = self.typeOf(bin_op.lhs);6867 const array_ty = self.typeOf(bin_op.lhs);
...@@ -6884,9 +6874,7 @@ pub const FuncGen = struct {...@@ -6884,9 +6874,7 @@ pub const FuncGen = struct {
6884 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), rhs,6874 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), rhs,
6885 };6875 };
6886 if (isByRef(elem_ty, zcu)) {6876 if (isByRef(elem_ty, zcu)) {
6887 const elem_ptr =6877 const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, "");
6888 try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, "");
6889 if (canElideLoad(self, body_tail)) return elem_ptr;
6890 const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();6878 const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();
6891 return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);6879 return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);
6892 } else {6880 } else {
...@@ -6900,11 +6888,10 @@ pub const FuncGen = struct {...@@ -6900,11 +6888,10 @@ pub const FuncGen = struct {
6900 return self.wip.extractElement(array_llvm_val, rhs, "");6888 return self.wip.extractElement(array_llvm_val, rhs, "");
6901 }6889 }
69026890
6903 fn airPtrElemVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {6891 fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6904 const o = self.ng.object;6892 const o = self.ng.object;
6905 const pt = self.ng.pt;6893 const pt = self.ng.pt;
6906 const zcu = pt.zcu;6894 const zcu = pt.zcu;
6907 const inst = body_tail[0];
6908 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6895 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6909 const ptr_ty = self.typeOf(bin_op.lhs);6896 const ptr_ty = self.typeOf(bin_op.lhs);
6910 const elem_ty = ptr_ty.childType(zcu);6897 const elem_ty = ptr_ty.childType(zcu);
...@@ -6918,10 +6905,7 @@ pub const FuncGen = struct {...@@ -6918,10 +6905,7 @@ pub const FuncGen = struct {
6918 else6905 else
6919 &.{rhs}, "");6906 &.{rhs}, "");
6920 if (isByRef(elem_ty, zcu)) {6907 if (isByRef(elem_ty, zcu)) {
6921 if (self.canElideLoad(body_tail)) return ptr;
6922
6923 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));6908 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
6924
6925 const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();6909 const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();
6926 return self.loadByRef(ptr, elem_ty, ptr_align, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal);6910 return self.loadByRef(ptr, elem_ty, ptr_align, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal);
6927 }6911 }
...@@ -6974,11 +6958,10 @@ pub const FuncGen = struct {...@@ -6974,11 +6958,10 @@ pub const FuncGen = struct {
6974 return self.fieldPtr(inst, struct_ptr, struct_ptr_ty, field_index);6958 return self.fieldPtr(inst, struct_ptr, struct_ptr_ty, field_index);
6975 }6959 }
69766960
6977 fn airStructFieldVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {6961 fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6978 const o = self.ng.object;6962 const o = self.ng.object;
6979 const pt = self.ng.pt;6963 const pt = self.ng.pt;
6980 const zcu = pt.zcu;6964 const zcu = pt.zcu;
6981 const inst = body_tail[0];
6982 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;6965 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6983 const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;6966 const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
6984 const struct_ty = self.typeOf(struct_field.struct_operand);6967 const struct_ty = self.typeOf(struct_field.struct_operand);
...@@ -7052,9 +7035,6 @@ pub const FuncGen = struct {...@@ -7052,9 +7035,6 @@ pub const FuncGen = struct {
7052 .flags = .{ .alignment = alignment },7035 .flags = .{ .alignment = alignment },
7053 });7036 });
7054 if (isByRef(field_ty, zcu)) {7037 if (isByRef(field_ty, zcu)) {
7055 if (canElideLoad(self, body_tail))
7056 return field_ptr;
7057
7058 assert(alignment != .none);7038 assert(alignment != .none);
7059 const field_alignment = alignment.toLlvm();7039 const field_alignment = alignment.toLlvm();
7060 return self.loadByRef(field_ptr, field_ty, field_alignment, .normal);7040 return self.loadByRef(field_ptr, field_ty, field_alignment, .normal);
...@@ -7070,7 +7050,6 @@ pub const FuncGen = struct {...@@ -7070,7 +7050,6 @@ pub const FuncGen = struct {
7070 try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, "");7050 try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, "");
7071 const payload_alignment = layout.payload_align.toLlvm();7051 const payload_alignment = layout.payload_align.toLlvm();
7072 if (isByRef(field_ty, zcu)) {7052 if (isByRef(field_ty, zcu)) {
7073 if (canElideLoad(self, body_tail)) return field_ptr;
7074 return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal);7053 return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal);
7075 } else {7054 } else {
7076 return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment);7055 return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment);
...@@ -7829,11 +7808,10 @@ pub const FuncGen = struct {...@@ -7829,11 +7808,10 @@ pub const FuncGen = struct {
7829 return self.wip.gepStruct(optional_llvm_ty, operand, 0, "");7808 return self.wip.gepStruct(optional_llvm_ty, operand, 0, "");
7830 }7809 }
78317810
7832 fn airOptionalPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {7811 fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7833 const o = self.ng.object;7812 const o = self.ng.object;
7834 const pt = self.ng.pt;7813 const pt = self.ng.pt;
7835 const zcu = pt.zcu;7814 const zcu = pt.zcu;
7836 const inst = body_tail[0];
7837 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;7815 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7838 const operand = try self.resolveInst(ty_op.operand);7816 const operand = try self.resolveInst(ty_op.operand);
7839 const optional_ty = self.typeOf(ty_op.operand);7817 const optional_ty = self.typeOf(ty_op.operand);
...@@ -7846,19 +7824,13 @@ pub const FuncGen = struct {...@@ -7846,19 +7824,13 @@ pub const FuncGen = struct {
7846 }7824 }
78477825
7848 const opt_llvm_ty = try o.lowerType(pt, optional_ty);7826 const opt_llvm_ty = try o.lowerType(pt, optional_ty);
7849 const can_elide_load = if (isByRef(payload_ty, zcu)) self.canElideLoad(body_tail) else false;7827 return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false);
7850 return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, can_elide_load);
7851 }7828 }
78527829
7853 fn airErrUnionPayload(7830 fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value {
7854 self: *FuncGen,
7855 body_tail: []const Air.Inst.Index,
7856 operand_is_ptr: bool,
7857 ) !Builder.Value {
7858 const o = self.ng.object;7831 const o = self.ng.object;
7859 const pt = self.ng.pt;7832 const pt = self.ng.pt;
7860 const zcu = pt.zcu;7833 const zcu = pt.zcu;
7861 const inst = body_tail[0];
7862 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;7834 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7863 const operand = try self.resolveInst(ty_op.operand);7835 const operand = try self.resolveInst(ty_op.operand);
7864 const operand_ty = self.typeOf(ty_op.operand);7836 const operand_ty = self.typeOf(ty_op.operand);
...@@ -7877,7 +7849,6 @@ pub const FuncGen = struct {...@@ -7877,7 +7849,6 @@ pub const FuncGen = struct {
7877 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();7849 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
7878 const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");7850 const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
7879 if (isByRef(payload_ty, zcu)) {7851 if (isByRef(payload_ty, zcu)) {
7880 if (self.canElideLoad(body_tail)) return payload_ptr;
7881 return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);7852 return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
7882 }7853 }
7883 const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset];7854 const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
...@@ -9740,45 +9711,14 @@ pub const FuncGen = struct {...@@ -9740,45 +9711,14 @@ pub const FuncGen = struct {
9740 return .none;9711 return .none;
9741 }9712 }
97429713
9743 /// As an optimization, we want to avoid unnecessary copies of isByRef=true9714 fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9744 /// types. Here, we scan forward in the current block, looking to see if
9745 /// this load dies before any side effects occur. In such case, we can
9746 /// safely return the operand without making a copy.
9747 ///
9748 /// The first instruction of `body_tail` is the one whose copy we want to elide.
9749 fn canElideLoad(fg: *FuncGen, body_tail: []const Air.Inst.Index) bool {
9750 const zcu = fg.ng.pt.zcu;
9751 const ip = &zcu.intern_pool;
9752 for (body_tail[1..]) |body_inst| {
9753 switch (fg.liveness.categorizeOperand(fg.air, zcu, body_inst, body_tail[0], ip)) {
9754 .none => continue,
9755 .write, .noret, .complex => return false,
9756 .tomb => return true,
9757 }
9758 }
9759 // The only way to get here is to hit the end of a loop instruction
9760 // (implicit repeat).
9761 return false;
9762 }
9763
9764 fn airLoad(fg: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
9765 const pt = fg.ng.pt;9715 const pt = fg.ng.pt;
9766 const zcu = pt.zcu;9716 const zcu = pt.zcu;
9767 const inst = body_tail[0];
9768 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;9717 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9769 const ptr_ty = fg.typeOf(ty_op.operand);9718 const ptr_ty = fg.typeOf(ty_op.operand);
9770 const ptr_info = ptr_ty.ptrInfo(zcu);9719 const ptr_info = ptr_ty.ptrInfo(zcu);
9771 const ptr = try fg.resolveInst(ty_op.operand);9720 const ptr = try fg.resolveInst(ty_op.operand);
9772
9773 elide: {
9774 if (ptr_info.flags.alignment != .none) break :elide;
9775 if (!isByRef(Type.fromInterned(ptr_info.child), zcu)) break :elide;
9776 if (!canElideLoad(fg, body_tail)) break :elide;
9777 return ptr;
9778 }
9779
9780 fg.maybeMarkAllowZeroAccess(ptr_info);9721 fg.maybeMarkAllowZeroAccess(ptr_info);
9781
9782 return fg.load(ptr, ptr_ty);9722 return fg.load(ptr, ptr_ty);
9783 }9723 }
97849724