| ... | @@ -584,7 +584,7 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void { | ... | @@ -584,7 +584,7 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void { |
| 584 | | 584 | |
| 585 | air_body_index += 1; | 585 | air_body_index += 1; |
| 586 | }, | 586 | }, |
| 587 | .@"try", .try_cold, .try_ptr, .try_ptr_cold => { | 587 | .@"try", .try_cold => { |
| 588 | const pl_op = air_data[@intFromEnum(air_inst_index)].pl_op; | 588 | const pl_op = air_data[@intFromEnum(air_inst_index)].pl_op; |
| 589 | const extra = isel.air.extraData(Air.Try, pl_op.payload); | 589 | const extra = isel.air.extraData(Air.Try, pl_op.payload); |
| 590 | | 590 | |
| ... | @@ -596,6 +596,18 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void { | ... | @@ -596,6 +596,18 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void { |
| 596 | air_inst_index = air_body[air_body_index]; | 596 | air_inst_index = air_body[air_body_index]; |
| 597 | continue :air_tag air_tags[@intFromEnum(air_inst_index)]; | 597 | continue :air_tag air_tags[@intFromEnum(air_inst_index)]; |
| 598 | }, | 598 | }, |
| | 599 | .try_ptr, .try_ptr_cold => { |
| | 600 | const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl; |
| | 601 | const extra = isel.air.extraData(Air.TryPtr, ty_pl.payload); |
| | 602 | |
| | 603 | try isel.analyzeUse(extra.data.ptr); |
| | 604 | try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len])); |
| | 605 | try isel.def_order.putNoClobber(gpa, air_inst_index, {}); |
| | 606 | |
| | 607 | air_body_index += 1; |
| | 608 | air_inst_index = air_body[air_body_index]; |
| | 609 | continue :air_tag air_tags[@intFromEnum(air_inst_index)]; |
| | 610 | }, |
| 599 | .ret, .ret_safe, .ret_load => { | 611 | .ret, .ret_safe, .ret_load => { |
| 600 | const un_op = air_data[@intFromEnum(air_inst_index)].un_op; | 612 | const un_op = air_data[@intFromEnum(air_inst_index)].un_op; |
| 601 | isel.returns = true; | 613 | isel.returns = true; |
| ... | @@ -4760,17 +4772,62 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, | ... | @@ -4760,17 +4772,62 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 4760 | const error_set_part_vi = try error_set_part_it.only(isel); | 4772 | const error_set_part_vi = try error_set_part_it.only(isel); |
| 4761 | const error_set_part_mat = try error_set_part_vi.?.matReg(isel); | 4773 | const error_set_part_mat = try error_set_part_vi.?.matReg(isel); |
| 4762 | try isel.emit(.cbz( | 4774 | try isel.emit(.cbz( |
| 4763 | switch (error_set_part_vi.?.size(isel)) { | 4775 | error_set_part_mat.ra.w(), |
| 4764 | else => unreachable, | | |
| 4765 | 1...4 => error_set_part_mat.ra.w(), | | |
| 4766 | 5...8 => error_set_part_mat.ra.x(), | | |
| 4767 | }, | | |
| 4768 | @intCast((isel.instructions.items.len + 1 - cont_label) << 2), | 4776 | @intCast((isel.instructions.items.len + 1 - cont_label) << 2), |
| 4769 | )); | 4777 | )); |
| 4770 | try error_set_part_mat.finish(isel); | 4778 | try error_set_part_mat.finish(isel); |
| 4771 | | 4779 | |
| 4772 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; | 4780 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 4773 | }, | 4781 | }, |
| | 4782 | .try_ptr, .try_ptr_cold => { |
| | 4783 | const ty_pl = air.data(air.inst_index).ty_pl; |
| | 4784 | const extra = isel.air.extraData(Air.TryPtr, ty_pl.payload); |
| | 4785 | const error_union_ty = isel.air.typeOf(extra.data.ptr, ip).childType(zcu); |
| | 4786 | const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type; |
| | 4787 | const payload_ty: ZigType = .fromInterned(error_union_info.payload_type); |
| | 4788 | |
| | 4789 | const error_union_ptr_vi = try isel.use(extra.data.ptr); |
| | 4790 | const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel); |
| | 4791 | if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| unused: { |
| | 4792 | defer payload_ptr_vi.value.deref(isel); |
| | 4793 | switch (codegen.errUnionPayloadOffset(ty_pl.ty.toType().childType(zcu), zcu)) { |
| | 4794 | 0 => try payload_ptr_vi.value.move(isel, extra.data.ptr), |
| | 4795 | else => |payload_offset| { |
| | 4796 | const payload_ptr_ra = try payload_ptr_vi.value.defReg(isel) orelse break :unused; |
| | 4797 | const lo12: u12 = @truncate(payload_offset >> 0); |
| | 4798 | const hi12: u12 = @intCast(payload_offset >> 12); |
| | 4799 | if (hi12 > 0) try isel.emit(.add( |
| | 4800 | payload_ptr_ra.x(), |
| | 4801 | if (lo12 > 0) payload_ptr_ra.x() else error_union_ptr_mat.ra.x(), |
| | 4802 | .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } }, |
| | 4803 | )); |
| | 4804 | if (lo12 > 0) try isel.emit(.add(payload_ptr_ra.x(), error_union_ptr_mat.ra.x(), .{ .immediate = lo12 })); |
| | 4805 | }, |
| | 4806 | } |
| | 4807 | } |
| | 4808 | |
| | 4809 | const cont_label = isel.instructions.items.len; |
| | 4810 | const cont_live_registers = isel.live_registers; |
| | 4811 | try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len])); |
| | 4812 | try isel.merge(&cont_live_registers, .{}); |
| | 4813 | |
| | 4814 | const error_set_ra = try isel.allocIntReg(); |
| | 4815 | defer isel.freeReg(error_set_ra); |
| | 4816 | try isel.loadReg( |
| | 4817 | error_set_ra, |
| | 4818 | ZigType.fromInterned(error_union_info.error_set_type).abiSize(zcu), |
| | 4819 | .unsigned, |
| | 4820 | error_union_ptr_mat.ra, |
| | 4821 | codegen.errUnionErrorOffset(payload_ty, zcu), |
| | 4822 | ); |
| | 4823 | try error_union_ptr_mat.finish(isel); |
| | 4824 | try isel.emit(.cbz( |
| | 4825 | error_set_ra.w(), |
| | 4826 | @intCast((isel.instructions.items.len + 1 - cont_label) << 2), |
| | 4827 | )); |
| | 4828 | |
| | 4829 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| | 4830 | }, |
| 4774 | .dbg_stmt => { | 4831 | .dbg_stmt => { |
| 4775 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; | 4832 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 4776 | }, | 4833 | }, |
| ... | @@ -5403,14 +5460,6 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, | ... | @@ -5403,14 +5460,6 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 5403 | } | 5460 | } |
| 5404 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; | 5461 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 5405 | }, | 5462 | }, |
| 5406 | .optional_payload_ptr => { | | |
| 5407 | if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| { | | |
| 5408 | defer dst_vi.value.deref(isel); | | |
| 5409 | const ty_op = air.data(air.inst_index).ty_op; | | |
| 5410 | try dst_vi.value.move(isel, ty_op.operand); | | |
| 5411 | } | | |
| 5412 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; | | |
| 5413 | }, | | |
| 5414 | .optional_payload => { | 5463 | .optional_payload => { |
| 5415 | if (isel.live_values.fetchRemove(air.inst_index)) |payload_vi| unused: { | 5464 | if (isel.live_values.fetchRemove(air.inst_index)) |payload_vi| unused: { |
| 5416 | defer payload_vi.value.deref(isel); | 5465 | defer payload_vi.value.deref(isel); |
| ... | @@ -5429,6 +5478,37 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, | ... | @@ -5429,6 +5478,37 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 5429 | } | 5478 | } |
| 5430 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; | 5479 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 5431 | }, | 5480 | }, |
| | 5481 | .optional_payload_ptr => { |
| | 5482 | if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| { |
| | 5483 | defer payload_ptr_vi.value.deref(isel); |
| | 5484 | const ty_op = air.data(air.inst_index).ty_op; |
| | 5485 | try payload_ptr_vi.value.move(isel, ty_op.operand); |
| | 5486 | } |
| | 5487 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| | 5488 | }, |
| | 5489 | .optional_payload_ptr_set => { |
| | 5490 | if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| { |
| | 5491 | defer payload_ptr_vi.value.deref(isel); |
| | 5492 | const ty_op = air.data(air.inst_index).ty_op; |
| | 5493 | const opt_ty = isel.air.typeOf(ty_op.operand, ip).childType(zcu); |
| | 5494 | if (!opt_ty.optionalReprIsPayload(zcu)) { |
| | 5495 | const opt_ptr_vi = try isel.use(ty_op.operand); |
| | 5496 | const opt_ptr_mat = try opt_ptr_vi.matReg(isel); |
| | 5497 | const has_value_ra = try isel.allocIntReg(); |
| | 5498 | defer isel.freeReg(has_value_ra); |
| | 5499 | try isel.storeReg( |
| | 5500 | has_value_ra, |
| | 5501 | 1, |
| | 5502 | opt_ptr_mat.ra, |
| | 5503 | opt_ty.optionalChild(zcu).abiSize(zcu), |
| | 5504 | ); |
| | 5505 | try opt_ptr_mat.finish(isel); |
| | 5506 | try isel.emit(.movz(has_value_ra.w(), 1, .{ .lsl = .@"0" })); |
| | 5507 | } |
| | 5508 | try payload_ptr_vi.value.move(isel, ty_op.operand); |
| | 5509 | } |
| | 5510 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| | 5511 | }, |
| 5432 | .wrap_optional => { | 5512 | .wrap_optional => { |
| 5433 | if (isel.live_values.fetchRemove(air.inst_index)) |opt_vi| unused: { | 5513 | if (isel.live_values.fetchRemove(air.inst_index)) |opt_vi| unused: { |
| 5434 | defer opt_vi.value.deref(isel); | 5514 | defer opt_vi.value.deref(isel); |
| ... | @@ -5486,6 +5566,93 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, | ... | @@ -5486,6 +5566,93 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 5486 | } | 5566 | } |
| 5487 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; | 5567 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 5488 | }, | 5568 | }, |
| | 5569 | .unwrap_errunion_payload_ptr => { |
| | 5570 | if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| unused: { |
| | 5571 | defer payload_ptr_vi.value.deref(isel); |
| | 5572 | const ty_op = air.data(air.inst_index).ty_op; |
| | 5573 | switch (codegen.errUnionPayloadOffset(ty_op.ty.toType().childType(zcu), zcu)) { |
| | 5574 | 0 => try payload_ptr_vi.value.move(isel, ty_op.operand), |
| | 5575 | else => |payload_offset| { |
| | 5576 | const payload_ptr_ra = try payload_ptr_vi.value.defReg(isel) orelse break :unused; |
| | 5577 | const error_union_ptr_vi = try isel.use(ty_op.operand); |
| | 5578 | const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel); |
| | 5579 | const lo12: u12 = @truncate(payload_offset >> 0); |
| | 5580 | const hi12: u12 = @intCast(payload_offset >> 12); |
| | 5581 | if (hi12 > 0) try isel.emit(.add( |
| | 5582 | payload_ptr_ra.x(), |
| | 5583 | if (lo12 > 0) payload_ptr_ra.x() else error_union_ptr_mat.ra.x(), |
| | 5584 | .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } }, |
| | 5585 | )); |
| | 5586 | if (lo12 > 0) try isel.emit(.add(payload_ptr_ra.x(), error_union_ptr_mat.ra.x(), .{ .immediate = lo12 })); |
| | 5587 | try error_union_ptr_mat.finish(isel); |
| | 5588 | }, |
| | 5589 | } |
| | 5590 | } |
| | 5591 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| | 5592 | }, |
| | 5593 | .unwrap_errunion_err_ptr => { |
| | 5594 | if (isel.live_values.fetchRemove(air.inst_index)) |error_ptr_vi| unused: { |
| | 5595 | defer error_ptr_vi.value.deref(isel); |
| | 5596 | const ty_op = air.data(air.inst_index).ty_op; |
| | 5597 | switch (codegen.errUnionErrorOffset( |
| | 5598 | isel.air.typeOf(ty_op.operand, ip).childType(zcu).errorUnionPayload(zcu), |
| | 5599 | zcu, |
| | 5600 | )) { |
| | 5601 | 0 => try error_ptr_vi.value.move(isel, ty_op.operand), |
| | 5602 | else => |error_offset| { |
| | 5603 | const error_ptr_ra = try error_ptr_vi.value.defReg(isel) orelse break :unused; |
| | 5604 | const error_union_ptr_vi = try isel.use(ty_op.operand); |
| | 5605 | const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel); |
| | 5606 | const lo12: u12 = @truncate(error_offset >> 0); |
| | 5607 | const hi12: u12 = @intCast(error_offset >> 12); |
| | 5608 | if (hi12 > 0) try isel.emit(.add( |
| | 5609 | error_ptr_ra.x(), |
| | 5610 | if (lo12 > 0) error_ptr_ra.x() else error_union_ptr_mat.ra.x(), |
| | 5611 | .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } }, |
| | 5612 | )); |
| | 5613 | if (lo12 > 0) try isel.emit(.add(error_ptr_ra.x(), error_union_ptr_mat.ra.x(), .{ .immediate = lo12 })); |
| | 5614 | try error_union_ptr_mat.finish(isel); |
| | 5615 | }, |
| | 5616 | } |
| | 5617 | } |
| | 5618 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| | 5619 | }, |
| | 5620 | .errunion_payload_ptr_set => { |
| | 5621 | if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| unused: { |
| | 5622 | defer payload_ptr_vi.value.deref(isel); |
| | 5623 | const ty_op = air.data(air.inst_index).ty_op; |
| | 5624 | const payload_ty = ty_op.ty.toType().childType(zcu); |
| | 5625 | const error_union_ty = isel.air.typeOf(ty_op.operand, ip).childType(zcu); |
| | 5626 | const error_set_size = error_union_ty.errorUnionSet(zcu).abiSize(zcu); |
| | 5627 | const error_union_ptr_vi = try isel.use(ty_op.operand); |
| | 5628 | const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel); |
| | 5629 | if (error_set_size > 0) try isel.storeReg( |
| | 5630 | .zr, |
| | 5631 | error_set_size, |
| | 5632 | error_union_ptr_mat.ra, |
| | 5633 | codegen.errUnionErrorOffset(payload_ty, zcu), |
| | 5634 | ); |
| | 5635 | switch (codegen.errUnionPayloadOffset(payload_ty, zcu)) { |
| | 5636 | 0 => { |
| | 5637 | try error_union_ptr_mat.finish(isel); |
| | 5638 | try payload_ptr_vi.value.move(isel, ty_op.operand); |
| | 5639 | }, |
| | 5640 | else => |payload_offset| { |
| | 5641 | const payload_ptr_ra = try payload_ptr_vi.value.defReg(isel) orelse break :unused; |
| | 5642 | const lo12: u12 = @truncate(payload_offset >> 0); |
| | 5643 | const hi12: u12 = @intCast(payload_offset >> 12); |
| | 5644 | if (hi12 > 0) try isel.emit(.add( |
| | 5645 | payload_ptr_ra.x(), |
| | 5646 | if (lo12 > 0) payload_ptr_ra.x() else error_union_ptr_mat.ra.x(), |
| | 5647 | .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } }, |
| | 5648 | )); |
| | 5649 | if (lo12 > 0) try isel.emit(.add(payload_ptr_ra.x(), error_union_ptr_mat.ra.x(), .{ .immediate = lo12 })); |
| | 5650 | try error_union_ptr_mat.finish(isel); |
| | 5651 | }, |
| | 5652 | } |
| | 5653 | } |
| | 5654 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| | 5655 | }, |
| 5489 | .wrap_errunion_payload => { | 5656 | .wrap_errunion_payload => { |
| 5490 | if (isel.live_values.fetchRemove(air.inst_index)) |error_union_vi| { | 5657 | if (isel.live_values.fetchRemove(air.inst_index)) |error_union_vi| { |
| 5491 | defer error_union_vi.value.deref(isel); | 5658 | defer error_union_vi.value.deref(isel); |
| ... | @@ -5672,6 +5839,32 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, | ... | @@ -5672,6 +5839,32 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 5672 | } | 5839 | } |
| 5673 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; | 5840 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 5674 | }, | 5841 | }, |
| | 5842 | .set_union_tag => { |
| | 5843 | const bin_op = air.data(air.inst_index).bin_op; |
| | 5844 | const union_ty = isel.air.typeOf(bin_op.lhs, ip).childType(zcu); |
| | 5845 | const union_layout = union_ty.unionGetLayout(zcu); |
| | 5846 | const tag_vi = try isel.use(bin_op.rhs); |
| | 5847 | const union_ptr_vi = try isel.use(bin_op.lhs); |
| | 5848 | const union_ptr_mat = try union_ptr_vi.matReg(isel); |
| | 5849 | try tag_vi.store(isel, isel.air.typeOf(bin_op.rhs, ip), union_ptr_mat.ra, .{ |
| | 5850 | .offset = union_layout.tagOffset(), |
| | 5851 | }); |
| | 5852 | try union_ptr_mat.finish(isel); |
| | 5853 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| | 5854 | }, |
| | 5855 | .get_union_tag => { |
| | 5856 | if (isel.live_values.fetchRemove(air.inst_index)) |tag_vi| { |
| | 5857 | defer tag_vi.value.deref(isel); |
| | 5858 | const ty_op = air.data(air.inst_index).ty_op; |
| | 5859 | const union_ty = isel.air.typeOf(ty_op.operand, ip); |
| | 5860 | const union_layout = union_ty.unionGetLayout(zcu); |
| | 5861 | const union_vi = try isel.use(ty_op.operand); |
| | 5862 | var tag_part_it = union_vi.field(union_ty, union_layout.tagOffset(), union_layout.tag_size); |
| | 5863 | const tag_part_vi = try tag_part_it.only(isel); |
| | 5864 | try tag_vi.value.copy(isel, ty_op.ty.toType(), tag_part_vi.?); |
| | 5865 | } |
| | 5866 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| | 5867 | }, |
| 5675 | .slice => { | 5868 | .slice => { |
| 5676 | if (isel.live_values.fetchRemove(air.inst_index)) |slice_vi| { | 5869 | if (isel.live_values.fetchRemove(air.inst_index)) |slice_vi| { |
| 5677 | defer slice_vi.value.deref(isel); | 5870 | defer slice_vi.value.deref(isel); |
| ... | @@ -6541,8 +6734,8 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, | ... | @@ -6541,8 +6734,8 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 6541 | if (ptr_part_ra == null and len_part_ra == null) break :unused; | 6734 | if (ptr_part_ra == null and len_part_ra == null) break :unused; |
| 6542 | | 6735 | |
| 6543 | const un_op = air.data(air.inst_index).un_op; | 6736 | const un_op = air.data(air.inst_index).un_op; |
| 6544 | const err_vi = try isel.use(un_op); | 6737 | const error_vi = try isel.use(un_op); |
| 6545 | const err_mat = try err_vi.matReg(isel); | 6738 | const error_mat = try error_vi.matReg(isel); |
| 6546 | const ptr_ra = try isel.allocIntReg(); | 6739 | const ptr_ra = try isel.allocIntReg(); |
| 6547 | defer isel.freeReg(ptr_ra); | 6740 | defer isel.freeReg(ptr_ra); |
| 6548 | const start_ra, const end_ra = range_ras: { | 6741 | const start_ra, const end_ra = range_ras: { |
| ... | @@ -6573,7 +6766,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, | ... | @@ -6573,7 +6766,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 6573 | if (len_part_ra) |_| try isel.emit(.sub(end_ra.w(), end_ra.w(), .{ .immediate = 1 })); | 6766 | if (len_part_ra) |_| try isel.emit(.sub(end_ra.w(), end_ra.w(), .{ .immediate = 1 })); |
| 6574 | try isel.emit(.ldp(start_ra.w(), end_ra.w(), .{ .base = start_ra.x() })); | 6767 | try isel.emit(.ldp(start_ra.w(), end_ra.w(), .{ .base = start_ra.x() })); |
| 6575 | try isel.emit(.add(start_ra.x(), ptr_ra.x(), .{ .extended_register = .{ | 6768 | try isel.emit(.add(start_ra.x(), ptr_ra.x(), .{ .extended_register = .{ |
| 6576 | .register = err_mat.ra.w(), | 6769 | .register = error_mat.ra.w(), |
| 6577 | .extend = switch (zcu.errorSetBits()) { | 6770 | .extend = switch (zcu.errorSetBits()) { |
| 6578 | else => unreachable, | 6771 | else => unreachable, |
| 6579 | 1...8 => .{ .uxtb = 2 }, | 6772 | 1...8 => .{ .uxtb = 2 }, |
| ... | @@ -6591,7 +6784,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, | ... | @@ -6591,7 +6784,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 6591 | .reloc = .{ .label = @intCast(isel.instructions.items.len) }, | 6784 | .reloc = .{ .label = @intCast(isel.instructions.items.len) }, |
| 6592 | }); | 6785 | }); |
| 6593 | try isel.emit(.adrp(ptr_ra.x(), 0)); | 6786 | try isel.emit(.adrp(ptr_ra.x(), 0)); |
| 6594 | try err_mat.finish(isel); | 6787 | try error_mat.finish(isel); |
| 6595 | } | 6788 | } |
| 6596 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; | 6789 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 6597 | }, | 6790 | }, |
| ... | @@ -6893,11 +7086,11 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, | ... | @@ -6893,11 +7086,11 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 6893 | try isel.emit(.csinc(is_ra.w(), .wzr, .wzr, .invert(.ls))); | 7086 | try isel.emit(.csinc(is_ra.w(), .wzr, .wzr, .invert(.ls))); |
| 6894 | | 7087 | |
| 6895 | const un_op = air.data(air.inst_index).un_op; | 7088 | const un_op = air.data(air.inst_index).un_op; |
| 6896 | const err_vi = try isel.use(un_op); | 7089 | const error_vi = try isel.use(un_op); |
| 6897 | const err_mat = try err_vi.matReg(isel); | 7090 | const error_mat = try error_vi.matReg(isel); |
| 6898 | const ptr_ra = try isel.allocIntReg(); | 7091 | const ptr_ra = try isel.allocIntReg(); |
| 6899 | defer isel.freeReg(ptr_ra); | 7092 | defer isel.freeReg(ptr_ra); |
| 6900 | try isel.emit(.subs(.wzr, err_mat.ra.w(), .{ .register = ptr_ra.w() })); | 7093 | try isel.emit(.subs(.wzr, error_mat.ra.w(), .{ .register = ptr_ra.w() })); |
| 6901 | try isel.lazy_relocs.append(gpa, .{ | 7094 | try isel.lazy_relocs.append(gpa, .{ |
| 6902 | .symbol = .{ .kind = .const_data, .ty = .anyerror_type }, | 7095 | .symbol = .{ .kind = .const_data, .ty = .anyerror_type }, |
| 6903 | .reloc = .{ .label = @intCast(isel.instructions.items.len) }, | 7096 | .reloc = .{ .label = @intCast(isel.instructions.items.len) }, |
| ... | @@ -6908,7 +7101,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, | ... | @@ -6908,7 +7101,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 6908 | .reloc = .{ .label = @intCast(isel.instructions.items.len) }, | 7101 | .reloc = .{ .label = @intCast(isel.instructions.items.len) }, |
| 6909 | }); | 7102 | }); |
| 6910 | try isel.emit(.adrp(ptr_ra.x(), 0)); | 7103 | try isel.emit(.adrp(ptr_ra.x(), 0)); |
| 6911 | try err_mat.finish(isel); | 7104 | try error_mat.finish(isel); |
| 6912 | } | 7105 | } |
| 6913 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; | 7106 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 6914 | }, | 7107 | }, |
| ... | @@ -9529,8 +9722,14 @@ pub const Value = struct { | ... | @@ -9529,8 +9722,14 @@ pub const Value = struct { |
| 9529 | } }, | 9722 | } }, |
| 9530 | }, | 9723 | }, |
| 9531 | .struct_type => { | 9724 | .struct_type => { |
| 9532 | const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0; | | |
| 9533 | const loaded_struct = ip.loadStructType(ty.toIntern()); | 9725 | const loaded_struct = ip.loadStructType(ty.toIntern()); |
| | 9726 | switch (loaded_struct.layout) { |
| | 9727 | .auto, .@"extern" => {}, |
| | 9728 | .@"packed" => continue :type_key .{ |
| | 9729 | .int_type = ip.indexToKey(loaded_struct.backingIntTypeUnordered(ip)).int_type, |
| | 9730 | }, |
| | 9731 | } |
| | 9732 | const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0; |
| 9534 | if (loaded_struct.field_types.len > Value.max_parts and | 9733 | if (loaded_struct.field_types.len > Value.max_parts and |
| 9535 | (std.math.divCeil(u64, size, @as(u64, 1) << min_part_log2_stride) catch unreachable) > Value.max_parts) | 9734 | (std.math.divCeil(u64, size, @as(u64, 1) << min_part_log2_stride) catch unreachable) > Value.max_parts) |
| 9536 | return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}); | 9735 | return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}); |
| ... | @@ -9638,6 +9837,77 @@ pub const Value = struct { | ... | @@ -9638,6 +9837,77 @@ pub const Value = struct { |
| 9638 | if (part.is_vector) subpart_vi.setIsVector(isel); | 9837 | if (part.is_vector) subpart_vi.setIsVector(isel); |
| 9639 | } | 9838 | } |
| 9640 | }, | 9839 | }, |
| | 9840 | .union_type => { |
| | 9841 | const loaded_union = ip.loadUnionType(ty.toIntern()); |
| | 9842 | switch (loaded_union.flagsUnordered(ip).layout) { |
| | 9843 | .auto, .@"extern" => {}, |
| | 9844 | .@"packed" => continue :type_key .{ .int_type = .{ |
| | 9845 | .signedness = .unsigned, |
| | 9846 | .bits = @intCast(ty.bitSize(zcu)), |
| | 9847 | } }, |
| | 9848 | } |
| | 9849 | const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0; |
| | 9850 | if ((std.math.divCeil(u64, size, @as(u64, 1) << min_part_log2_stride) catch unreachable) > Value.max_parts) |
| | 9851 | return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}); |
| | 9852 | const union_layout = ZigType.getUnionLayout(loaded_union, zcu); |
| | 9853 | const alignment = vi.alignment(isel); |
| | 9854 | const tag_offset = union_layout.tagOffset(); |
| | 9855 | const payload_offset = union_layout.payloadOffset(); |
| | 9856 | const Part = struct { offset: u64, size: u64, signedness: ?std.builtin.Signedness }; |
| | 9857 | var parts: [2]Part = undefined; |
| | 9858 | var parts_len: Value.PartsLen = 0; |
| | 9859 | var field_end: u64 = 0; |
| | 9860 | for (0..2) |field_index| { |
| | 9861 | const field: enum { tag, payload } = switch (field_index) { |
| | 9862 | 0 => if (tag_offset < payload_offset) .tag else .payload, |
| | 9863 | 1 => if (tag_offset < payload_offset) .payload else .tag, |
| | 9864 | else => unreachable, |
| | 9865 | }; |
| | 9866 | const field_size, const field_begin = switch (field) { |
| | 9867 | .tag => .{ union_layout.tag_size, tag_offset }, |
| | 9868 | .payload => .{ union_layout.payload_size, payload_offset }, |
| | 9869 | }; |
| | 9870 | if (field_begin >= offset + size) break; |
| | 9871 | if (field_size == 0) continue; |
| | 9872 | field_end = field_begin + field_size; |
| | 9873 | if (field_end <= offset) continue; |
| | 9874 | const field_signedness = field_signedness: switch (field) { |
| | 9875 | .tag => { |
| | 9876 | if (offset >= field_begin and offset + size <= field_begin + field_size) { |
| | 9877 | ty = .fromInterned(loaded_union.enum_tag_ty); |
| | 9878 | ty_size = field_size; |
| | 9879 | offset -= field_begin; |
| | 9880 | continue :type_key ip.indexToKey(loaded_union.enum_tag_ty); |
| | 9881 | } |
| | 9882 | break :field_signedness ip.indexToKey(loaded_union.loadTagType(ip).tag_ty).int_type.signedness; |
| | 9883 | }, |
| | 9884 | .payload => null, |
| | 9885 | }; |
| | 9886 | if (parts_len > 0) combine: { |
| | 9887 | const prev_part = &parts[parts_len - 1]; |
| | 9888 | const combined_size = field_end - prev_part.offset; |
| | 9889 | if (combined_size > @as(u64, 1) << @min( |
| | 9890 | min_part_log2_stride, |
| | 9891 | alignment.toLog2Units(), |
| | 9892 | @ctz(prev_part.offset), |
| | 9893 | )) break :combine; |
| | 9894 | prev_part.size = combined_size; |
| | 9895 | prev_part.signedness = null; |
| | 9896 | continue; |
| | 9897 | } |
| | 9898 | parts[parts_len] = .{ |
| | 9899 | .offset = field_begin, |
| | 9900 | .size = field_size, |
| | 9901 | .signedness = field_signedness, |
| | 9902 | }; |
| | 9903 | parts_len += 1; |
| | 9904 | } |
| | 9905 | vi.setParts(isel, parts_len); |
| | 9906 | for (parts[0..parts_len]) |part| { |
| | 9907 | const subpart_vi = vi.addPart(isel, part.offset - offset, part.size); |
| | 9908 | if (part.signedness) |signedness| subpart_vi.setSignedness(isel, signedness); |
| | 9909 | } |
| | 9910 | }, |
| 9641 | .opaque_type, .func_type => continue :type_key .{ .simple_type = .anyopaque }, | 9911 | .opaque_type, .func_type => continue :type_key .{ .simple_type = .anyopaque }, |
| 9642 | .enum_type => continue :type_key ip.indexToKey(ip.loadEnumType(ty.toIntern()).tag_ty), | 9912 | .enum_type => continue :type_key ip.indexToKey(ip.loadEnumType(ty.toIntern()).tag_ty), |
| 9643 | .error_set_type, | 9913 | .error_set_type, |
| ... | @@ -10075,7 +10345,11 @@ pub const Value = struct { | ... | @@ -10075,7 +10345,11 @@ pub const Value = struct { |
| 10075 | }; | 10345 | }; |
| 10076 | }, | 10346 | }, |
| 10077 | .slice => |slice| switch (offset) { | 10347 | .slice => |slice| switch (offset) { |
| 10078 | 0 => continue :constant_key .{ .ptr = ip.indexToKey(slice.ptr).ptr }, | 10348 | 0 => continue :constant_key switch (ip.indexToKey(slice.ptr)) { |
| | 10349 | else => unreachable, |
| | 10350 | .undef => |undef| .{ .undef = undef }, |
| | 10351 | .ptr => |ptr| .{ .ptr = ptr }, |
| | 10352 | }, |
| 10079 | else => { | 10353 | else => { |
| 10080 | assert(offset == @divExact(isel.target.ptrBitWidth(), 8)); | 10354 | assert(offset == @divExact(isel.target.ptrBitWidth(), 8)); |
| 10081 | offset = 0; | 10355 | offset = 0; |
| ... | @@ -11128,16 +11402,14 @@ pub const CallAbiIterator = struct { | ... | @@ -11128,16 +11402,14 @@ pub const CallAbiIterator = struct { |
| 11128 | { | 11402 | { |
| 11129 | const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type); | 11403 | const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type); |
| 11130 | const offset = codegen.errUnionErrorOffset(payload_ty, zcu); | 11404 | const offset = codegen.errUnionErrorOffset(payload_ty, zcu); |
| 11131 | const size = error_set_ty.abiSize(zcu); | 11405 | const end = offset % 8 + error_set_ty.abiSize(zcu); |
| 11132 | const end = offset % 8 + size; | | |
| 11133 | const part_index: usize = @intCast(offset / 8); | 11406 | const part_index: usize = @intCast(offset / 8); |
| 11134 | sizes[part_index] = @max(sizes[part_index], @min(end, 8)); | 11407 | sizes[part_index] = @max(sizes[part_index], @min(end, 8)); |
| 11135 | if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8); | 11408 | if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8); |
| 11136 | } | 11409 | } |
| 11137 | { | 11410 | { |
| 11138 | const offset = codegen.errUnionPayloadOffset(payload_ty, zcu); | 11411 | const offset = codegen.errUnionPayloadOffset(payload_ty, zcu); |
| 11139 | const size = payload_ty.abiSize(zcu); | 11412 | const end = offset % 8 + payload_ty.abiSize(zcu); |
| 11140 | const end = offset % 8 + size; | | |
| 11141 | const part_index: usize = @intCast(offset / 8); | 11413 | const part_index: usize = @intCast(offset / 8); |
| 11142 | sizes[part_index] = @max(sizes[part_index], @min(end, 8)); | 11414 | sizes[part_index] = @max(sizes[part_index], @min(end, 8)); |
| 11143 | if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8); | 11415 | if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8); |
| ... | @@ -11181,8 +11453,14 @@ pub const CallAbiIterator = struct { | ... | @@ -11181,8 +11453,14 @@ pub const CallAbiIterator = struct { |
| 11181 | => unreachable, | 11453 | => unreachable, |
| 11182 | }, | 11454 | }, |
| 11183 | .struct_type => { | 11455 | .struct_type => { |
| 11184 | const size = wip_vi.size(isel); | | |
| 11185 | const loaded_struct = ip.loadStructType(ty.toIntern()); | 11456 | const loaded_struct = ip.loadStructType(ty.toIntern()); |
| | 11457 | switch (loaded_struct.layout) { |
| | 11458 | .auto, .@"extern" => {}, |
| | 11459 | .@"packed" => continue :type_key .{ |
| | 11460 | .int_type = ip.indexToKey(loaded_struct.backingIntTypeUnordered(ip)).int_type, |
| | 11461 | }, |
| | 11462 | } |
| | 11463 | const size = wip_vi.size(isel); |
| 11186 | if (size <= 16 * 4) homogeneous_aggregate: { | 11464 | if (size <= 16 * 4) homogeneous_aggregate: { |
| 11187 | const fdt = homogeneousStructBaseType(zcu, &loaded_struct) orelse break :homogeneous_aggregate; | 11465 | const fdt = homogeneousStructBaseType(zcu, &loaded_struct) orelse break :homogeneous_aggregate; |
| 11188 | const parts_len = @shrExact(size, fdt.log2Size()); | 11466 | const parts_len = @shrExact(size, fdt.log2Size()); |
| ... | @@ -11267,6 +11545,40 @@ pub const CallAbiIterator = struct { | ... | @@ -11267,6 +11545,40 @@ pub const CallAbiIterator = struct { |
| 11267 | else => it.indirect(isel, wip_vi), | 11545 | else => it.indirect(isel, wip_vi), |
| 11268 | } | 11546 | } |
| 11269 | }, | 11547 | }, |
| | 11548 | .union_type => { |
| | 11549 | const loaded_union = ip.loadUnionType(ty.toIntern()); |
| | 11550 | switch (loaded_union.flagsUnordered(ip).layout) { |
| | 11551 | .auto, .@"extern" => {}, |
| | 11552 | .@"packed" => continue :type_key .{ .int_type = .{ |
| | 11553 | .signedness = .unsigned, |
| | 11554 | .bits = @intCast(ty.bitSize(zcu)), |
| | 11555 | } }, |
| | 11556 | } |
| | 11557 | switch (wip_vi.size(isel)) { |
| | 11558 | 0 => unreachable, |
| | 11559 | 1...8 => it.integer(isel, wip_vi), |
| | 11560 | 9...16 => { |
| | 11561 | const union_layout = ZigType.getUnionLayout(loaded_union, zcu); |
| | 11562 | var sizes: [2]u64 = @splat(0); |
| | 11563 | { |
| | 11564 | const offset = union_layout.tagOffset(); |
| | 11565 | const end = offset % 8 + union_layout.tag_size; |
| | 11566 | const part_index: usize = @intCast(offset / 8); |
| | 11567 | sizes[part_index] = @max(sizes[part_index], @min(end, 8)); |
| | 11568 | if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8); |
| | 11569 | } |
| | 11570 | { |
| | 11571 | const offset = union_layout.payloadOffset(); |
| | 11572 | const end = offset % 8 + union_layout.payload_size; |
| | 11573 | const part_index: usize = @intCast(offset / 8); |
| | 11574 | sizes[part_index] = @max(sizes[part_index], @min(end, 8)); |
| | 11575 | if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8); |
| | 11576 | } |
| | 11577 | it.integers(isel, wip_vi, sizes); |
| | 11578 | }, |
| | 11579 | else => it.indirect(isel, wip_vi), |
| | 11580 | } |
| | 11581 | }, |
| 11270 | .opaque_type, .func_type => continue :type_key .{ .simple_type = .anyopaque }, | 11582 | .opaque_type, .func_type => continue :type_key .{ .simple_type = .anyopaque }, |
| 11271 | .enum_type => continue :type_key ip.indexToKey(ip.loadEnumType(ty.toIntern()).tag_ty), | 11583 | .enum_type => continue :type_key ip.indexToKey(ip.loadEnumType(ty.toIntern()).tag_ty), |
| 11272 | .error_set_type, | 11584 | .error_set_type, |
Jacob Young <jacobly@ziglang.org> 2025-07-26 21:39:43-04:00
src