| ... | @@ -709,22 +709,25 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif | ... | @@ -709,22 +709,25 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif |
| 709 | .multiple_llvm_types => { | 709 | .multiple_llvm_types => { |
| 710 | const arg = args[it.zig_index - 1]; | 710 | const arg = args[it.zig_index - 1]; |
| 711 | const param_ty = self.typeOf(arg); | 711 | const param_ty = self.typeOf(arg); |
| 712 | const llvm_types = it.types_buffer[0..it.types_len]; | | |
| 713 | const llvm_arg = try self.resolveInst(arg); | 712 | const llvm_arg = try self.resolveInst(arg); |
| 714 | const is_by_ref = isByRef(param_ty, zcu); | 713 | const is_by_ref = isByRef(param_ty, zcu); |
| 715 | const arg_ptr = if (is_by_ref) llvm_arg else ptr: { | 714 | const param_alignment = param_ty.abiAlignment(zcu); |
| 716 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | 715 | const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8); |
| 717 | const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment); | 716 | const arg_ptr = try self.buildAlloca(llvm_ty, param_alignment.toLlvm()); |
| 718 | _ = try self.wip.store(.normal, llvm_arg, ptr, alignment); | 717 | if (is_by_ref) _ = try self.wip.callMemCpy( |
| 719 | break :ptr ptr; | 718 | arg_ptr, |
| 720 | }; | 719 | param_alignment.toLlvm(), |
| | 720 | llvm_arg, |
| | 721 | param_alignment.toLlvm(), |
| | 722 | try o.builder.intValue(try o.lowerType(.usize), param_ty.abiSize(zcu)), |
| | 723 | .normal, |
| | 724 | self.disable_intrinsics, |
| | 725 | ) else _ = try self.wip.store(.normal, llvm_arg, arg_ptr, param_alignment.toLlvm()); |
| 721 | | 726 | |
| 722 | const llvm_ty = try o.builder.structType(.normal, llvm_types); | | |
| 723 | try llvm_args.ensureUnusedCapacity(it.types_len); | 727 | try llvm_args.ensureUnusedCapacity(it.types_len); |
| 724 | for (llvm_types, 0..) |field_ty, i| { | 728 | for (it.types_buffer[0..it.types_len], it.offsets_buffer[0..it.types_len]) |field_ty, offset| { |
| 725 | const alignment: Builder.Alignment = .fromByteUnits(@divExact(target.ptrBitWidth(), 8)); | 729 | const field_ptr = try self.ptraddConst(arg_ptr, offset); |
| 726 | const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, ""); | 730 | const loaded = try self.wip.load(.normal, field_ty, field_ptr, param_alignment.offset(offset).toLlvm(), ""); |
| 727 | const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, ""); | | |
| 728 | llvm_args.appendAssumeCapacity(loaded); | 731 | llvm_args.appendAssumeCapacity(loaded); |
| 729 | } | 732 | } |
| 730 | }, | 733 | }, |
| ... | @@ -2269,10 +2272,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build | ... | @@ -2269,10 +2272,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build |
| 2269 | | 2272 | |
| 2270 | const struct_ptr_align = struct_ty.abiAlignment(zcu); | 2273 | const struct_ptr_align = struct_ty.abiAlignment(zcu); |
| 2271 | const field_ptr = try self.ptraddConst(struct_llvm_val, offset); | 2274 | const field_ptr = try self.ptraddConst(struct_llvm_val, offset); |
| 2272 | const field_ptr_align: InternPool.Alignment = switch (offset) { | 2275 | const field_ptr_align = struct_ptr_align.offset(offset); |
| 2273 | 0 => struct_ptr_align, | | |
| 2274 | else => struct_ptr_align.minStrict(.fromLog2Units(@ctz(offset))), | | |
| 2275 | }; | | |
| 2276 | | 2276 | |
| 2277 | if (isByRef(field_ty, zcu)) { | 2277 | if (isByRef(field_ty, zcu)) { |
| 2278 | return self.loadByRef(field_ptr, field_ty, field_ptr_align.toLlvm(), .normal); | 2278 | return self.loadByRef(field_ptr, field_ty, field_ptr_align.toLlvm(), .normal); |
| ... | @@ -3120,11 +3120,7 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er | ... | @@ -3120,11 +3120,7 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er |
| 3120 | | 3120 | |
| 3121 | const field_ty = struct_ty.fieldType(field_index, zcu); | 3121 | const field_ty = struct_ty.fieldType(field_index, zcu); |
| 3122 | const field_offset = struct_ty.structFieldOffset(field_index, zcu); | 3122 | const field_offset = struct_ty.structFieldOffset(field_index, zcu); |
| 3123 | const field_align = switch (field_offset) { | 3123 | const field_align = struct_ty.abiAlignment(zcu).offset(field_offset); |
| 3124 | 0 => struct_ty.abiAlignment(zcu), | | |
| 3125 | else => struct_ty.abiAlignment(zcu).minStrict(.fromLog2Units(@ctz(field_offset))), | | |
| 3126 | }; | | |
| 3127 | | | |
| 3128 | const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset); | 3124 | const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset); |
| 3129 | return self.load(field_ptr, field_ty, field_align.toLlvm(), .normal); | 3125 | return self.load(field_ptr, field_ty, field_align.toLlvm(), .normal); |
| 3130 | } | 3126 | } |
| ... | @@ -5279,10 +5275,7 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder. | ... | @@ -5279,10 +5275,7 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder. |
| 5279 | return .none; | 5275 | return .none; |
| 5280 | } | 5276 | } |
| 5281 | const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset()); | 5277 | const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset()); |
| 5282 | const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) { | 5278 | const tag_ptr_align = union_ptr_align.offset(layout.tagOffset()); |
| 5283 | 0 => union_ptr_align, | | |
| 5284 | else => |off| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))), | | |
| 5285 | }; | | |
| 5286 | _ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm()); | 5279 | _ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm()); |
| 5287 | return .none; | 5280 | return .none; |
| 5288 | } | 5281 | } |
| ... | @@ -5922,10 +5915,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde | ... | @@ -5922,10 +5915,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde |
| 5922 | if (!field_ty.hasRuntimeBits(zcu)) continue; | 5915 | if (!field_ty.hasRuntimeBits(zcu)) continue; |
| 5923 | const offset = result_ty.structFieldOffset(field_index, zcu); | 5916 | const offset = result_ty.structFieldOffset(field_index, zcu); |
| 5924 | const field_ptr = try self.ptraddConst(alloca_inst, offset); | 5917 | const field_ptr = try self.ptraddConst(alloca_inst, offset); |
| 5925 | const field_ptr_align: InternPool.Alignment = switch (offset) { | 5918 | const field_ptr_align = struct_align.offset(offset); |
| 5926 | 0 => struct_align, | | |
| 5927 | else => struct_align.minStrict(.fromLog2Units(@ctz(offset))), | | |
| 5928 | }; | | |
| 5929 | | 5919 | |
| 5930 | const llvm_field_val = try self.resolveInst(elem); | 5920 | const llvm_field_val = try self.resolveInst(elem); |
| 5931 | | 5921 | |
| ... | @@ -6581,6 +6571,7 @@ const ParamTypeIterator = struct { | ... | @@ -6581,6 +6571,7 @@ const ParamTypeIterator = struct { |
| 6581 | llvm_index: u32, | 6571 | llvm_index: u32, |
| 6582 | types_len: u32, | 6572 | types_len: u32, |
| 6583 | types_buffer: [8]Builder.Type, | 6573 | types_buffer: [8]Builder.Type, |
| | 6574 | offsets_buffer: [9]u64, |
| 6584 | byval_attr: bool, | 6575 | byval_attr: bool, |
| 6585 | | 6576 | |
| 6586 | const Lowering = union(enum) { | 6577 | const Lowering = union(enum) { |
| ... | @@ -6672,7 +6663,12 @@ const ParamTypeIterator = struct { | ... | @@ -6672,7 +6663,12 @@ const ParamTypeIterator = struct { |
| 6672 | .memory => return .byref_mut, | 6663 | .memory => return .byref_mut, |
| 6673 | .float_array => |len| return Lowering{ .float_array = len }, | 6664 | .float_array => |len| return Lowering{ .float_array = len }, |
| 6674 | .byval => return .byval, | 6665 | .byval => return .byval, |
| 6675 | .integer => return .abi_sized_int, | 6666 | .integer => { |
| | 6667 | it.types_len = 1; |
| | 6668 | it.types_buffer[0..1].* = .{.i64}; |
| | 6669 | it.offsets_buffer[0..2].* = .{ 0, 8 }; |
| | 6670 | return .multiple_llvm_types; |
| | 6671 | }, |
| 6676 | .double_integer => return Lowering{ .i64_array = 2 }, | 6672 | .double_integer => return Lowering{ .i64_array = 2 }, |
| 6677 | } | 6673 | } |
| 6678 | }, | 6674 | }, |
| ... | @@ -6711,12 +6707,17 @@ const ParamTypeIterator = struct { | ... | @@ -6711,12 +6707,17 @@ const ParamTypeIterator = struct { |
| 6711 | .double_integer => return Lowering{ .i64_array = 2 }, | 6707 | .double_integer => return Lowering{ .i64_array = 2 }, |
| 6712 | .fields => { | 6708 | .fields => { |
| 6713 | it.types_len = 0; | 6709 | it.types_len = 0; |
| | 6710 | var offset: u64 = 0; |
| 6714 | for (0..ty.structFieldCount(zcu)) |field_index| { | 6711 | for (0..ty.structFieldCount(zcu)) |field_index| { |
| 6715 | const field_ty = ty.fieldType(field_index, zcu); | 6712 | const field_ty = ty.fieldType(field_index, zcu); |
| 6716 | if (!field_ty.hasRuntimeBits(zcu)) continue; | 6713 | if (!field_ty.hasRuntimeBits(zcu)) continue; |
| | 6714 | offset = field_ty.abiAlignment(zcu).forward(offset); |
| 6717 | it.types_buffer[it.types_len] = try it.object.lowerType(field_ty); | 6715 | it.types_buffer[it.types_len] = try it.object.lowerType(field_ty); |
| | 6716 | it.offsets_buffer[it.types_len] = offset; |
| 6718 | it.types_len += 1; | 6717 | it.types_len += 1; |
| | 6718 | offset += field_ty.abiSize(zcu); |
| 6719 | } | 6719 | } |
| | 6720 | it.offsets_buffer[it.types_len] = offset; |
| 6720 | it.llvm_index += it.types_len - 1; | 6721 | it.llvm_index += it.types_len - 1; |
| 6721 | return .multiple_llvm_types; | 6722 | return .multiple_llvm_types; |
| 6722 | }, | 6723 | }, |
| ... | @@ -6729,9 +6730,8 @@ const ParamTypeIterator = struct { | ... | @@ -6729,9 +6730,8 @@ const ParamTypeIterator = struct { |
| 6729 | it.llvm_index += 1; | 6730 | it.llvm_index += 1; |
| 6730 | return .byval; | 6731 | return .byval; |
| 6731 | } else { | 6732 | } else { |
| 6732 | var types_buffer: [8]Builder.Type = undefined; | 6733 | it.types_buffer[0..1].* = .{try it.object.lowerType(scalar_ty)}; |
| 6733 | types_buffer[0] = try it.object.lowerType(scalar_ty); | 6734 | it.offsets_buffer[0..2].* = .{ 0, scalar_ty.abiSize(zcu) }; |
| 6734 | it.types_buffer = types_buffer; | | |
| 6735 | it.types_len = 1; | 6735 | it.types_len = 1; |
| 6736 | it.llvm_index += 1; | 6736 | it.llvm_index += 1; |
| 6737 | it.zig_index += 1; | 6737 | it.zig_index += 1; |
| ... | @@ -6804,31 +6804,36 @@ const ParamTypeIterator = struct { | ... | @@ -6804,31 +6804,36 @@ const ParamTypeIterator = struct { |
| 6804 | return .byval; | 6804 | return .byval; |
| 6805 | } | 6805 | } |
| 6806 | var types_index: u32 = 0; | 6806 | var types_index: u32 = 0; |
| 6807 | var types_buffer: [8]Builder.Type = undefined; | 6807 | var offset: u64 = 0; |
| 6808 | for (classes) |class| { | 6808 | for (classes) |class| { |
| 6809 | switch (class) { | 6809 | switch (class) { |
| 6810 | .integer => { | 6810 | .integer => { |
| 6811 | types_buffer[types_index] = .i64; | 6811 | it.types_buffer[types_index] = .i64; |
| | 6812 | it.offsets_buffer[types_index] = offset; |
| 6812 | types_index += 1; | 6813 | types_index += 1; |
| 6813 | }, | 6814 | }, |
| 6814 | .sse => { | 6815 | .sse => { |
| 6815 | types_buffer[types_index] = .double; | 6816 | it.types_buffer[types_index] = .double; |
| | 6817 | it.offsets_buffer[types_index] = offset; |
| 6816 | types_index += 1; | 6818 | types_index += 1; |
| 6817 | }, | 6819 | }, |
| 6818 | .sseup => { | 6820 | .sseup => { |
| 6819 | if (types_buffer[types_index - 1] == .double) { | 6821 | if (it.types_buffer[types_index - 1] == .double) { |
| 6820 | types_buffer[types_index - 1] = .fp128; | 6822 | it.types_buffer[types_index - 1] = .fp128; |
| 6821 | } else { | 6823 | } else { |
| 6822 | types_buffer[types_index] = .double; | 6824 | it.types_buffer[types_index] = .double; |
| | 6825 | it.offsets_buffer[types_index] = offset; |
| 6823 | types_index += 1; | 6826 | types_index += 1; |
| 6824 | } | 6827 | } |
| 6825 | }, | 6828 | }, |
| 6826 | .float => { | 6829 | .float => { |
| 6827 | types_buffer[types_index] = .float; | 6830 | it.types_buffer[types_index] = .float; |
| | 6831 | it.offsets_buffer[types_index] = offset; |
| 6828 | types_index += 1; | 6832 | types_index += 1; |
| 6829 | }, | 6833 | }, |
| 6830 | .float_combine => { | 6834 | .float_combine => { |
| 6831 | types_buffer[types_index] = try it.object.builder.vectorType(.normal, 2, .float); | 6835 | it.types_buffer[types_index] = try it.object.builder.vectorType(.normal, 2, .float); |
| | 6836 | it.offsets_buffer[types_index] = offset; |
| 6832 | types_index += 1; | 6837 | types_index += 1; |
| 6833 | }, | 6838 | }, |
| 6834 | .x87 => { | 6839 | .x87 => { |
| ... | @@ -6845,6 +6850,7 @@ const ParamTypeIterator = struct { | ... | @@ -6845,6 +6850,7 @@ const ParamTypeIterator = struct { |
| 6845 | @panic("TODO"); | 6850 | @panic("TODO"); |
| 6846 | }, | 6851 | }, |
| 6847 | } | 6852 | } |
| | 6853 | offset += 8; |
| 6848 | } | 6854 | } |
| 6849 | const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer}); | 6855 | const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer}); |
| 6850 | if (first_non_integer == null or classes[first_non_integer.?] == .none) { | 6856 | if (first_non_integer == null or classes[first_non_integer.?] == .none) { |
| ... | @@ -6865,15 +6871,15 @@ const ParamTypeIterator = struct { | ... | @@ -6865,15 +6871,15 @@ const ParamTypeIterator = struct { |
| 6865 | const size = ty.abiSize(zcu); | 6871 | const size = ty.abiSize(zcu); |
| 6866 | assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index); | 6872 | assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index); |
| 6867 | if (size % 8 > 0) { | 6873 | if (size % 8 > 0) { |
| 6868 | types_buffer[types_index - 1] = | 6874 | it.types_buffer[types_index - 1] = |
| 6869 | try it.object.builder.intType(@intCast(size % 8 * 8)); | 6875 | try it.object.builder.intType(@intCast(size % 8 * 8)); |
| 6870 | } | 6876 | } |
| 6871 | }, | 6877 | }, |
| 6872 | else => {}, | 6878 | else => {}, |
| 6873 | } | 6879 | } |
| 6874 | } | 6880 | } |
| | 6881 | it.offsets_buffer[types_index] = offset; |
| 6875 | it.types_len = types_index; | 6882 | it.types_len = types_index; |
| 6876 | it.types_buffer = types_buffer; | | |
| 6877 | it.llvm_index += types_index; | 6883 | it.llvm_index += types_index; |
| 6878 | it.zig_index += 1; | 6884 | it.zig_index += 1; |
| 6879 | return .multiple_llvm_types; | 6885 | return .multiple_llvm_types; |
| ... | @@ -6887,6 +6893,7 @@ pub fn iterateParamTypes(object: *Object, fn_info: InternPool.Key.FuncType) Para | ... | @@ -6887,6 +6893,7 @@ pub fn iterateParamTypes(object: *Object, fn_info: InternPool.Key.FuncType) Para |
| 6887 | .llvm_index = 0, | 6893 | .llvm_index = 0, |
| 6888 | .types_len = 0, | 6894 | .types_len = 0, |
| 6889 | .types_buffer = undefined, | 6895 | .types_buffer = undefined, |
| | 6896 | .offsets_buffer = undefined, |
| 6890 | .byval_attr = false, | 6897 | .byval_attr = false, |
| 6891 | }; | 6898 | }; |
| 6892 | } | 6899 | } |
| ... | @@ -6965,7 +6972,7 @@ pub fn lowerFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Erro | ... | @@ -6965,7 +6972,7 @@ pub fn lowerFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Erro |
| 6965 | .memory => return .void, | 6972 | .memory => return .void, |
| 6966 | .float_array => return o.lowerType(return_type), | 6973 | .float_array => return o.lowerType(return_type), |
| 6967 | .byval => return o.lowerType(return_type), | 6974 | .byval => return o.lowerType(return_type), |
| 6968 | .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))), | 6975 | .integer => return .i64, |
| 6969 | .double_integer => return o.builder.arrayType(2, .i64), | 6976 | .double_integer => return o.builder.arrayType(2, .i64), |
| 6970 | }, | 6977 | }, |
| 6971 | .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) { | 6978 | .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) { |
| ... | @@ -7285,11 +7292,7 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E | ... | @@ -7285,11 +7292,7 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E |
| 7285 | } | 7292 | } |
| 7286 | | 7293 | |
| 7287 | fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value { | 7294 | fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value { |
| 7288 | if (offset == 0) return ptr; | 7295 | return fg.object.ptraddConst(&fg.wip, ptr, offset); |
| 7289 | const o = fg.object; | | |
| 7290 | const llvm_usize_ty = try o.lowerType(.usize); | | |
| 7291 | const offset_val = try o.builder.intValue(llvm_usize_ty, offset); | | |
| 7292 | return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, ""); | | |
| 7293 | } | 7296 | } |
| 7294 | fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value { | 7297 | fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value { |
| 7295 | if (scale == 0) return ptr; | 7298 | if (scale == 0) return ptr; |