| ... | ... | @@ -630,6 +630,7 @@ ZigType *get_error_union_type(CodeGen *g, ZigType *err_set_type, ZigType *payloa |
| 630 | 630 | size_t field2_offset = next_field_offset(0, entry->abi_align, field_sizes[0], field_aligns[1]); |
| 631 | 631 | entry->abi_size = next_field_offset(field2_offset, entry->abi_align, field_sizes[1], entry->abi_align); |
| 632 | 632 | entry->size_in_bits = entry->abi_size * 8; |
| 633 | entry->data.error_union.pad_bytes = entry->abi_size - (field2_offset + field_sizes[1]); |
| 633 | 634 | } |
| 634 | 635 | |
| 635 | 636 | g->type_table.put(type_id, entry); |
| ... | ... | @@ -1499,7 +1500,7 @@ bool type_is_invalid(ZigType *type_entry) { |
| 1499 | 1500 | } |
| 1500 | 1501 | |
| 1501 | 1502 | |
| 1502 | | ZigType *get_struct_type(CodeGen *g, const char *type_name, const char *field_names[], |
| 1503 | static ZigType *get_struct_type(CodeGen *g, const char *type_name, const char *field_names[], |
| 1503 | 1504 | ZigType *field_types[], size_t field_count, unsigned min_abi_align) |
| 1504 | 1505 | { |
| 1505 | 1506 | ZigType *struct_type = new_type_table_entry(ZigTypeIdStruct); |
| ... | ... | @@ -1524,10 +1525,6 @@ ZigType *get_struct_type(CodeGen *g, const char *type_name, const char *field_na |
| 1524 | 1525 | if (field->type_entry->abi_align > abi_align) { |
| 1525 | 1526 | abi_align = field->type_entry->abi_align; |
| 1526 | 1527 | } |
| 1527 | | field->gen_index = struct_type->data.structure.gen_field_count; |
| 1528 | | struct_type->data.structure.gen_field_count += 1; |
| 1529 | | } else { |
| 1530 | | field->gen_index = SIZE_MAX; |
| 1531 | 1528 | } |
| 1532 | 1529 | |
| 1533 | 1530 | auto prev_entry = struct_type->data.structure.fields_by_name.put_unique(field->name, field); |
| ... | ... | @@ -1537,14 +1534,16 @@ ZigType *get_struct_type(CodeGen *g, const char *type_name, const char *field_na |
| 1537 | 1534 | size_t next_offset = 0; |
| 1538 | 1535 | for (size_t i = 0; i < field_count; i += 1) { |
| 1539 | 1536 | TypeStructField *field = &struct_type->data.structure.fields[i]; |
| 1540 | | if (field->gen_index == SIZE_MAX) |
| 1537 | if (!type_has_bits(field->type_entry)) |
| 1541 | 1538 | continue; |
| 1539 | |
| 1542 | 1540 | field->offset = next_offset; |
| 1541 | |
| 1542 | // find the next non-zero-byte field for offset calculations |
| 1543 | 1543 | size_t next_src_field_index = i + 1; |
| 1544 | 1544 | for (; next_src_field_index < field_count; next_src_field_index += 1) { |
| 1545 | | if (struct_type->data.structure.fields[next_src_field_index].gen_index != SIZE_MAX) { |
| 1545 | if (type_has_bits(struct_type->data.structure.fields[next_src_field_index].type_entry)) |
| 1546 | 1546 | break; |
| 1547 | | } |
| 1548 | 1547 | } |
| 1549 | 1548 | size_t next_abi_align = (next_src_field_index == field_count) ? |
| 1550 | 1549 | abi_align : struct_type->data.structure.fields[next_src_field_index].type_entry->abi_align; |
| ... | ... | @@ -5304,6 +5303,7 @@ static Error resolve_coro_frame(CodeGen *g, ZigType *frame_type) { |
| 5304 | 5303 | |
| 5305 | 5304 | for (size_t alloca_i = 0; alloca_i < fn->alloca_gen_list.length; alloca_i += 1) { |
| 5306 | 5305 | IrInstructionAllocaGen *instruction = fn->alloca_gen_list.at(alloca_i); |
| 5306 | instruction->field_index = SIZE_MAX; |
| 5307 | 5307 | ZigType *ptr_type = instruction->base.value.type; |
| 5308 | 5308 | assert(ptr_type->id == ZigTypeIdPointer); |
| 5309 | 5309 | ZigType *child_type = ptr_type->data.pointer.child_type; |
| ... | ... | @@ -5327,6 +5327,7 @@ static Error resolve_coro_frame(CodeGen *g, ZigType *frame_type) { |
| 5327 | 5327 | } else { |
| 5328 | 5328 | name = buf_ptr(buf_sprintf("%s.%" ZIG_PRI_usize, instruction->name_hint, alloca_i)); |
| 5329 | 5329 | } |
| 5330 | instruction->field_index = field_types.length; |
| 5330 | 5331 | field_names.append(name); |
| 5331 | 5332 | field_types.append(child_type); |
| 5332 | 5333 | } |
| ... | ... | @@ -5949,6 +5950,15 @@ ZigType *make_int_type(CodeGen *g, bool is_signed, uint32_t size_in_bits) { |
| 5949 | 5950 | entry->llvm_type = LLVMIntType(size_in_bits); |
| 5950 | 5951 | entry->abi_size = LLVMABISizeOfType(g->target_data_ref, entry->llvm_type); |
| 5951 | 5952 | entry->abi_align = LLVMABIAlignmentOfType(g->target_data_ref, entry->llvm_type); |
| 5953 | |
| 5954 | if (size_in_bits >= 128) { |
| 5955 | // Override the incorrect alignment reported by LLVM. Clang does this as well. |
| 5956 | // On x86_64 there are some instructions like CMPXCHG16B which require this. |
| 5957 | // On all targets, integers 128 bits and above have ABI alignment of 16. |
| 5958 | // See: https://github.com/ziglang/zig/issues/2987 |
| 5959 | assert(entry->abi_align == 8); // if this trips we can remove the workaround |
| 5960 | entry->abi_align = 16; |
| 5961 | } |
| 5952 | 5962 | } |
| 5953 | 5963 | |
| 5954 | 5964 | const char u_or_i = is_signed ? 'i' : 'u'; |
| ... | ... | @@ -6810,10 +6820,9 @@ static void resolve_llvm_types_struct(CodeGen *g, ZigType *struct_type, ResolveS |
| 6810 | 6820 | } |
| 6811 | 6821 | |
| 6812 | 6822 | size_t field_count = struct_type->data.structure.src_field_count; |
| 6813 | | size_t gen_field_count = struct_type->data.structure.gen_field_count; |
| 6814 | | LLVMTypeRef *element_types = allocate<LLVMTypeRef>(gen_field_count); |
| 6823 | // Every field could potentially have a generated padding field after it. |
| 6824 | LLVMTypeRef *element_types = allocate<LLVMTypeRef>(field_count * 2); |
| 6815 | 6825 | |
| 6816 | | size_t gen_field_index = 0; |
| 6817 | 6826 | bool packed = (struct_type->data.structure.layout == ContainerLayoutPacked); |
| 6818 | 6827 | size_t packed_bits_offset = 0; |
| 6819 | 6828 | size_t first_packed_bits_offset_misalign = SIZE_MAX; |
| ... | ... | @@ -6821,20 +6830,36 @@ static void resolve_llvm_types_struct(CodeGen *g, ZigType *struct_type, ResolveS |
| 6821 | 6830 | |
| 6822 | 6831 | // trigger all the recursive get_llvm_type calls |
| 6823 | 6832 | for (size_t i = 0; i < field_count; i += 1) { |
| 6824 | | TypeStructField *type_struct_field = &struct_type->data.structure.fields[i]; |
| 6825 | | ZigType *field_type = type_struct_field->type_entry; |
| 6833 | TypeStructField *field = &struct_type->data.structure.fields[i]; |
| 6834 | ZigType *field_type = field->type_entry; |
| 6826 | 6835 | if (!type_has_bits(field_type)) |
| 6827 | 6836 | continue; |
| 6828 | 6837 | (void)get_llvm_type(g, field_type); |
| 6829 | 6838 | if (struct_type->data.structure.resolve_status >= wanted_resolve_status) return; |
| 6830 | 6839 | } |
| 6831 | 6840 | |
| 6832 | | for (size_t i = 0; i < field_count; i += 1) { |
| 6833 | | TypeStructField *type_struct_field = &struct_type->data.structure.fields[i]; |
| 6834 | | ZigType *field_type = type_struct_field->type_entry; |
| 6841 | size_t gen_field_index = 0; |
| 6835 | 6842 | |
| 6843 | // Calculate what LLVM thinks the ABI align of the struct will be. We do this to avoid |
| 6844 | // inserting padding bytes where LLVM would do it automatically. |
| 6845 | size_t llvm_struct_abi_align = 0; |
| 6846 | for (size_t i = 0; i < field_count; i += 1) { |
| 6847 | ZigType *field_type = struct_type->data.structure.fields[i].type_entry; |
| 6836 | 6848 | if (!type_has_bits(field_type)) |
| 6837 | 6849 | continue; |
| 6850 | LLVMTypeRef field_llvm_type = get_llvm_type(g, field_type); |
| 6851 | size_t llvm_field_abi_align = LLVMABIAlignmentOfType(g->target_data_ref, field_llvm_type); |
| 6852 | llvm_struct_abi_align = max(llvm_struct_abi_align, llvm_field_abi_align); |
| 6853 | } |
| 6854 | |
| 6855 | for (size_t i = 0; i < field_count; i += 1) { |
| 6856 | TypeStructField *field = &struct_type->data.structure.fields[i]; |
| 6857 | ZigType *field_type = field->type_entry; |
| 6858 | |
| 6859 | if (!type_has_bits(field_type)) { |
| 6860 | field->gen_index = SIZE_MAX; |
| 6861 | continue; |
| 6862 | } |
| 6838 | 6863 | |
| 6839 | 6864 | if (packed) { |
| 6840 | 6865 | size_t field_size_in_bits = type_size_bits(g, field_type); |
| ... | ... | @@ -6871,11 +6896,44 @@ static void resolve_llvm_types_struct(CodeGen *g, ZigType *struct_type, ResolveS |
| 6871 | 6896 | llvm_type = get_llvm_type(g, field_type); |
| 6872 | 6897 | } |
| 6873 | 6898 | element_types[gen_field_index] = llvm_type; |
| 6874 | | |
| 6899 | field->gen_index = gen_field_index; |
| 6875 | 6900 | gen_field_index += 1; |
| 6901 | |
| 6902 | // find the next non-zero-byte field for offset calculations |
| 6903 | size_t next_src_field_index = i + 1; |
| 6904 | for (; next_src_field_index < field_count; next_src_field_index += 1) { |
| 6905 | if (type_has_bits(struct_type->data.structure.fields[next_src_field_index].type_entry)) |
| 6906 | break; |
| 6907 | } |
| 6908 | size_t next_abi_align = (next_src_field_index == field_count) ? |
| 6909 | struct_type->abi_align : |
| 6910 | struct_type->data.structure.fields[next_src_field_index].type_entry->abi_align; |
| 6911 | size_t llvm_next_abi_align = (next_src_field_index == field_count) ? |
| 6912 | llvm_struct_abi_align : |
| 6913 | LLVMABIAlignmentOfType(g->target_data_ref, |
| 6914 | get_llvm_type(g, struct_type->data.structure.fields[next_src_field_index].type_entry)); |
| 6915 | |
| 6916 | size_t next_offset = next_field_offset(field->offset, struct_type->abi_align, |
| 6917 | field_type->abi_size, next_abi_align); |
| 6918 | size_t llvm_next_offset = next_field_offset(field->offset, llvm_struct_abi_align, |
| 6919 | LLVMABISizeOfType(g->target_data_ref, llvm_type), llvm_next_abi_align); |
| 6920 | |
| 6921 | assert(next_offset >= llvm_next_offset); |
| 6922 | if (next_offset > llvm_next_offset) { |
| 6923 | size_t pad_bytes = next_offset - (field->offset + field_type->abi_size); |
| 6924 | if (pad_bytes != 0) { |
| 6925 | LLVMTypeRef pad_llvm_type = LLVMArrayType(LLVMInt8Type(), pad_bytes); |
| 6926 | element_types[gen_field_index] = pad_llvm_type; |
| 6927 | gen_field_index += 1; |
| 6928 | } |
| 6929 | } |
| 6876 | 6930 | } |
| 6877 | 6931 | debug_field_count += 1; |
| 6878 | 6932 | } |
| 6933 | if (!packed) { |
| 6934 | struct_type->data.structure.gen_field_count = gen_field_index; |
| 6935 | } |
| 6936 | |
| 6879 | 6937 | if (first_packed_bits_offset_misalign != SIZE_MAX) { |
| 6880 | 6938 | size_t full_bit_count = packed_bits_offset - first_packed_bits_offset_misalign; |
| 6881 | 6939 | size_t full_abi_size = get_abi_size_bytes(full_bit_count, g->pointer_size_bytes); |
| ... | ... | @@ -6884,19 +6942,20 @@ static void resolve_llvm_types_struct(CodeGen *g, ZigType *struct_type, ResolveS |
| 6884 | 6942 | } |
| 6885 | 6943 | |
| 6886 | 6944 | if (type_has_bits(struct_type)) { |
| 6887 | | LLVMStructSetBody(struct_type->llvm_type, element_types, (unsigned)gen_field_count, packed); |
| 6945 | LLVMStructSetBody(struct_type->llvm_type, element_types, |
| 6946 | (unsigned)struct_type->data.structure.gen_field_count, packed); |
| 6888 | 6947 | } |
| 6889 | 6948 | |
| 6890 | 6949 | ZigLLVMDIType **di_element_types = allocate<ZigLLVMDIType*>(debug_field_count); |
| 6891 | 6950 | size_t debug_field_index = 0; |
| 6892 | 6951 | for (size_t i = 0; i < field_count; i += 1) { |
| 6893 | | TypeStructField *type_struct_field = &struct_type->data.structure.fields[i]; |
| 6894 | | size_t gen_field_index = type_struct_field->gen_index; |
| 6952 | TypeStructField *field = &struct_type->data.structure.fields[i]; |
| 6953 | size_t gen_field_index = field->gen_index; |
| 6895 | 6954 | if (gen_field_index == SIZE_MAX) { |
| 6896 | 6955 | continue; |
| 6897 | 6956 | } |
| 6898 | 6957 | |
| 6899 | | ZigType *field_type = type_struct_field->type_entry; |
| 6958 | ZigType *field_type = field->type_entry; |
| 6900 | 6959 | |
| 6901 | 6960 | // if the field is a function, actually the debug info should be a pointer. |
| 6902 | 6961 | ZigLLVMDIType *field_di_type; |
| ... | ... | @@ -6914,13 +6973,13 @@ static void resolve_llvm_types_struct(CodeGen *g, ZigType *struct_type, ResolveS |
| 6914 | 6973 | uint64_t debug_align_in_bits; |
| 6915 | 6974 | uint64_t debug_offset_in_bits; |
| 6916 | 6975 | if (packed) { |
| 6917 | | debug_size_in_bits = type_struct_field->type_entry->size_in_bits; |
| 6918 | | debug_align_in_bits = 8 * type_struct_field->type_entry->abi_align; |
| 6919 | | debug_offset_in_bits = 8 * type_struct_field->offset + type_struct_field->bit_offset_in_host; |
| 6976 | debug_size_in_bits = field->type_entry->size_in_bits; |
| 6977 | debug_align_in_bits = 8 * field->type_entry->abi_align; |
| 6978 | debug_offset_in_bits = 8 * field->offset + field->bit_offset_in_host; |
| 6920 | 6979 | } else { |
| 6921 | 6980 | debug_size_in_bits = 8 * get_store_size_bytes(field_type->size_in_bits); |
| 6922 | 6981 | debug_align_in_bits = 8 * field_type->abi_align; |
| 6923 | | debug_offset_in_bits = 8 * type_struct_field->offset; |
| 6982 | debug_offset_in_bits = 8 * field->offset; |
| 6924 | 6983 | } |
| 6925 | 6984 | unsigned line; |
| 6926 | 6985 | if (decl_node != nullptr) { |
| ... | ... | @@ -6930,7 +6989,7 @@ static void resolve_llvm_types_struct(CodeGen *g, ZigType *struct_type, ResolveS |
| 6930 | 6989 | line = 0; |
| 6931 | 6990 | } |
| 6932 | 6991 | di_element_types[debug_field_index] = ZigLLVMCreateDebugMemberType(g->dbuilder, |
| 6933 | | ZigLLVMTypeToScope(struct_type->llvm_di_type), buf_ptr(type_struct_field->name), |
| 6992 | ZigLLVMTypeToScope(struct_type->llvm_di_type), buf_ptr(field->name), |
| 6934 | 6993 | di_file, line, |
| 6935 | 6994 | debug_size_in_bits, |
| 6936 | 6995 | debug_align_in_bits, |
| ... | ... | @@ -7171,7 +7230,7 @@ static void resolve_llvm_types_union(CodeGen *g, ZigType *union_type, ResolveSta |
| 7171 | 7230 | union_type->data.unionation.resolve_status = ResolveStatusLLVMFull; |
| 7172 | 7231 | } |
| 7173 | 7232 | |
| 7174 | | static void resolve_llvm_types_pointer(CodeGen *g, ZigType *type) { |
| 7233 | static void resolve_llvm_types_pointer(CodeGen *g, ZigType *type, ResolveStatus wanted_resolve_status) { |
| 7175 | 7234 | if (type->llvm_di_type != nullptr) return; |
| 7176 | 7235 | |
| 7177 | 7236 | if (!type_has_bits(type)) { |
| ... | ... | @@ -7200,7 +7259,7 @@ static void resolve_llvm_types_pointer(CodeGen *g, ZigType *type) { |
| 7200 | 7259 | uint64_t debug_align_in_bits = 8*type->abi_align; |
| 7201 | 7260 | type->llvm_di_type = ZigLLVMCreateDebugPointerType(g->dbuilder, elem_type->llvm_di_type, |
| 7202 | 7261 | debug_size_in_bits, debug_align_in_bits, buf_ptr(&type->name)); |
| 7203 | | assertNoError(type_resolve(g, elem_type, ResolveStatusLLVMFull)); |
| 7262 | assertNoError(type_resolve(g, elem_type, wanted_resolve_status)); |
| 7204 | 7263 | } else { |
| 7205 | 7264 | ZigType *host_int_type = get_int_type(g, false, type->data.pointer.host_int_bytes * 8); |
| 7206 | 7265 | LLVMTypeRef host_int_llvm_type = get_llvm_type(g, host_int_type); |
| ... | ... | @@ -7326,10 +7385,17 @@ static void resolve_llvm_types_error_union(CodeGen *g, ZigType *type) { |
| 7326 | 7385 | } else { |
| 7327 | 7386 | LLVMTypeRef err_set_llvm_type = get_llvm_type(g, err_set_type); |
| 7328 | 7387 | LLVMTypeRef payload_llvm_type = get_llvm_type(g, payload_type); |
| 7329 | | LLVMTypeRef elem_types[2]; |
| 7388 | LLVMTypeRef elem_types[3]; |
| 7330 | 7389 | elem_types[err_union_err_index] = err_set_llvm_type; |
| 7331 | 7390 | elem_types[err_union_payload_index] = payload_llvm_type; |
| 7391 | |
| 7332 | 7392 | type->llvm_type = LLVMStructType(elem_types, 2, false); |
| 7393 | if (LLVMABISizeOfType(g->target_data_ref, type->llvm_type) != type->abi_size) { |
| 7394 | // we need to do our own padding |
| 7395 | type->data.error_union.pad_llvm_type = LLVMArrayType(LLVMInt8Type(), type->data.error_union.pad_bytes); |
| 7396 | elem_types[2] = type->data.error_union.pad_llvm_type; |
| 7397 | type->llvm_type = LLVMStructType(elem_types, 3, false); |
| 7398 | } |
| 7333 | 7399 | |
| 7334 | 7400 | ZigLLVMDIScope *compile_unit_scope = ZigLLVMCompileUnitToScope(g->compile_unit); |
| 7335 | 7401 | ZigLLVMDIFile *di_file = nullptr; |
| ... | ... | @@ -7511,6 +7577,7 @@ static void resolve_llvm_types_fn_type(CodeGen *g, ZigType *fn_type) { |
| 7511 | 7577 | } |
| 7512 | 7578 | |
| 7513 | 7579 | void resolve_llvm_types_fn(CodeGen *g, ZigFn *fn) { |
| 7580 | Error err; |
| 7514 | 7581 | if (fn->raw_di_type != nullptr) return; |
| 7515 | 7582 | |
| 7516 | 7583 | ZigType *fn_type = fn->type_entry; |
| ... | ... | @@ -7529,8 +7596,10 @@ void resolve_llvm_types_fn(CodeGen *g, ZigFn *fn) { |
| 7529 | 7596 | |
| 7530 | 7597 | ZigType *frame_type = get_coro_frame_type(g, fn); |
| 7531 | 7598 | ZigType *ptr_type = get_pointer_to_type(g, frame_type, false); |
| 7532 | | gen_param_types.append(get_llvm_type(g, ptr_type)); |
| 7533 | | param_di_types.append(get_llvm_di_type(g, ptr_type)); |
| 7599 | if ((err = type_resolve(g, ptr_type, ResolveStatusLLVMFwdDecl))) |
| 7600 | zig_unreachable(); |
| 7601 | gen_param_types.append(ptr_type->llvm_type); |
| 7602 | param_di_types.append(ptr_type->llvm_di_type); |
| 7534 | 7603 | |
| 7535 | 7604 | // this parameter is used to pass the result pointer when await completes |
| 7536 | 7605 | gen_param_types.append(get_llvm_type(g, g->builtin_types.entry_usize)); |
| ... | ... | @@ -7726,7 +7795,7 @@ static void resolve_llvm_types(CodeGen *g, ZigType *type, ResolveStatus wanted_r |
| 7726 | 7795 | case ZigTypeIdUnion: |
| 7727 | 7796 | return resolve_llvm_types_union(g, type, wanted_resolve_status); |
| 7728 | 7797 | case ZigTypeIdPointer: |
| 7729 | | return resolve_llvm_types_pointer(g, type); |
| 7798 | return resolve_llvm_types_pointer(g, type, wanted_resolve_status); |
| 7730 | 7799 | case ZigTypeIdInt: |
| 7731 | 7800 | return resolve_llvm_types_integer(g, type); |
| 7732 | 7801 | case ZigTypeIdOptional: |