| ... | @@ -1,16 +1,24 @@ | ... | @@ -1,16 +1,24 @@ |
| 1 | const builtin = @import("builtin"); | 1 | const builtin = @import("builtin"); |
| 2 | | 2 | |
| | 3 | const FuncGen = @import("llvm/FuncGen.zig"); |
| | 4 | const buildAllocaInner = FuncGen.buildAllocaInner; |
| | 5 | const isByRef = FuncGen.isByRef; |
| | 6 | const firstParamSRet = FuncGen.firstParamSRet; |
| | 7 | const lowerFnRetTy = FuncGen.lowerFnRetTy; |
| | 8 | const iterateParamTypes = FuncGen.iterateParamTypes; |
| | 9 | const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt; |
| | 10 | const aarch64_c_abi = @import("aarch64/abi.zig"); |
| | 11 | |
| 3 | const std = @import("std"); | 12 | const std = @import("std"); |
| 4 | const Io = std.Io; | 13 | const Io = std.Io; |
| 5 | const assert = std.debug.assert; | 14 | const assert = std.debug.assert; |
| 6 | const Allocator = std.mem.Allocator; | 15 | const Allocator = std.mem.Allocator; |
| 7 | const log = std.log.scoped(.codegen); | 16 | const log = std.log.scoped(.codegen); |
| 8 | const math = std.math; | | |
| 9 | const DW = std.dwarf; | 17 | const DW = std.dwarf; |
| 10 | const Builder = std.zig.llvm.Builder; | 18 | const Builder = std.zig.llvm.Builder; |
| 11 | | 19 | |
| 12 | const build_options = @import("build_options"); | 20 | const build_options = @import("build_options"); |
| 13 | const llvm = if (build_options.have_llvm) | 21 | const bindings = if (build_options.have_llvm) |
| 14 | @import("llvm/bindings.zig") | 22 | @import("llvm/bindings.zig") |
| 15 | else | 23 | else |
| 16 | @compileError("LLVM unavailable"); | 24 | @compileError("LLVM unavailable"); |
| ... | @@ -24,21 +32,9 @@ const Air = @import("../Air.zig"); | ... | @@ -24,21 +32,9 @@ const Air = @import("../Air.zig"); |
| 24 | const Value = @import("../Value.zig"); | 32 | const Value = @import("../Value.zig"); |
| 25 | const Type = @import("../Type.zig"); | 33 | const Type = @import("../Type.zig"); |
| 26 | const codegen = @import("../codegen.zig"); | 34 | const codegen = @import("../codegen.zig"); |
| 27 | const x86_64_abi = @import("x86_64/abi.zig"); | | |
| 28 | const wasm_c_abi = @import("wasm/abi.zig"); | | |
| 29 | const aarch64_c_abi = @import("aarch64/abi.zig"); | | |
| 30 | const arm_c_abi = @import("arm/abi.zig"); | | |
| 31 | const riscv_c_abi = @import("riscv64/abi.zig"); | | |
| 32 | const mips_c_abi = @import("mips/abi.zig"); | | |
| 33 | const dev = @import("../dev.zig"); | 35 | const dev = @import("../dev.zig"); |
| 34 | | 36 | |
| 35 | const target_util = @import("../target.zig"); | 37 | const target_util = @import("../target.zig"); |
| 36 | const libcFloatPrefix = target_util.libcFloatPrefix; | | |
| 37 | const libcFloatSuffix = target_util.libcFloatSuffix; | | |
| 38 | const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev; | | |
| 39 | const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev; | | |
| 40 | | | |
| 41 | const Error = error{ OutOfMemory, CodegenFail }; | | |
| 42 | | 38 | |
| 43 | pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features { | 39 | pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features { |
| 44 | return comptime &.initMany(&.{ | 40 | return comptime &.initMany(&.{ |
| ... | @@ -493,7 +489,7 @@ pub fn dataLayout(target: *const std.Target) []const u8 { | ... | @@ -493,7 +489,7 @@ pub fn dataLayout(target: *const std.Target) []const u8 { |
| 493 | }; | 489 | }; |
| 494 | } | 490 | } |
| 495 | | 491 | |
| 496 | // Avoid depending on `llvm.CodeModel` in the bitcode-only case. | 492 | // Avoid depending on `bindings.CodeModel` in the bitcode-only case. |
| 497 | const CodeModel = enum { | 493 | const CodeModel = enum { |
| 498 | default, | 494 | default, |
| 499 | tiny, | 495 | tiny, |
| ... | @@ -553,20 +549,16 @@ pub const Object = struct { | ... | @@ -553,20 +549,16 @@ pub const Object = struct { |
| 553 | /// type from the global error set. | 549 | /// type from the global error set. |
| 554 | debug_anyerror_fwd_ref: Builder.Metadata.Optional, | 550 | debug_anyerror_fwd_ref: Builder.Metadata.Optional, |
| 555 | | 551 | |
| 556 | target: *const std.Target, | 552 | zcu: *Zcu, |
| 557 | /// Ideally we would use `llvm_module.getNamedFunction` to go from *Decl to LLVM function, | 553 | /// Maps a `Nav` to the corresponding LLVM global. |
| 558 | /// but that has some downsides: | | |
| 559 | /// * we have to compute the fully qualified name every time we want to do the lookup | | |
| 560 | /// * for externally linked functions, the name is not fully qualified, but when | | |
| 561 | /// a Decl goes from exported to not exported and vice-versa, we would use the wrong | | |
| 562 | /// version of the name and incorrectly get function not found in the llvm module. | | |
| 563 | /// * it works for functions not all globals. | | |
| 564 | /// Therefore, this table keeps track of the mapping. | | |
| 565 | nav_map: std.AutoHashMapUnmanaged(InternPool.Nav.Index, Builder.Global.Index), | 554 | nav_map: std.AutoHashMapUnmanaged(InternPool.Nav.Index, Builder.Global.Index), |
| 566 | /// Same deal as `decl_map` but for anonymous declarations, which are always global constants. | 555 | /// Same as `nav_map` but for UAVs (which are always global constants). |
| 567 | uav_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Global.Index), | 556 | uav_map: std.AutoHashMapUnmanaged(struct { |
| | 557 | val: InternPool.Index, |
| | 558 | @"addrspace": std.builtin.AddressSpace, |
| | 559 | }, Builder.Variable.Index), |
| 568 | /// Maps enum types to their corresponding LLVM functions for implementing the `tag_name` instruction. | 560 | /// Maps enum types to their corresponding LLVM functions for implementing the `tag_name` instruction. |
| 569 | enum_tag_name_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Global.Index), | 561 | enum_tag_name_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index), |
| 570 | /// Serves the same purpose as `enum_tag_name_map` but for the `is_named_enum_value` instruction. | 562 | /// Serves the same purpose as `enum_tag_name_map` but for the `is_named_enum_value` instruction. |
| 571 | named_enum_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index), | 563 | named_enum_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index), |
| 572 | /// Maps Zig types to LLVM types. The table memory is backed by the GPA of | 564 | /// Maps Zig types to LLVM types. The table memory is backed by the GPA of |
| ... | @@ -578,36 +570,25 @@ pub const Object = struct { | ... | @@ -578,36 +570,25 @@ pub const Object = struct { |
| 578 | /// Note that the values are not added until `emit`, when all errors in | 570 | /// Note that the values are not added until `emit`, when all errors in |
| 579 | /// the compilation are known. | 571 | /// the compilation are known. |
| 580 | error_name_table: Builder.Variable.Index, | 572 | error_name_table: Builder.Variable.Index, |
| 581 | | 573 | /// Constant variable whose value is the number of errors in the Zcu. |
| 582 | /// Memoizes a null `?usize` value. | 574 | /// |
| 583 | null_opt_usize: Builder.Constant, | 575 | /// Initially `.none`---populated lazily by `getErrorsLen`. |
| 584 | | 576 | /// |
| 585 | /// When an LLVM struct type is created, an entry is inserted into this | 577 | /// If this is not `.none`, the variable's initializer is set in `emit`. |
| 586 | /// table for every zig source field of the struct that has a corresponding | 578 | errors_len_variable: Builder.Variable.Index, |
| 587 | /// LLVM struct field. comptime fields are not included. Zero-bit fields are | | |
| 588 | /// mapped to a field at the correct byte, which may be a padding field, or | | |
| 589 | /// are not mapped, in which case they are semantically at the end of the | | |
| 590 | /// struct. | | |
| 591 | /// The value is the LLVM struct field index. | | |
| 592 | /// This is denormalized data. | | |
| 593 | struct_field_map: std.AutoHashMapUnmanaged(ZigStructField, c_uint), | | |
| 594 | | 579 | |
| 595 | /// Values for `@llvm.used`. | 580 | /// Values for `@llvm.used`. |
| 596 | used: std.ArrayList(Builder.Constant), | 581 | used: std.ArrayList(Builder.Constant), |
| 597 | | 582 | |
| 598 | const ZigStructField = struct { | | |
| 599 | struct_ty: InternPool.Index, | | |
| 600 | field_index: u32, | | |
| 601 | }; | | |
| 602 | | | |
| 603 | pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn; | 583 | pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn; |
| 604 | | 584 | |
| 605 | pub const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type); | 585 | const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type); |
| 606 | | 586 | |
| 607 | pub fn create(arena: Allocator, comp: *Compilation) !Ptr { | 587 | pub fn create(arena: Allocator, zcu: *Zcu) !Ptr { |
| 608 | dev.check(.llvm_backend); | 588 | dev.check(.llvm_backend); |
| | 589 | const comp = zcu.comp; |
| 609 | const gpa = comp.gpa; | 590 | const gpa = comp.gpa; |
| 610 | const target = &comp.root_mod.resolved_target.result; | 591 | const target = zcu.getTarget(); |
| 611 | const llvm_target_triple = try targetTriple(arena, target); | 592 | const llvm_target_triple = try targetTriple(arena, target); |
| 612 | | 593 | |
| 613 | var builder = try Builder.init(.{ | 594 | var builder = try Builder.init(.{ |
| ... | @@ -635,10 +616,7 @@ pub const Object = struct { | ... | @@ -635,10 +616,7 @@ pub const Object = struct { |
| 635 | // way already, but here we throw all that sweet information | 616 | // way already, but here we throw all that sweet information |
| 636 | // into the garbage can by converting into absolute paths. What | 617 | // into the garbage can by converting into absolute paths. What |
| 637 | // a terrible tragedy. | 618 | // a terrible tragedy. |
| 638 | const compile_unit_dir = blk: { | 619 | const compile_unit_dir = try zcu.main_mod.root.toAbsolute(comp.dirs, arena); |
| 639 | const zcu = comp.zcu orelse break :blk comp.dirs.cwd; | | |
| 640 | break :blk try zcu.main_mod.root.toAbsolute(comp.dirs, arena); | | |
| 641 | }; | | |
| 642 | | 620 | |
| 643 | const debug_file = try builder.debugFile( | 621 | const debug_file = try builder.debugFile( |
| 644 | try builder.metadataString(comp.root_name), | 622 | try builder.metadataString(comp.root_name), |
| ... | @@ -684,15 +662,14 @@ pub const Object = struct { | ... | @@ -684,15 +662,14 @@ pub const Object = struct { |
| 684 | .debug_file_map = .empty, | 662 | .debug_file_map = .empty, |
| 685 | .debug_types = .empty, | 663 | .debug_types = .empty, |
| 686 | .debug_anyerror_fwd_ref = .none, | 664 | .debug_anyerror_fwd_ref = .none, |
| 687 | .target = target, | 665 | .zcu = zcu, |
| 688 | .nav_map = .empty, | 666 | .nav_map = .empty, |
| 689 | .uav_map = .empty, | 667 | .uav_map = .empty, |
| 690 | .enum_tag_name_map = .empty, | 668 | .enum_tag_name_map = .empty, |
| 691 | .named_enum_map = .empty, | 669 | .named_enum_map = .empty, |
| 692 | .type_map = .empty, | 670 | .type_map = .empty, |
| 693 | .error_name_table = .none, | 671 | .error_name_table = .none, |
| 694 | .null_opt_usize = .no_init, | 672 | .errors_len_variable = .none, |
| 695 | .struct_field_map = .empty, | | |
| 696 | .used = .empty, | 673 | .used = .empty, |
| 697 | }; | 674 | }; |
| 698 | return obj; | 675 | return obj; |
| ... | @@ -712,15 +689,14 @@ pub const Object = struct { | ... | @@ -712,15 +689,14 @@ pub const Object = struct { |
| 712 | self.named_enum_map.deinit(gpa); | 689 | self.named_enum_map.deinit(gpa); |
| 713 | self.type_map.deinit(gpa); | 690 | self.type_map.deinit(gpa); |
| 714 | self.builder.deinit(); | 691 | self.builder.deinit(); |
| 715 | self.struct_field_map.deinit(gpa); | | |
| 716 | self.* = undefined; | 692 | self.* = undefined; |
| 717 | } | 693 | } |
| 718 | | 694 | |
| 719 | fn genErrorNameTable(o: *Object, pt: Zcu.PerThread) Allocator.Error!void { | 695 | fn genErrorNameTable(o: *Object) Allocator.Error!void { |
| 720 | // If o.error_name_table is null, then it was not referenced by any instructions. | 696 | // If o.error_name_table is null, then it was not referenced by any instructions. |
| 721 | if (o.error_name_table == .none) return; | 697 | if (o.error_name_table == .none) return; |
| 722 | | 698 | |
| 723 | const zcu = pt.zcu; | 699 | const zcu = o.zcu; |
| 724 | const ip = &zcu.intern_pool; | 700 | const ip = &zcu.intern_pool; |
| 725 | | 701 | |
| 726 | const error_name_list = ip.global_error_set.getNamesFromMainThread(); | 702 | const error_name_list = ip.global_error_set.getNamesFromMainThread(); |
| ... | @@ -729,21 +705,21 @@ pub const Object = struct { | ... | @@ -729,21 +705,21 @@ pub const Object = struct { |
| 729 | | 705 | |
| 730 | // TODO: Address space | 706 | // TODO: Address space |
| 731 | const slice_ty = Type.slice_const_u8_sentinel_0; | 707 | const slice_ty = Type.slice_const_u8_sentinel_0; |
| 732 | const llvm_usize_ty = try o.lowerType(pt, Type.usize); | 708 | const llvm_usize_ty = try o.lowerType(.usize); |
| 733 | const llvm_slice_ty = try o.lowerType(pt, slice_ty); | 709 | const llvm_slice_ty = try o.lowerType(slice_ty); |
| 734 | const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty); | 710 | const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty); |
| 735 | | 711 | |
| 736 | llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty); | 712 | llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty); |
| 737 | for (llvm_errors[1..], error_name_list) |*llvm_error, name| { | 713 | for (llvm_errors[1..], error_name_list) |*llvm_error, name| { |
| 738 | const name_string = try o.builder.stringNull(name.toSlice(ip)); | 714 | const name_string = try o.builder.stringNull(name.toSlice(ip)); |
| 739 | const name_init = try o.builder.stringConst(name_string); | 715 | const name_init = try o.builder.stringConst(name_string); |
| 740 | const name_variable_index = | 716 | const name_variable_index = try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); |
| 741 | try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); | | |
| 742 | try name_variable_index.setInitializer(name_init, &o.builder); | 717 | try name_variable_index.setInitializer(name_init, &o.builder); |
| 743 | name_variable_index.setLinkage(.private, &o.builder); | | |
| 744 | name_variable_index.setMutability(.constant, &o.builder); | 718 | name_variable_index.setMutability(.constant, &o.builder); |
| 745 | name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); | 719 | name_variable_index.setAlignment(comptime .fromByteUnits(1), &o.builder); |
| 746 | name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); | 720 | const global_index = name_variable_index.ptrConst(&o.builder).global; |
| | 721 | global_index.setLinkage(.private, &o.builder); |
| | 722 | global_index.setUnnamedAddr(.unnamed_addr, &o.builder); |
| 747 | | 723 | |
| 748 | llvm_error.* = try o.builder.structConst(llvm_slice_ty, &.{ | 724 | llvm_error.* = try o.builder.structConst(llvm_slice_ty, &.{ |
| 749 | name_variable_index.toConst(&o.builder), | 725 | name_variable_index.toConst(&o.builder), |
| ... | @@ -751,52 +727,17 @@ pub const Object = struct { | ... | @@ -751,52 +727,17 @@ pub const Object = struct { |
| 751 | }); | 727 | }); |
| 752 | } | 728 | } |
| 753 | | 729 | |
| 754 | const table_variable_index = try o.builder.addVariable(.empty, llvm_table_ty, .default); | 730 | try o.error_name_table.setInitializer( |
| 755 | try table_variable_index.setInitializer( | | |
| 756 | try o.builder.arrayConst(llvm_table_ty, llvm_errors), | 731 | try o.builder.arrayConst(llvm_table_ty, llvm_errors), |
| 757 | &o.builder, | 732 | &o.builder, |
| 758 | ); | 733 | ); |
| 759 | table_variable_index.setLinkage(.private, &o.builder); | | |
| 760 | table_variable_index.setMutability(.constant, &o.builder); | | |
| 761 | table_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); | | |
| 762 | table_variable_index.setAlignment( | | |
| 763 | slice_ty.abiAlignment(zcu).toLlvm(), | | |
| 764 | &o.builder, | | |
| 765 | ); | | |
| 766 | | | |
| 767 | try o.error_name_table.setInitializer(table_variable_index.toConst(&o.builder), &o.builder); | | |
| 768 | } | | |
| 769 | | | |
| 770 | fn genCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !void { | | |
| 771 | // If there is no such function in the module, it means the source code does not need it. | | |
| 772 | const name = o.builder.strtabStringIfExists(lt_errors_fn_name) orelse return; | | |
| 773 | const llvm_fn = o.builder.getGlobal(name) orelse return; | | |
| 774 | const errors_len = pt.zcu.intern_pool.global_error_set.getNamesFromMainThread().len; | | |
| 775 | | | |
| 776 | var wip = try Builder.WipFunction.init(&o.builder, .{ | | |
| 777 | .function = llvm_fn.ptrConst(&o.builder).kind.function, | | |
| 778 | .strip = true, | | |
| 779 | }); | | |
| 780 | defer wip.deinit(); | | |
| 781 | wip.cursor = .{ .block = try wip.block(0, "Entry") }; | | |
| 782 | | | |
| 783 | // Example source of the following LLVM IR: | | |
| 784 | // fn __zig_lt_errors_len(index: u16) bool { | | |
| 785 | // return index <= total_errors_len; | | |
| 786 | // } | | |
| 787 | | | |
| 788 | const lhs = wip.arg(0); | | |
| 789 | const rhs = try o.builder.intValue(try o.errorIntType(pt), errors_len); | | |
| 790 | const is_lt = try wip.icmp(.ule, lhs, rhs, ""); | | |
| 791 | _ = try wip.ret(is_lt); | | |
| 792 | try wip.finish(); | | |
| 793 | } | 734 | } |
| 794 | | 735 | |
| 795 | fn genModuleLevelAssembly(object: *Object, pt: Zcu.PerThread) Allocator.Error!void { | 736 | fn genModuleLevelAssembly(object: *Object) Allocator.Error!void { |
| 796 | const b = &object.builder; | 737 | const b = &object.builder; |
| 797 | const gpa = b.gpa; | 738 | const gpa = b.gpa; |
| 798 | b.module_asm.clearRetainingCapacity(); | 739 | b.module_asm.clearRetainingCapacity(); |
| 799 | for (pt.zcu.global_assembly.values()) |assembly| { | 740 | for (object.zcu.global_assembly.values()) |assembly| { |
| 800 | try b.module_asm.ensureUnusedCapacity(gpa, assembly.len + 1); | 741 | try b.module_asm.ensureUnusedCapacity(gpa, assembly.len + 1); |
| 801 | b.module_asm.appendSliceAssumeCapacity(assembly); | 742 | b.module_asm.appendSliceAssumeCapacity(assembly); |
| 802 | b.module_asm.appendAssumeCapacity('\n'); | 743 | b.module_asm.appendAssumeCapacity('\n'); |
| ... | @@ -823,15 +764,19 @@ pub const Object = struct { | ... | @@ -823,15 +764,19 @@ pub const Object = struct { |
| 823 | }; | 764 | }; |
| 824 | | 765 | |
| 825 | pub fn emit(o: *Object, pt: Zcu.PerThread, options: EmitOptions) error{ LinkFailure, OutOfMemory }!void { | 766 | pub fn emit(o: *Object, pt: Zcu.PerThread, options: EmitOptions) error{ LinkFailure, OutOfMemory }!void { |
| 826 | const zcu = pt.zcu; | 767 | const zcu = o.zcu; |
| 827 | const comp = zcu.comp; | 768 | const comp = zcu.comp; |
| 828 | const io = comp.io; | 769 | const io = comp.io; |
| 829 | const diags = &comp.link_diags; | 770 | const diags = &comp.link_diags; |
| 830 | | 771 | |
| 831 | { | 772 | { |
| 832 | try o.genErrorNameTable(pt); | 773 | if (o.errors_len_variable != .none) { |
| 833 | try o.genCmpLtErrorsLenFunction(pt); | 774 | const errors_len = zcu.intern_pool.global_error_set.getNamesFromMainThread().len; |
| 834 | try o.genModuleLevelAssembly(pt); | 775 | const init_val = try o.builder.intConst(try o.errorIntType(), errors_len); |
| | 776 | try o.errors_len_variable.setInitializer(init_val, &o.builder); |
| | 777 | } |
| | 778 | try o.genErrorNameTable(); |
| | 779 | try o.genModuleLevelAssembly(); |
| 835 | | 780 | |
| 836 | if (o.used.items.len > 0) { | 781 | if (o.used.items.len > 0) { |
| 837 | const array_llvm_ty = try o.builder.arrayType(o.used.items.len, .ptr); | 782 | const array_llvm_ty = try o.builder.arrayType(o.used.items.len, .ptr); |
| ... | @@ -841,14 +786,14 @@ pub const Object = struct { | ... | @@ -841,14 +786,14 @@ pub const Object = struct { |
| 841 | array_llvm_ty, | 786 | array_llvm_ty, |
| 842 | .default, | 787 | .default, |
| 843 | ); | 788 | ); |
| 844 | compiler_used_variable.setLinkage(.appending, &o.builder); | | |
| 845 | compiler_used_variable.setSection(try o.builder.string("llvm.metadata"), &o.builder); | | |
| 846 | try compiler_used_variable.setInitializer(init_val, &o.builder); | 789 | try compiler_used_variable.setInitializer(init_val, &o.builder); |
| | 790 | compiler_used_variable.setSection(try o.builder.string("llvm.metadata"), &o.builder); |
| | 791 | compiler_used_variable.ptrConst(&o.builder).global.setLinkage(.appending, &o.builder); |
| 847 | } | 792 | } |
| 848 | | 793 | |
| 849 | if (!o.builder.strip) { | 794 | if (!o.builder.strip) { |
| 850 | if (o.debug_anyerror_fwd_ref.unwrap()) |fwd_ref| { | 795 | if (o.debug_anyerror_fwd_ref.unwrap()) |fwd_ref| { |
| 851 | const debug_anyerror_type = try o.lowerDebugAnyerrorType(pt); | 796 | const debug_anyerror_type = try o.lowerDebugAnyerrorType(); |
| 852 | o.builder.resolveDebugForwardReference(fwd_ref, debug_anyerror_type); | 797 | o.builder.resolveDebugForwardReference(fwd_ref, debug_anyerror_type); |
| 853 | } | 798 | } |
| 854 | | 799 | |
| ... | @@ -995,7 +940,6 @@ pub const Object = struct { | ... | @@ -995,7 +940,6 @@ pub const Object = struct { |
| 995 | .version = build_options.semver, | 940 | .version = build_options.semver, |
| 996 | }); | 941 | }); |
| 997 | defer o.gpa.free(bitcode); | 942 | defer o.gpa.free(bitcode); |
| 998 | o.builder.clearAndFree(); | | |
| 999 | | 943 | |
| 1000 | if (options.pre_bc_path) |path| { | 944 | if (options.pre_bc_path) |path| { |
| 1001 | var file = Io.Dir.cwd().createFile(io, path, .{}) catch |err| | 945 | var file = Io.Dir.cwd().createFile(io, path, .{}) catch |err| |
| ... | @@ -1026,20 +970,20 @@ pub const Object = struct { | ... | @@ -1026,20 +970,20 @@ pub const Object = struct { |
| 1026 | | 970 | |
| 1027 | initializeLLVMTarget(comp.root_mod.resolved_target.result.cpu.arch); | 971 | initializeLLVMTarget(comp.root_mod.resolved_target.result.cpu.arch); |
| 1028 | | 972 | |
| 1029 | const context: *llvm.Context = llvm.Context.create(); | 973 | const context: *bindings.Context = .create(); |
| 1030 | errdefer context.dispose(); | 974 | errdefer context.dispose(); |
| 1031 | | 975 | |
| 1032 | const bitcode_memory_buffer = llvm.MemoryBuffer.createMemoryBufferWithMemoryRange( | 976 | const bitcode_memory_buffer = bindings.MemoryBuffer.createMemoryBufferWithMemoryRange( |
| 1033 | @ptrCast(bitcode.ptr), | 977 | @ptrCast(bitcode.ptr), |
| 1034 | bitcode.len * 4, | 978 | bitcode.len * 4, |
| 1035 | "BitcodeBuffer", | 979 | "BitcodeBuffer", |
| 1036 | llvm.Bool.False, | 980 | bindings.Bool.False, |
| 1037 | ); | 981 | ); |
| 1038 | defer bitcode_memory_buffer.dispose(); | 982 | defer bitcode_memory_buffer.dispose(); |
| 1039 | | 983 | |
| 1040 | context.enableBrokenDebugInfoCheck(); | 984 | context.enableBrokenDebugInfoCheck(); |
| 1041 | | 985 | |
| 1042 | var module: *llvm.Module = undefined; | 986 | var module: *bindings.Module = undefined; |
| 1043 | if (context.parseBitcodeInContext2(bitcode_memory_buffer, &module).toBool() or context.getBrokenDebugInfo()) { | 987 | if (context.parseBitcodeInContext2(bitcode_memory_buffer, &module).toBool() or context.getBrokenDebugInfo()) { |
| 1044 | return diags.fail("Failed to parse bitcode", .{}); | 988 | return diags.fail("Failed to parse bitcode", .{}); |
| 1045 | } | 989 | } |
| ... | @@ -1047,28 +991,28 @@ pub const Object = struct { | ... | @@ -1047,28 +991,28 @@ pub const Object = struct { |
| 1047 | }; | 991 | }; |
| 1048 | defer context.dispose(); | 992 | defer context.dispose(); |
| 1049 | | 993 | |
| 1050 | var target: *llvm.Target = undefined; | 994 | var target: *bindings.Target = undefined; |
| 1051 | var error_message: [*:0]const u8 = undefined; | 995 | var error_message: [*:0]const u8 = undefined; |
| 1052 | if (llvm.Target.getFromTriple(target_triple_sentinel, &target, &error_message).toBool()) { | 996 | if (bindings.Target.getFromTriple(target_triple_sentinel, &target, &error_message).toBool()) { |
| 1053 | defer llvm.disposeMessage(error_message); | 997 | defer bindings.disposeMessage(error_message); |
| 1054 | return diags.fail("LLVM failed to parse '{s}': {s}", .{ target_triple_sentinel, error_message }); | 998 | return diags.fail("LLVM failed to parse '{s}': {s}", .{ target_triple_sentinel, error_message }); |
| 1055 | } | 999 | } |
| 1056 | | 1000 | |
| 1057 | const optimize_mode = comp.root_mod.optimize_mode; | 1001 | const optimize_mode = comp.root_mod.optimize_mode; |
| 1058 | | 1002 | |
| 1059 | const opt_level: llvm.CodeGenOptLevel = if (optimize_mode == .Debug) | 1003 | const opt_level: bindings.CodeGenOptLevel = if (optimize_mode == .Debug) |
| 1060 | .None | 1004 | .None |
| 1061 | else | 1005 | else |
| 1062 | .Aggressive; | 1006 | .Aggressive; |
| 1063 | | 1007 | |
| 1064 | const reloc_mode: llvm.RelocMode = if (comp.root_mod.pic) | 1008 | const reloc_mode: bindings.RelocMode = if (comp.root_mod.pic) |
| 1065 | .PIC | 1009 | .PIC |
| 1066 | else if (comp.config.link_mode == .dynamic) | 1010 | else if (comp.config.link_mode == .dynamic) |
| 1067 | llvm.RelocMode.DynamicNoPIC | 1011 | bindings.RelocMode.DynamicNoPIC |
| 1068 | else | 1012 | else |
| 1069 | .Static; | 1013 | .Static; |
| 1070 | | 1014 | |
| 1071 | const code_model: llvm.CodeModel = switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) { | 1015 | const code_model: bindings.CodeModel = switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) { |
| 1072 | .default => .Default, | 1016 | .default => .Default, |
| 1073 | .tiny => .Tiny, | 1017 | .tiny => .Tiny, |
| 1074 | .small => .Small, | 1018 | .small => .Small, |
| ... | @@ -1077,12 +1021,12 @@ pub const Object = struct { | ... | @@ -1077,12 +1021,12 @@ pub const Object = struct { |
| 1077 | .large => .Large, | 1021 | .large => .Large, |
| 1078 | }; | 1022 | }; |
| 1079 | | 1023 | |
| 1080 | const float_abi: llvm.TargetMachine.FloatABI = if (comp.root_mod.resolved_target.result.abi.float() == .hard) | 1024 | const float_abi: bindings.TargetMachine.FloatABI = if (comp.root_mod.resolved_target.result.abi.float() == .hard) |
| 1081 | .Hard | 1025 | .Hard |
| 1082 | else | 1026 | else |
| 1083 | .Soft; | 1027 | .Soft; |
| 1084 | | 1028 | |
| 1085 | var target_machine = llvm.TargetMachine.create( | 1029 | var target_machine = bindings.TargetMachine.create( |
| 1086 | target, | 1030 | target, |
| 1087 | target_triple_sentinel, | 1031 | target_triple_sentinel, |
| 1088 | if (comp.root_mod.resolved_target.result.cpu.model.llvm_name) |s| s.ptr else null, | 1032 | if (comp.root_mod.resolved_target.result.cpu.model.llvm_name) |s| s.ptr else null, |
| ... | @@ -1105,7 +1049,7 @@ pub const Object = struct { | ... | @@ -1105,7 +1049,7 @@ pub const Object = struct { |
| 1105 | // Unfortunately, LLVM shits the bed when we ask for both binary and assembly. | 1049 | // Unfortunately, LLVM shits the bed when we ask for both binary and assembly. |
| 1106 | // So we call the entire pipeline multiple times if this is requested. | 1050 | // So we call the entire pipeline multiple times if this is requested. |
| 1107 | // var error_message: [*:0]const u8 = undefined; | 1051 | // var error_message: [*:0]const u8 = undefined; |
| 1108 | var lowered_options: llvm.TargetMachine.EmitOptions = .{ | 1052 | var lowered_options: bindings.TargetMachine.EmitOptions = .{ |
| 1109 | .is_debug = options.is_debug, | 1053 | .is_debug = options.is_debug, |
| 1110 | .is_small = options.is_small, | 1054 | .is_small = options.is_small, |
| 1111 | .time_report_out = null, // set below to make sure it's only set for a single `emitToFile` | 1055 | .time_report_out = null, // set below to make sure it's only set for a single `emitToFile` |
| ... | @@ -1154,7 +1098,7 @@ pub const Object = struct { | ... | @@ -1154,7 +1098,7 @@ pub const Object = struct { |
| 1154 | }; | 1098 | }; |
| 1155 | if (options.asm_path != null and options.bin_path != null) { | 1099 | if (options.asm_path != null and options.bin_path != null) { |
| 1156 | if (target_machine.emitToFile(module, &error_message, &lowered_options)) { | 1100 | if (target_machine.emitToFile(module, &error_message, &lowered_options)) { |
| 1157 | defer llvm.disposeMessage(error_message); | 1101 | defer bindings.disposeMessage(error_message); |
| 1158 | return diags.fail("LLVM failed to emit bin={s} ir={s}: {s}", .{ | 1102 | return diags.fail("LLVM failed to emit bin={s} ir={s}: {s}", .{ |
| 1159 | emit_bin_msg, post_llvm_ir_msg, error_message, | 1103 | emit_bin_msg, post_llvm_ir_msg, error_message, |
| 1160 | }); | 1104 | }); |
| ... | @@ -1170,7 +1114,7 @@ pub const Object = struct { | ... | @@ -1170,7 +1114,7 @@ pub const Object = struct { |
| 1170 | | 1114 | |
| 1171 | lowered_options.asm_filename = if (options.asm_path) |x| x.ptr else null; | 1115 | lowered_options.asm_filename = if (options.asm_path) |x| x.ptr else null; |
| 1172 | if (target_machine.emitToFile(module, &error_message, &lowered_options)) { | 1116 | if (target_machine.emitToFile(module, &error_message, &lowered_options)) { |
| 1173 | defer llvm.disposeMessage(error_message); | 1117 | defer bindings.disposeMessage(error_message); |
| 1174 | return diags.fail("LLVM failed to emit asm={s} bin={s} ir={s} bc={s}: {s}", .{ | 1118 | return diags.fail("LLVM failed to emit asm={s} bin={s} ir={s} bc={s}: {s}", .{ |
| 1175 | emit_asm_msg, emit_bin_msg, post_llvm_ir_msg, post_llvm_bc_msg, error_message, | 1119 | emit_asm_msg, emit_bin_msg, post_llvm_ir_msg, post_llvm_bc_msg, error_message, |
| 1176 | }); | 1120 | }); |
| ... | @@ -1189,9 +1133,10 @@ pub const Object = struct { | ... | @@ -1189,9 +1133,10 @@ pub const Object = struct { |
| 1189 | func_index: InternPool.Index, | 1133 | func_index: InternPool.Index, |
| 1190 | air: *const Air, | 1134 | air: *const Air, |
| 1191 | liveness: *const ?Air.Liveness, | 1135 | liveness: *const ?Air.Liveness, |
| 1192 | ) !void { | 1136 | ) Zcu.CodegenFailError!void { |
| 1193 | const zcu = pt.zcu; | 1137 | const zcu = o.zcu; |
| 1194 | const comp = zcu.comp; | 1138 | const comp = zcu.comp; |
| | 1139 | const gpa = comp.gpa; |
| 1195 | const ip = &zcu.intern_pool; | 1140 | const ip = &zcu.intern_pool; |
| 1196 | const func = zcu.funcInfo(func_index); | 1141 | const func = zcu.funcInfo(func_index); |
| 1197 | const nav = ip.getNav(func.owner_nav); | 1142 | const nav = ip.getNav(func.owner_nav); |
| ... | @@ -1201,16 +1146,42 @@ pub const Object = struct { | ... | @@ -1201,16 +1146,42 @@ pub const Object = struct { |
| 1201 | const fn_info = zcu.typeToFunc(fn_ty).?; | 1146 | const fn_info = zcu.typeToFunc(fn_ty).?; |
| 1202 | const target = &owner_mod.resolved_target.result; | 1147 | const target = &owner_mod.resolved_target.result; |
| 1203 | | 1148 | |
| 1204 | var ng: NavGen = .{ | 1149 | const gop = try o.nav_map.getOrPut(gpa, func.owner_nav); |
| 1205 | .object = o, | 1150 | if (!gop.found_existing) { |
| 1206 | .nav_index = func.owner_nav, | 1151 | errdefer assert(o.nav_map.remove(func.owner_nav)); |
| 1207 | .pt = pt, | 1152 | // First time lowering this NAV! Create a fresh global. |
| 1208 | .err_msg = null, | 1153 | const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip)); |
| 1209 | }; | 1154 | gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{ |
| | 1155 | .type = .void, // placeholder; populated below |
| | 1156 | .kind = .{ .alias = .none }, // placeholder; populated below |
| | 1157 | }); |
| | 1158 | } |
| | 1159 | const llvm_global = gop.value_ptr.*; |
| 1210 | | 1160 | |
| 1211 | const function_index = try o.resolveLlvmFunction(pt, func.owner_nav); | 1161 | const llvm_function: Builder.Function.Index = switch (llvm_global.ptrConst(&o.builder).kind) { |
| | 1162 | .function => |function| function, // re-use existing `Builder.Function` |
| | 1163 | .replaced, .alias, .variable => try llvm_global.toNewFunction(&o.builder), |
| | 1164 | }; |
| | 1165 | { |
| | 1166 | const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder); |
| | 1167 | global.type = try o.lowerType(fn_ty); |
| | 1168 | global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target); |
| | 1169 | global.linkage = if (o.builder.strip) .private else .internal; |
| | 1170 | global.visibility = .default; |
| | 1171 | global.dll_storage_class = .default; |
| | 1172 | global.unnamed_addr = .unnamed_addr; |
| | 1173 | } |
| | 1174 | llvm_function.setAlignment(switch (nav.resolved.?.@"align") { |
| | 1175 | .none => fn_ty.abiAlignment(zcu).toLlvm(), |
| | 1176 | else => |a| a.toLlvm(), |
| | 1177 | }, &o.builder); |
| | 1178 | llvm_function.setSection(s: { |
| | 1179 | const section = nav.resolved.?.@"linksection".toSlice(ip) orelse break :s .none; |
| | 1180 | break :s try o.builder.string(section); |
| | 1181 | }, &o.builder); |
| | 1182 | try o.addLlvmFunctionAttributes(pt, func.owner_nav, llvm_function); |
| 1212 | | 1183 | |
| 1213 | var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); | 1184 | var attributes = try llvm_function.ptrConst(&o.builder).attributes.toWip(&o.builder); |
| 1214 | defer attributes.deinit(&o.builder); | 1185 | defer attributes.deinit(&o.builder); |
| 1215 | | 1186 | |
| 1216 | const func_analysis = func.analysisUnordered(ip); | 1187 | const func_analysis = func.analysisUnordered(ip); |
| ... | @@ -1280,49 +1251,41 @@ pub const Object = struct { | ... | @@ -1280,49 +1251,41 @@ pub const Object = struct { |
| 1280 | } }, &o.builder); | 1251 | } }, &o.builder); |
| 1281 | } | 1252 | } |
| 1282 | | 1253 | |
| 1283 | if (nav.resolved.?.@"linksection".toSlice(ip)) |section| | | |
| 1284 | function_index.setSection(try o.builder.string(section), &o.builder); | | |
| 1285 | | | |
| 1286 | var deinit_wip = true; | 1254 | var deinit_wip = true; |
| 1287 | var wip = try Builder.WipFunction.init(&o.builder, .{ | 1255 | var wip = try Builder.WipFunction.init(&o.builder, .{ |
| 1288 | .function = function_index, | 1256 | .function = llvm_function, |
| 1289 | .strip = owner_mod.strip, | 1257 | .strip = owner_mod.strip, |
| 1290 | }); | 1258 | }); |
| 1291 | defer if (deinit_wip) wip.deinit(); | 1259 | defer if (deinit_wip) wip.deinit(); |
| 1292 | wip.cursor = .{ .block = try wip.block(0, "Entry") }; | 1260 | wip.cursor = .{ .block = try wip.block(0, "Entry") }; |
| 1293 | | 1261 | |
| 1294 | var llvm_arg_i: u32 = 0; | | |
| 1295 | | | |
| 1296 | // This gets the LLVM values from the function and stores them in `ng.args`. | | |
| 1297 | const sret = firstParamSRet(fn_info, zcu, target); | | |
| 1298 | const ret_ptr: Builder.Value = if (sret) param: { | | |
| 1299 | const param = wip.arg(llvm_arg_i); | | |
| 1300 | llvm_arg_i += 1; | | |
| 1301 | break :param param; | | |
| 1302 | } else .none; | | |
| 1303 | | | |
| 1304 | if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) { | 1262 | if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) { |
| 1305 | .signed => try attributes.addRetAttr(.signext, &o.builder), | 1263 | .signed => try attributes.addRetAttr(.signext, &o.builder), |
| 1306 | .unsigned => try attributes.addRetAttr(.zeroext, &o.builder), | 1264 | .unsigned => try attributes.addRetAttr(.zeroext, &o.builder), |
| 1307 | }; | 1265 | }; |
| 1308 | | 1266 | |
| 1309 | const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing; | | |
| 1310 | | | |
| 1311 | const err_ret_trace: Builder.Value = if (err_return_tracing) param: { | | |
| 1312 | const param = wip.arg(llvm_arg_i); | | |
| 1313 | llvm_arg_i += 1; | | |
| 1314 | break :param param; | | |
| 1315 | } else .none; | | |
| 1316 | | | |
| 1317 | // This is the list of args we will use that correspond directly to the AIR arg | 1267 | // This is the list of args we will use that correspond directly to the AIR arg |
| 1318 | // instructions. Depending on the calling convention, this list is not necessarily | 1268 | // instructions. Depending on the calling convention, this list is not necessarily |
| 1319 | // a bijection with the actual LLVM parameters of the function. | 1269 | // a bijection with the actual LLVM parameters of the function. |
| 1320 | const gpa = o.gpa; | | |
| 1321 | var args: std.ArrayList(Builder.Value) = .empty; | 1270 | var args: std.ArrayList(Builder.Value) = .empty; |
| 1322 | defer args.deinit(gpa); | 1271 | defer args.deinit(gpa); |
| 1323 | | 1272 | |
| 1324 | { | 1273 | const ret_ptr: Builder.Value, const err_ret_trace: Builder.Value = implicit_args: { |
| 1325 | var it = iterateParamTypes(o, pt, fn_info); | 1274 | var it = iterateParamTypes(o, fn_info); |
| | 1275 | |
| | 1276 | const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: { |
| | 1277 | const param = wip.arg(it.llvm_index); |
| | 1278 | it.llvm_index += 1; |
| | 1279 | break :param param; |
| | 1280 | } else .none; |
| | 1281 | |
| | 1282 | const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing; |
| | 1283 | const err_ret_trace: Builder.Value = if (err_return_tracing) param: { |
| | 1284 | const param = wip.arg(it.llvm_index); |
| | 1285 | it.llvm_index += 1; |
| | 1286 | break :param param; |
| | 1287 | } else .none; |
| | 1288 | |
| 1326 | while (try it.next()) |lowering| { | 1289 | while (try it.next()) |lowering| { |
| 1327 | try args.ensureUnusedCapacity(gpa, 1); | 1290 | try args.ensureUnusedCapacity(gpa, 1); |
| 1328 | | 1291 | |
| ... | @@ -1332,7 +1295,7 @@ pub const Object = struct { | ... | @@ -1332,7 +1295,7 @@ pub const Object = struct { |
| 1332 | assert(!it.byval_attr); | 1295 | assert(!it.byval_attr); |
| 1333 | const param_index = it.zig_index - 1; | 1296 | const param_index = it.zig_index - 1; |
| 1334 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); | 1297 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); |
| 1335 | const param = wip.arg(llvm_arg_i); | 1298 | const param = wip.arg(it.llvm_index - 1); |
| 1336 | | 1299 | |
| 1337 | if (isByRef(param_ty, zcu)) { | 1300 | if (isByRef(param_ty, zcu)) { |
| 1338 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | 1301 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); |
| ... | @@ -1342,149 +1305,119 @@ pub const Object = struct { | ... | @@ -1342,149 +1305,119 @@ pub const Object = struct { |
| 1342 | args.appendAssumeCapacity(arg_ptr); | 1305 | args.appendAssumeCapacity(arg_ptr); |
| 1343 | } else { | 1306 | } else { |
| 1344 | args.appendAssumeCapacity(param); | 1307 | args.appendAssumeCapacity(param); |
| 1345 | | | |
| 1346 | try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, llvm_arg_i); | | |
| 1347 | } | 1308 | } |
| 1348 | llvm_arg_i += 1; | | |
| 1349 | }, | 1309 | }, |
| 1350 | .byref => { | 1310 | .byref => { |
| 1351 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | 1311 | const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| 1352 | const param_llvm_ty = try o.lowerType(pt, param_ty); | 1312 | const param = wip.arg(it.llvm_index - 1); |
| 1353 | const param = wip.arg(llvm_arg_i); | | |
| 1354 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | | |
| 1355 | | | |
| 1356 | try o.addByRefParamAttrs(&attributes, llvm_arg_i, alignment, it.byval_attr, param_llvm_ty); | | |
| 1357 | llvm_arg_i += 1; | | |
| 1358 | | 1313 | |
| 1359 | if (isByRef(param_ty, zcu)) { | 1314 | if (isByRef(param_ty, zcu)) { |
| 1360 | args.appendAssumeCapacity(param); | 1315 | args.appendAssumeCapacity(param); |
| 1361 | } else { | 1316 | } else { |
| | 1317 | const param_llvm_ty = try o.lowerType(param_ty); |
| | 1318 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); |
| 1362 | args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, "")); | 1319 | args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, "")); |
| 1363 | } | 1320 | } |
| 1364 | }, | 1321 | }, |
| 1365 | .byref_mut => { | 1322 | .byref_mut => { |
| 1366 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | 1323 | const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| 1367 | const param_llvm_ty = try o.lowerType(pt, param_ty); | 1324 | const param = wip.arg(it.llvm_index - 1); |
| 1368 | const param = wip.arg(llvm_arg_i); | | |
| 1369 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | | |
| 1370 | | | |
| 1371 | try attributes.addParamAttr(llvm_arg_i, .noundef, &o.builder); | | |
| 1372 | llvm_arg_i += 1; | | |
| 1373 | | 1325 | |
| 1374 | if (isByRef(param_ty, zcu)) { | 1326 | if (isByRef(param_ty, zcu)) { |
| 1375 | args.appendAssumeCapacity(param); | 1327 | args.appendAssumeCapacity(param); |
| 1376 | } else { | 1328 | } else { |
| | 1329 | const param_llvm_ty = try o.lowerType(param_ty); |
| | 1330 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); |
| 1377 | args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, "")); | 1331 | args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, "")); |
| 1378 | } | 1332 | } |
| 1379 | }, | 1333 | }, |
| 1380 | .abi_sized_int => { | 1334 | .abi_sized_int => { |
| 1381 | assert(!it.byval_attr); | 1335 | assert(!it.byval_attr); |
| 1382 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | 1336 | const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| 1383 | const param = wip.arg(llvm_arg_i); | 1337 | const param = wip.arg(it.llvm_index - 1); |
| 1384 | llvm_arg_i += 1; | | |
| 1385 | | 1338 | |
| 1386 | const param_llvm_ty = try o.lowerType(pt, param_ty); | 1339 | const param_llvm_ty = try o.lowerType(param_ty); |
| 1387 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | 1340 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); |
| 1388 | const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target); | 1341 | const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target); |
| 1389 | _ = try wip.store(.normal, param, arg_ptr, alignment); | 1342 | _ = try wip.store(.normal, param, arg_ptr, alignment); |
| 1390 | | 1343 | |
| 1391 | args.appendAssumeCapacity(if (isByRef(param_ty, zcu)) | 1344 | if (isByRef(param_ty, zcu)) { |
| 1392 | arg_ptr | 1345 | args.appendAssumeCapacity(arg_ptr); |
| 1393 | else | 1346 | } else { |
| 1394 | try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); | 1347 | args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); |
| | 1348 | } |
| 1395 | }, | 1349 | }, |
| 1396 | .slice => { | 1350 | .slice => { |
| 1397 | assert(!it.byval_attr); | 1351 | assert(!it.byval_attr); |
| 1398 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | 1352 | const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| 1399 | const ptr_info = param_ty.ptrInfo(zcu); | 1353 | assert(!isByRef(param_ty, zcu)); |
| 1400 | | 1354 | const slice_val = try wip.buildAggregate( |
| 1401 | if (math.cast(u5, it.zig_index - 1)) |i| { | 1355 | try o.lowerType(param_ty), |
| 1402 | if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { | 1356 | &.{ wip.arg(it.llvm_index - 2), wip.arg(it.llvm_index - 1) }, |
| 1403 | try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); | 1357 | "", |
| 1404 | } | | |
| 1405 | } | | |
| 1406 | if (param_ty.zigTypeTag(zcu) != .optional and | | |
| 1407 | !ptr_info.flags.is_allowzero and | | |
| 1408 | ptr_info.flags.address_space == .generic) | | |
| 1409 | { | | |
| 1410 | try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); | | |
| 1411 | } | | |
| 1412 | if (ptr_info.flags.is_const) { | | |
| 1413 | try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder); | | |
| 1414 | } | | |
| 1415 | const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) { | | |
| 1416 | else => |a| .wrap(a.toLlvm()), | | |
| 1417 | .none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)), | | |
| 1418 | }; | | |
| 1419 | try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder); | | |
| 1420 | const ptr_param = wip.arg(llvm_arg_i); | | |
| 1421 | llvm_arg_i += 1; | | |
| 1422 | const len_param = wip.arg(llvm_arg_i); | | |
| 1423 | llvm_arg_i += 1; | | |
| 1424 | | | |
| 1425 | const slice_llvm_ty = try o.lowerType(pt, param_ty); | | |
| 1426 | args.appendAssumeCapacity( | | |
| 1427 | try wip.buildAggregate(slice_llvm_ty, &.{ ptr_param, len_param }, ""), | | |
| 1428 | ); | 1358 | ); |
| | 1359 | args.appendAssumeCapacity(slice_val); |
| 1429 | }, | 1360 | }, |
| 1430 | .multiple_llvm_types => { | 1361 | .multiple_llvm_types => { |
| 1431 | assert(!it.byval_attr); | 1362 | assert(!it.byval_attr); |
| 1432 | const field_types = it.types_buffer[0..it.types_len]; | 1363 | const field_types = it.types_buffer[0..it.types_len]; |
| 1433 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | 1364 | const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| 1434 | const param_llvm_ty = try o.lowerType(pt, param_ty); | 1365 | const param_llvm_ty = try o.lowerType(param_ty); |
| 1435 | const param_alignment = param_ty.abiAlignment(zcu).toLlvm(); | 1366 | const param_alignment = param_ty.abiAlignment(zcu).toLlvm(); |
| 1436 | const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, param_alignment, target); | 1367 | const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, param_alignment, target); |
| 1437 | const llvm_ty = try o.builder.structType(.normal, field_types); | 1368 | const llvm_ty = try o.builder.structType(.normal, field_types); |
| 1438 | for (0..field_types.len) |field_i| { | 1369 | const llvm_args_start = it.llvm_index - field_types.len; |
| 1439 | const param = wip.arg(llvm_arg_i); | 1370 | for (0..field_types.len, llvm_args_start..) |field_i, llvm_arg_index| { |
| 1440 | llvm_arg_i += 1; | 1371 | const param = wip.arg(@intCast(llvm_arg_index)); |
| 1441 | const field_ptr = try wip.gepStruct(llvm_ty, arg_ptr, field_i, ""); | 1372 | const field_ptr = try wip.gepStruct(llvm_ty, arg_ptr, field_i, ""); |
| 1442 | const alignment = Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8)); | 1373 | const alignment: Builder.Alignment = .fromByteUnits(@divExact(target.ptrBitWidth(), 8)); |
| 1443 | _ = try wip.store(.normal, param, field_ptr, alignment); | 1374 | _ = try wip.store(.normal, param, field_ptr, alignment); |
| 1444 | } | 1375 | } |
| 1445 | | 1376 | |
| 1446 | const is_by_ref = isByRef(param_ty, zcu); | 1377 | if (isByRef(param_ty, zcu)) { |
| 1447 | args.appendAssumeCapacity(if (is_by_ref) | 1378 | args.appendAssumeCapacity(arg_ptr); |
| 1448 | arg_ptr | 1379 | } else { |
| 1449 | else | 1380 | args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment, "")); |
| 1450 | try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment, "")); | 1381 | } |
| 1451 | }, | 1382 | }, |
| 1452 | .float_array => { | 1383 | .float_array => { |
| 1453 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | 1384 | const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| 1454 | const param_llvm_ty = try o.lowerType(pt, param_ty); | 1385 | const param_llvm_ty = try o.lowerType(param_ty); |
| 1455 | const param = wip.arg(llvm_arg_i); | 1386 | const param = wip.arg(it.llvm_index - 1); |
| 1456 | llvm_arg_i += 1; | | |
| 1457 | | 1387 | |
| 1458 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | 1388 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); |
| 1459 | const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target); | 1389 | const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target); |
| 1460 | _ = try wip.store(.normal, param, arg_ptr, alignment); | 1390 | _ = try wip.store(.normal, param, arg_ptr, alignment); |
| 1461 | | 1391 | |
| 1462 | args.appendAssumeCapacity(if (isByRef(param_ty, zcu)) | 1392 | if (isByRef(param_ty, zcu)) { |
| 1463 | arg_ptr | 1393 | args.appendAssumeCapacity(arg_ptr); |
| 1464 | else | 1394 | } else { |
| 1465 | try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); | 1395 | args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); |
| | 1396 | } |
| 1466 | }, | 1397 | }, |
| 1467 | .i32_array, .i64_array => { | 1398 | .i32_array, .i64_array => { |
| 1468 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | 1399 | const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| 1469 | const param_llvm_ty = try o.lowerType(pt, param_ty); | 1400 | const param_llvm_ty = try o.lowerType(param_ty); |
| 1470 | const param = wip.arg(llvm_arg_i); | 1401 | const param = wip.arg(it.llvm_index - 1); |
| 1471 | llvm_arg_i += 1; | | |
| 1472 | | 1402 | |
| 1473 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | 1403 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); |
| 1474 | const arg_ptr = try buildAllocaInner(&wip, param.typeOfWip(&wip), alignment, target); | 1404 | const arg_ptr = try buildAllocaInner(&wip, param.typeOfWip(&wip), alignment, target); |
| 1475 | _ = try wip.store(.normal, param, arg_ptr, alignment); | 1405 | _ = try wip.store(.normal, param, arg_ptr, alignment); |
| 1476 | | 1406 | |
| 1477 | args.appendAssumeCapacity(if (isByRef(param_ty, zcu)) | 1407 | if (isByRef(param_ty, zcu)) { |
| 1478 | arg_ptr | 1408 | args.appendAssumeCapacity(arg_ptr); |
| 1479 | else | 1409 | } else { |
| 1480 | try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); | 1410 | args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); |
| | 1411 | } |
| 1481 | }, | 1412 | }, |
| 1482 | } | 1413 | } |
| 1483 | } | 1414 | } |
| 1484 | } | 1415 | |
| | 1416 | break :implicit_args .{ ret_ptr, err_ret_trace }; |
| | 1417 | }; |
| 1485 | | 1418 | |
| 1486 | const file, const subprogram = if (!wip.strip) debug_info: { | 1419 | const file, const subprogram = if (!wip.strip) debug_info: { |
| 1487 | const file = try o.getDebugFile(pt, file_scope); | 1420 | const file = try o.getDebugFile(file_scope); |
| 1488 | | 1421 | |
| 1489 | const line_number = zcu.navSrcLine(func.owner_nav) + 1; | 1422 | const line_number = zcu.navSrcLine(func.owner_nav) + 1; |
| 1490 | const is_internal_linkage = ip.indexToKey(nav.resolved.?.value) != .@"extern"; | 1423 | const is_internal_linkage = ip.indexToKey(nav.resolved.?.value) != .@"extern"; |
| ... | @@ -1493,7 +1426,7 @@ pub const Object = struct { | ... | @@ -1493,7 +1426,7 @@ pub const Object = struct { |
| 1493 | const subprogram = try o.builder.debugSubprogram( | 1426 | const subprogram = try o.builder.debugSubprogram( |
| 1494 | file, | 1427 | file, |
| 1495 | try o.builder.metadataString(nav.name.toSlice(ip)), | 1428 | try o.builder.metadataString(nav.name.toSlice(ip)), |
| 1496 | try o.builder.metadataStringFromStrtabString(function_index.name(&o.builder)), | 1429 | try o.builder.metadataString(nav.fqn.toSlice(ip)), |
| 1497 | line_number, | 1430 | line_number, |
| 1498 | line_number + func.lbrace_line, | 1431 | line_number + func.lbrace_line, |
| 1499 | debug_decl_type, | 1432 | debug_decl_type, |
| ... | @@ -1510,7 +1443,7 @@ pub const Object = struct { | ... | @@ -1510,7 +1443,7 @@ pub const Object = struct { |
| 1510 | }, | 1443 | }, |
| 1511 | o.debug_compile_unit.unwrap().?, | 1444 | o.debug_compile_unit.unwrap().?, |
| 1512 | ); | 1445 | ); |
| 1513 | function_index.setSubprogram(subprogram, &o.builder); | 1446 | llvm_function.setSubprogram(subprogram, &o.builder); |
| 1514 | break :debug_info .{ file, subprogram }; | 1447 | break :debug_info .{ file, subprogram }; |
| 1515 | } else .{undefined} ** 2; | 1448 | } else .{undefined} ** 2; |
| 1516 | | 1449 | |
| ... | @@ -1527,7 +1460,7 @@ pub const Object = struct { | ... | @@ -1527,7 +1460,7 @@ pub const Object = struct { |
| 1527 | const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len}); | 1460 | const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len}); |
| 1528 | const counters_variable = try o.builder.addVariable(anon_name, .void, .default); | 1461 | const counters_variable = try o.builder.addVariable(anon_name, .void, .default); |
| 1529 | try o.used.append(gpa, counters_variable.toConst(&o.builder)); | 1462 | try o.used.append(gpa, counters_variable.toConst(&o.builder)); |
| 1530 | counters_variable.setLinkage(.private, &o.builder); | 1463 | counters_variable.ptrConst(&o.builder).global.setLinkage(.private, &o.builder); |
| 1531 | counters_variable.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); | 1464 | counters_variable.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); |
| 1532 | | 1465 | |
| 1533 | if (target.ofmt == .macho) { | 1466 | if (target.ofmt == .macho) { |
| ... | @@ -1543,10 +1476,12 @@ pub const Object = struct { | ... | @@ -1543,10 +1476,12 @@ pub const Object = struct { |
| 1543 | }; | 1476 | }; |
| 1544 | | 1477 | |
| 1545 | var fg: FuncGen = .{ | 1478 | var fg: FuncGen = .{ |
| | 1479 | .object = o, |
| | 1480 | .nav_index = func.owner_nav, |
| | 1481 | .pt = pt, |
| 1546 | .gpa = gpa, | 1482 | .gpa = gpa, |
| 1547 | .air = air.*, | 1483 | .air = air.*, |
| 1548 | .liveness = liveness.*.?, | 1484 | .liveness = liveness.*.?, |
| 1549 | .ng = &ng, | | |
| 1550 | .wip = wip, | 1485 | .wip = wip, |
| 1551 | .is_naked = fn_info.cc == .naked, | 1486 | .is_naked = fn_info.cc == .naked, |
| 1552 | .fuzz = fuzz, | 1487 | .fuzz = fuzz, |
| ... | @@ -1561,22 +1496,18 @@ pub const Object = struct { | ... | @@ -1561,22 +1496,18 @@ pub const Object = struct { |
| 1561 | .sync_scope = if (owner_mod.single_threaded) .singlethread else .system, | 1496 | .sync_scope = if (owner_mod.single_threaded) .singlethread else .system, |
| 1562 | .file = file, | 1497 | .file = file, |
| 1563 | .scope = subprogram, | 1498 | .scope = subprogram, |
| | 1499 | .inlined_at = .none, |
| 1564 | .base_line = zcu.navSrcLine(func.owner_nav), | 1500 | .base_line = zcu.navSrcLine(func.owner_nav), |
| 1565 | .prev_dbg_line = 0, | 1501 | .prev_dbg_line = 0, |
| 1566 | .prev_dbg_column = 0, | 1502 | .prev_dbg_column = 0, |
| 1567 | .err_ret_trace = err_ret_trace, | 1503 | .err_ret_trace = err_ret_trace, |
| 1568 | .disable_intrinsics = disable_intrinsics, | 1504 | .disable_intrinsics = disable_intrinsics, |
| | 1505 | .allowzero_access = false, |
| 1569 | }; | 1506 | }; |
| 1570 | defer fg.deinit(); | 1507 | defer fg.deinit(); |
| 1571 | deinit_wip = false; | 1508 | deinit_wip = false; |
| 1572 | | 1509 | |
| 1573 | fg.genBody(air.getMainBody(), .poi) catch |err| switch (err) { | 1510 | try fg.genBody(air.getMainBody(), .poi); |
| 1574 | error.CodegenFail => switch (zcu.codegenFailMsg(func.owner_nav, ng.err_msg.?)) { | | |
| 1575 | error.CodegenFail => return, | | |
| 1576 | error.OutOfMemory => |e| return e, | | |
| 1577 | }, | | |
| 1578 | else => |e| return e, | | |
| 1579 | }; | | |
| 1580 | | 1511 | |
| 1581 | // If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole | 1512 | // If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole |
| 1582 | // function as considering null pointers valid so that LLVM's optimizers don't remove these | 1513 | // function as considering null pointers valid so that LLVM's optimizers don't remove these |
| ... | @@ -1587,7 +1518,7 @@ pub const Object = struct { | ... | @@ -1587,7 +1518,7 @@ pub const Object = struct { |
| 1587 | _ = try attributes.removeFnAttr(.null_pointer_is_valid); | 1518 | _ = try attributes.removeFnAttr(.null_pointer_is_valid); |
| 1588 | } | 1519 | } |
| 1589 | | 1520 | |
| 1590 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); | 1521 | llvm_function.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 1591 | | 1522 | |
| 1592 | if (fg.fuzz) |*f| { | 1523 | if (fg.fuzz) |*f| { |
| 1593 | { | 1524 | { |
| ... | @@ -1602,31 +1533,162 @@ pub const Object = struct { | ... | @@ -1602,31 +1533,162 @@ pub const Object = struct { |
| 1602 | // Due to error "members of llvm.compiler.used must be named", this global needs a name. | 1533 | // Due to error "members of llvm.compiler.used must be named", this global needs a name. |
| 1603 | const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len}); | 1534 | const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len}); |
| 1604 | const pcs_variable = try o.builder.addVariable(anon_name, array_llvm_ty, .default); | 1535 | const pcs_variable = try o.builder.addVariable(anon_name, array_llvm_ty, .default); |
| 1605 | try o.used.append(gpa, pcs_variable.toConst(&o.builder)); | 1536 | try pcs_variable.setInitializer(init_val, &o.builder); |
| 1606 | pcs_variable.setLinkage(.private, &o.builder); | | |
| 1607 | pcs_variable.setMutability(.constant, &o.builder); | 1537 | pcs_variable.setMutability(.constant, &o.builder); |
| | 1538 | pcs_variable.setSection(switch (target.ofmt) { |
| | 1539 | .macho => try o.builder.string("__DATA,__sancov_pcs1"), |
| | 1540 | else => try o.builder.string("__sancov_pcs1"), |
| | 1541 | }, &o.builder); |
| 1608 | pcs_variable.setAlignment(Type.usize.abiAlignment(zcu).toLlvm(), &o.builder); | 1542 | pcs_variable.setAlignment(Type.usize.abiAlignment(zcu).toLlvm(), &o.builder); |
| 1609 | if (target.ofmt == .macho) { | 1543 | const pcs_global = pcs_variable.ptrConst(&o.builder).global; |
| 1610 | pcs_variable.setSection(try o.builder.string("__DATA,__sancov_pcs1"), &o.builder); | 1544 | pcs_global.setLinkage(.private, &o.builder); |
| 1611 | } else { | 1545 | try o.used.append(gpa, pcs_global.toConst()); |
| 1612 | pcs_variable.setSection(try o.builder.string("__sancov_pcs1"), &o.builder); | | |
| 1613 | } | | |
| 1614 | try pcs_variable.setInitializer(init_val, &o.builder); | | |
| 1615 | } | 1546 | } |
| 1616 | | 1547 | |
| 1617 | try fg.wip.finish(); | 1548 | try fg.wip.finish(); |
| 1618 | try o.flushTypePool(pt); | 1549 | try o.flushTypePool(pt); |
| 1619 | } | 1550 | } |
| 1620 | | 1551 | |
| 1621 | pub fn updateNav(self: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void { | 1552 | pub fn updateNav(o: *Object, pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) !void { |
| 1622 | var ng: NavGen = .{ | 1553 | const zcu = o.zcu; |
| 1623 | .object = self, | 1554 | const ip = &zcu.intern_pool; |
| 1624 | .nav_index = nav_index, | 1555 | const comp = zcu.comp; |
| 1625 | .pt = pt, | 1556 | const gpa = comp.gpa; |
| 1626 | .err_msg = null, | 1557 | |
| | 1558 | const nav = ip.getNav(nav_id); |
| | 1559 | const resolved = nav.resolved.?; |
| | 1560 | |
| | 1561 | const opt_extern: ?InternPool.Key.Extern = switch (ip.indexToKey(resolved.value)) { |
| | 1562 | .@"extern" => |@"extern"| @"extern", |
| | 1563 | else => null, |
| | 1564 | }; |
| | 1565 | const nav_ty: Type = .fromInterned(resolved.type); |
| | 1566 | const llvm_ty: Builder.Type = if (opt_extern != null) ty: { |
| | 1567 | // We *must* lower this declaration no matter what. If it has a type we can't actually |
| | 1568 | // represent (because it doesn't have runtime bits), we instead lower as the zero-size |
| | 1569 | // type `[0 x i8]`. I don't think the type on an extern declaration actually does much |
| | 1570 | // anyway. |
| | 1571 | if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty); |
| | 1572 | break :ty try o.builder.arrayType(0, .i8); |
| | 1573 | } else if (nav_ty.hasRuntimeBits(zcu)) ty: { |
| | 1574 | break :ty try o.lowerType(nav_ty); |
| | 1575 | } else { |
| | 1576 | // This is a non-extern zero-bit `Nav`---we're not interested in it. |
| | 1577 | // TODO: we might need to rethink this a little under incremental compilation. If a |
| | 1578 | // declaration becomes zero-bit, we can't just leave its old value there, because it |
| | 1579 | // might now be ill-formed. |
| | 1580 | return; |
| | 1581 | }; |
| | 1582 | |
| | 1583 | const gop = try o.nav_map.getOrPut(gpa, nav_id); |
| | 1584 | if (!gop.found_existing) { |
| | 1585 | errdefer assert(o.nav_map.remove(nav_id)); |
| | 1586 | // First time lowering this NAV! Create a fresh global. |
| | 1587 | const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip)); |
| | 1588 | gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{ |
| | 1589 | .type = .void, // placeholder; populated below |
| | 1590 | .kind = .{ .alias = .none }, // placeholder; populated below |
| | 1591 | }); |
| | 1592 | } |
| | 1593 | const llvm_global = gop.value_ptr.*; |
| | 1594 | |
| | 1595 | llvm_global.ptr(&o.builder).type = llvm_ty; |
| | 1596 | llvm_global.ptr(&o.builder).addr_space = toLlvmAddressSpace(resolved.@"addrspace", zcu.getTarget()); |
| | 1597 | |
| | 1598 | if (opt_extern) |@"extern"| { |
| | 1599 | const name = name: { |
| | 1600 | const name_slice = nav.name.toSlice(ip); |
| | 1601 | if (zcu.getTarget().cpu.arch.isWasm() and nav_ty.zigTypeTag(zcu) == .@"fn") { |
| | 1602 | if (@"extern".lib_name.toSlice(ip)) |lib_name_slice| { |
| | 1603 | if (!std.mem.eql(u8, lib_name_slice, "c")) { |
| | 1604 | break :name try o.builder.strtabStringFmt("{s}|{s}", .{ name_slice, lib_name_slice }); |
| | 1605 | } |
| | 1606 | } |
| | 1607 | } |
| | 1608 | break :name try o.builder.strtabString(name_slice); |
| | 1609 | }; |
| | 1610 | if (o.builder.getGlobal(name)) |other_global| { |
| | 1611 | if (other_global != llvm_global) { |
| | 1612 | // Another global already has this name; just use it in place of this global. |
| | 1613 | try llvm_global.replace(other_global, &o.builder); |
| | 1614 | return; |
| | 1615 | } |
| | 1616 | } |
| | 1617 | try llvm_global.rename(name, &o.builder); |
| | 1618 | llvm_global.ptr(&o.builder).unnamed_addr = .default; |
| | 1619 | llvm_global.ptr(&o.builder).dll_storage_class = switch (@"extern".is_dll_import) { |
| | 1620 | true => .dllimport, |
| | 1621 | false => .default, |
| | 1622 | }; |
| | 1623 | llvm_global.ptr(&o.builder).linkage = switch (@"extern".linkage) { |
| | 1624 | .internal => if (o.builder.strip) .private else .internal, |
| | 1625 | .strong => .external, |
| | 1626 | .weak => .extern_weak, |
| | 1627 | .link_once => unreachable, |
| | 1628 | }; |
| | 1629 | llvm_global.ptr(&o.builder).visibility = .fromSymbolVisibility(@"extern".visibility); |
| | 1630 | } else { |
| | 1631 | llvm_global.ptr(&o.builder).linkage = if (o.builder.strip) .private else .internal; |
| | 1632 | llvm_global.ptr(&o.builder).visibility = .default; |
| | 1633 | llvm_global.ptr(&o.builder).dll_storage_class = .default; |
| | 1634 | llvm_global.ptr(&o.builder).unnamed_addr = .unnamed_addr; |
| | 1635 | } |
| | 1636 | |
| | 1637 | const llvm_align = switch (resolved.@"align") { |
| | 1638 | .none => nav_ty.abiAlignment(zcu).toLlvm(), |
| | 1639 | else => |a| a.toLlvm(), |
| 1627 | }; | 1640 | }; |
| 1628 | try ng.genDecl(); | 1641 | const llvm_section: Builder.String = if (resolved.@"linksection".toSlice(ip)) |section| s: { |
| 1629 | try self.flushTypePool(pt); | 1642 | break :s try o.builder.string(section); |
| | 1643 | } else .none; |
| | 1644 | |
| | 1645 | // Actual function bodies with AIR go through `updateFunc` instead, so the only functions we |
| | 1646 | // can see are extern functions or other comptime function body values (e.g. undefined). Of |
| | 1647 | // these, only extern functions need to be lowered to LLVM functions. |
| | 1648 | if (opt_extern != null and nav_ty.zigTypeTag(zcu) == .@"fn" and nav_ty.fnHasRuntimeBits(zcu)) { |
| | 1649 | const llvm_function: Builder.Function.Index = switch (llvm_global.ptrConst(&o.builder).kind) { |
| | 1650 | .function => |function| function, // re-use existing `Builder.Function` |
| | 1651 | .replaced, .alias, .variable => try llvm_global.toNewFunction(&o.builder), |
| | 1652 | }; |
| | 1653 | llvm_function.setAlignment(llvm_align, &o.builder); |
| | 1654 | llvm_function.setSection(llvm_section, &o.builder); |
| | 1655 | try o.addLlvmFunctionAttributes(pt, nav_id, llvm_function); |
| | 1656 | } else { |
| | 1657 | const file_scope = nav.srcInst(ip).resolveFile(ip); |
| | 1658 | const mod = zcu.fileByIndex(file_scope).mod.?; |
| | 1659 | |
| | 1660 | const llvm_variable: Builder.Variable.Index = switch (llvm_global.ptrConst(&o.builder).kind) { |
| | 1661 | .variable => |variable| variable, // re-use existing `Builder.Variable` |
| | 1662 | .replaced, .alias, .function => try llvm_global.toNewVariable(&o.builder), |
| | 1663 | }; |
| | 1664 | llvm_variable.setAlignment(llvm_align, &o.builder); |
| | 1665 | llvm_variable.setSection(llvm_section, &o.builder); |
| | 1666 | llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder); |
| | 1667 | try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value), &o.builder); |
| | 1668 | llvm_variable.setThreadLocal(tl: { |
| | 1669 | if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic; |
| | 1670 | break :tl .default; |
| | 1671 | }, &o.builder); |
| | 1672 | |
| | 1673 | if (!mod.strip) { |
| | 1674 | const debug_file = try o.getDebugFile(file_scope); |
| | 1675 | const debug_global_var_expr = try o.builder.debugGlobalVarExpression( |
| | 1676 | try o.builder.debugGlobalVar( |
| | 1677 | try o.builder.metadataString(nav.name.toSlice(ip)), // Name |
| | 1678 | try o.builder.metadataString(nav.fqn.toSlice(ip)), // Linkage name |
| | 1679 | debug_file, // File |
| | 1680 | debug_file, // Scope |
| | 1681 | zcu.navSrcLine(nav_id) + 1, |
| | 1682 | try o.getDebugType(pt, nav_ty), |
| | 1683 | llvm_variable, |
| | 1684 | .{ .local = llvm_global.ptrConst(&o.builder).linkage == .internal }, |
| | 1685 | ), |
| | 1686 | try o.builder.debugExpression(&.{}), |
| | 1687 | ); |
| | 1688 | llvm_variable.setGlobalVariableExpression(debug_global_var_expr, &o.builder); |
| | 1689 | try o.debug_globals.append(o.gpa, debug_global_var_expr); |
| | 1690 | } |
| | 1691 | } |
| 1630 | } | 1692 | } |
| 1631 | | 1693 | |
| 1632 | fn flushTypePool(o: *Object, pt: Zcu.PerThread) Allocator.Error!void { | 1694 | fn flushTypePool(o: *Object, pt: Zcu.PerThread) Allocator.Error!void { |
| ... | @@ -1634,19 +1696,43 @@ pub const Object = struct { | ... | @@ -1634,19 +1696,43 @@ pub const Object = struct { |
| 1634 | } | 1696 | } |
| 1635 | | 1697 | |
| 1636 | pub fn updateExports( | 1698 | pub fn updateExports( |
| 1637 | self: *Object, | 1699 | o: *Object, |
| 1638 | pt: Zcu.PerThread, | | |
| 1639 | exported: Zcu.Exported, | 1700 | exported: Zcu.Exported, |
| 1640 | export_indices: []const Zcu.Export.Index, | 1701 | export_indices: []const Zcu.Export.Index, |
| 1641 | ) link.File.UpdateExportsError!void { | 1702 | ) link.File.UpdateExportsError!void { |
| 1642 | const zcu = pt.zcu; | 1703 | const zcu = o.zcu; |
| 1643 | const nav_index = switch (exported) { | | |
| 1644 | .nav => |nav| nav, | | |
| 1645 | .uav => |uav| return updateExportedValue(self, pt, uav, export_indices), | | |
| 1646 | }; | | |
| 1647 | const ip = &zcu.intern_pool; | 1704 | const ip = &zcu.intern_pool; |
| 1648 | const global_index = self.nav_map.get(nav_index).?; | 1705 | const ty: Type, const llvm_ptr: Builder.Constant = switch (exported) { |
| | 1706 | .nav => |nav| exp: { |
| | 1707 | const nav_ty: Type = .fromInterned(ip.getNav(nav).resolved.?.type); |
| | 1708 | const nav_ref = try o.lowerNavRef(nav); |
| | 1709 | break :exp .{ nav_ty, nav_ref }; |
| | 1710 | }, |
| | 1711 | .uav => |uav| exp: { |
| | 1712 | const uav_ty = Value.fromInterned(uav).typeOf(zcu); |
| | 1713 | const uav_ref = try o.lowerUavRef( |
| | 1714 | uav, |
| | 1715 | uav_ty.abiAlignment(zcu), |
| | 1716 | target_util.defaultAddressSpace(zcu.getTarget(), .global_constant), |
| | 1717 | ); |
| | 1718 | break :exp .{ uav_ty, uav_ref }; |
| | 1719 | }, |
| | 1720 | }; |
| | 1721 | switch (llvm_ptr.unwrap()) { |
| | 1722 | .global => |global| return o.updateExportedGlobal(global, ty, export_indices), |
| | 1723 | .constant => @panic("LLVM TODO: export zero-bit value"), |
| | 1724 | } |
| | 1725 | } |
| | 1726 | |
| | 1727 | fn updateExportedGlobal( |
| | 1728 | o: *Object, |
| | 1729 | global_index: Builder.Global.Index, |
| | 1730 | ty: Type, |
| | 1731 | export_indices: []const Zcu.Export.Index, |
| | 1732 | ) link.File.UpdateExportsError!void { |
| | 1733 | const zcu = o.zcu; |
| 1649 | const comp = zcu.comp; | 1734 | const comp = zcu.comp; |
| | 1735 | const ip = &zcu.intern_pool; |
| 1650 | | 1736 | |
| 1651 | // If we're on COFF and linking with LLD, the linker cares about our exports to determine the subsystem in use. | 1737 | // If we're on COFF and linking with LLD, the linker cares about our exports to determine the subsystem in use. |
| 1652 | coff_export_flags: { | 1738 | coff_export_flags: { |
| ... | @@ -1656,7 +1742,7 @@ pub const Object = struct { | ... | @@ -1656,7 +1742,7 @@ pub const Object = struct { |
| 1656 | .elf, .wasm => break :coff_export_flags, | 1742 | .elf, .wasm => break :coff_export_flags, |
| 1657 | .coff => |*coff| coff, | 1743 | .coff => |*coff| coff, |
| 1658 | }; | 1744 | }; |
| 1659 | if (!ip.isFunctionType(ip.getNav(nav_index).resolved.?.type)) break :coff_export_flags; | 1745 | if (ty.zigTypeTag(zcu) != .@"fn") break :coff_export_flags; |
| 1660 | const flags = &coff.lld_export_flags; | 1746 | const flags = &coff.lld_export_flags; |
| 1661 | for (export_indices) |export_index| { | 1747 | for (export_indices) |export_index| { |
| 1662 | const name = export_index.ptr(zcu).opts.name; | 1748 | const name = export_index.ptr(zcu).opts.name; |
| ... | @@ -1669,153 +1755,88 @@ pub const Object = struct { | ... | @@ -1669,153 +1755,88 @@ pub const Object = struct { |
| 1669 | } | 1755 | } |
| 1670 | } | 1756 | } |
| 1671 | | 1757 | |
| 1672 | if (export_indices.len != 0) { | 1758 | // If the first export specifies a linksection, set the exported variable's section to that |
| 1673 | return updateExportedGlobal(self, zcu, global_index, export_indices); | 1759 | // one. This is kind of a hack because `std.builtin.ExportOptions.section` doesn't actually |
| 1674 | } else { | 1760 | // make much sense: the linksection should be associated with the declaration itself rather |
| 1675 | const fqn = try self.builder.strtabString(ip.getNav(nav_index).fqn.toSlice(ip)); | 1761 | // than some particular symbol it is exported as! |
| 1676 | try global_index.rename(fqn, &self.builder); | 1762 | if (export_indices[0].ptr(zcu).opts.section.toSlice(ip)) |section_slice| { |
| 1677 | global_index.setLinkage(if (self.builder.strip) .private else .internal, &self.builder); | 1763 | const variable = &global_index.ptrConst(&o.builder).kind.variable; |
| 1678 | if (comp.config.dll_export_fns) | 1764 | variable.setSection(try o.builder.string(section_slice), &o.builder); |
| 1679 | global_index.setDllStorageClass(.default, &self.builder); | | |
| 1680 | global_index.setUnnamedAddr(.unnamed_addr, &self.builder); | | |
| 1681 | } | 1765 | } |
| 1682 | } | | |
| 1683 | | 1766 | |
| 1684 | fn updateExportedValue( | 1767 | const llvm_global_ty = global_index.typeOf(&o.builder); |
| 1685 | o: *Object, | | |
| 1686 | pt: Zcu.PerThread, | | |
| 1687 | exported_value: InternPool.Index, | | |
| 1688 | export_indices: []const Zcu.Export.Index, | | |
| 1689 | ) link.File.UpdateExportsError!void { | | |
| 1690 | const zcu = pt.zcu; | | |
| 1691 | const gpa = zcu.gpa; | | |
| 1692 | const ip = &zcu.intern_pool; | | |
| 1693 | const main_exp_name = try o.builder.strtabString(export_indices[0].ptr(zcu).opts.name.toSlice(ip)); | | |
| 1694 | const global_index = i: { | | |
| 1695 | const gop = try o.uav_map.getOrPut(gpa, exported_value); | | |
| 1696 | if (gop.found_existing) { | | |
| 1697 | const global_index = gop.value_ptr.*; | | |
| 1698 | try global_index.rename(main_exp_name, &o.builder); | | |
| 1699 | break :i global_index; | | |
| 1700 | } | | |
| 1701 | const llvm_addr_space = toLlvmAddressSpace(.generic, o.target); | | |
| 1702 | const variable_index = try o.builder.addVariable( | | |
| 1703 | main_exp_name, | | |
| 1704 | try o.lowerType(pt, Type.fromInterned(ip.typeOf(exported_value))), | | |
| 1705 | llvm_addr_space, | | |
| 1706 | ); | | |
| 1707 | const global_index = variable_index.ptrConst(&o.builder).global; | | |
| 1708 | gop.value_ptr.* = global_index; | | |
| 1709 | // This line invalidates `gop`. | | |
| 1710 | const init_val = try o.lowerValue(pt, exported_value); | | |
| 1711 | try variable_index.setInitializer(init_val, &o.builder); | | |
| 1712 | break :i global_index; | | |
| 1713 | }; | | |
| 1714 | return updateExportedGlobal(o, zcu, global_index, export_indices); | | |
| 1715 | } | | |
| 1716 | | 1768 | |
| 1717 | fn updateExportedGlobal( | 1769 | // All exports are represented as aliases to the original global. |
| 1718 | o: *Object, | | |
| 1719 | zcu: *Zcu, | | |
| 1720 | global_index: Builder.Global.Index, | | |
| 1721 | export_indices: []const Zcu.Export.Index, | | |
| 1722 | ) link.File.UpdateExportsError!void { | | |
| 1723 | const comp = zcu.comp; | | |
| 1724 | const ip = &zcu.intern_pool; | | |
| 1725 | const first_export = export_indices[0].ptr(zcu); | | |
| 1726 | | | |
| 1727 | // We will rename this global to have a name matching `first_export`. | | |
| 1728 | // Successive exports become aliases. | | |
| 1729 | // If the first export name already exists, then there is a corresponding | | |
| 1730 | // extern global - we replace it with this global. | | |
| 1731 | const first_exp_name = try o.builder.strtabString(first_export.opts.name.toSlice(ip)); | | |
| 1732 | if (o.builder.getGlobal(first_exp_name)) |other_global| replace: { | | |
| 1733 | if (other_global.toConst().getBase(&o.builder) == global_index.toConst().getBase(&o.builder)) { | | |
| 1734 | break :replace; // this global already has the name we want | | |
| 1735 | } | | |
| 1736 | try global_index.takeName(other_global, &o.builder); | | |
| 1737 | try other_global.replace(global_index, &o.builder); | | |
| 1738 | // Problem: now we need to replace in the decl_map that | | |
| 1739 | // the extern decl index points to this new global. However we don't | | |
| 1740 | // know the decl index. | | |
| 1741 | // Even if we did, a future incremental update to the extern would then | | |
| 1742 | // treat the LLVM global as an extern rather than an export, so it would | | |
| 1743 | // need a way to check that. | | |
| 1744 | // This is a TODO that needs to be solved when making | | |
| 1745 | // the LLVM backend support incremental compilation. | | |
| 1746 | } else { | | |
| 1747 | try global_index.rename(first_exp_name, &o.builder); | | |
| 1748 | } | | |
| 1749 | | 1770 | |
| 1750 | global_index.setUnnamedAddr(.default, &o.builder); | 1771 | // TODO: we currently do not delete old exports. To do that we'll need to track which |
| 1751 | if (comp.config.dll_export_fns and first_export.opts.visibility != .hidden) | 1772 | // globals actually *are* exports. |
| 1752 | global_index.setDllStorageClass(.dllexport, &o.builder); | | |
| 1753 | global_index.setLinkage(switch (first_export.opts.linkage) { | | |
| 1754 | .internal => unreachable, | | |
| 1755 | .strong => .external, | | |
| 1756 | .weak => .weak_odr, | | |
| 1757 | .link_once => .linkonce_odr, | | |
| 1758 | }, &o.builder); | | |
| 1759 | global_index.setVisibility(switch (first_export.opts.visibility) { | | |
| 1760 | .default => .default, | | |
| 1761 | .hidden => .hidden, | | |
| 1762 | .protected => .protected, | | |
| 1763 | }, &o.builder); | | |
| 1764 | if (first_export.opts.section.toSlice(ip)) |section| | | |
| 1765 | switch (global_index.ptrConst(&o.builder).kind) { | | |
| 1766 | .variable => |impl_index| impl_index.setSection( | | |
| 1767 | try o.builder.string(section), | | |
| 1768 | &o.builder, | | |
| 1769 | ), | | |
| 1770 | .function => unreachable, | | |
| 1771 | .alias => unreachable, | | |
| 1772 | .replaced => unreachable, | | |
| 1773 | }; | | |
| 1774 | | 1773 | |
| 1775 | // If a Decl is exported more than one time (which is rare), | 1774 | for (export_indices) |export_idx| { |
| 1776 | // we add aliases for all but the first export. | | |
| 1777 | // TODO LLVM C API does not support deleting aliases. | | |
| 1778 | // The planned solution to this is https://github.com/ziglang/zig/issues/13265 | | |
| 1779 | // Until then we iterate over existing aliases and make them point | | |
| 1780 | // to the correct decl, or otherwise add a new alias. Old aliases are leaked. | | |
| 1781 | for (export_indices[1..]) |export_idx| { | | |
| 1782 | const exp = export_idx.ptr(zcu); | 1775 | const exp = export_idx.ptr(zcu); |
| 1783 | const exp_name = try o.builder.strtabString(exp.opts.name.toSlice(ip)); | 1776 | const exp_name = try o.builder.strtabString(exp.opts.name.toSlice(ip)); |
| 1784 | if (o.builder.getGlobal(exp_name)) |global| { | 1777 | |
| 1785 | switch (global.ptrConst(&o.builder).kind) { | 1778 | // Our goal is to make an alias with the name `exp_name`, but if that name is already |
| | 1779 | // taken by some existing global, we need to figure out what to do with that existing |
| | 1780 | // global. |
| | 1781 | // |
| | 1782 | // The name, aliasee, and type will be set within this block. Other properties of the |
| | 1783 | // alias will be set below. |
| | 1784 | const alias_global: Builder.Global.Index = global: { |
| | 1785 | const existing_global = o.builder.getGlobal(exp_name) orelse { |
| | 1786 | // There is no existing global with this name, so make a new alias. |
| | 1787 | const alias = try o.builder.addAlias( |
| | 1788 | exp_name, |
| | 1789 | llvm_global_ty, |
| | 1790 | .default, |
| | 1791 | global_index.toConst(), |
| | 1792 | ); |
| | 1793 | break :global alias.ptrConst(&o.builder).global; |
| | 1794 | }; |
| | 1795 | // There is an existing global with this name, so we can't just create an alias. We |
| | 1796 | // need to figure out what to do with the existing global instead. |
| | 1797 | switch (existing_global.ptrConst(&o.builder).kind) { |
| 1786 | .alias => |alias| { | 1798 | .alias => |alias| { |
| | 1799 | // We can just repurpose the existing alias. |
| 1787 | alias.setAliasee(global_index.toConst(), &o.builder); | 1800 | alias.setAliasee(global_index.toConst(), &o.builder); |
| 1788 | continue; | 1801 | alias.ptrConst(&o.builder).global.ptr(&o.builder).type = global_index.typeOf(&o.builder); |
| | 1802 | break :global existing_global; |
| 1789 | }, | 1803 | }, |
| 1790 | .variable, .function => { | 1804 | .variable, .function => { |
| 1791 | // This existing global is an `extern` corresponding to this export. | 1805 | // This must be an extern, which is no good to us---we need an alias. The |
| 1792 | // Replace it with the global being exported. | 1806 | // extern should refer to the value we're exporting, so replace it with the |
| 1793 | // This existing global must be replaced with the alias. | 1807 | // exported value. That will free up the name for us to create a new alias. |
| 1794 | try global.rename(.empty, &o.builder); | 1808 | // We need to make a new global which is an alias. Replace this existing one |
| 1795 | try global.replace(global_index, &o.builder); | 1809 | // with the target global, making the name available and fixing references |
| | 1810 | // to this global to point to the target. |
| | 1811 | try existing_global.replace(global_index, &o.builder); |
| | 1812 | // The name is now free, so create an alias. |
| | 1813 | const alias = try o.builder.addAlias( |
| | 1814 | exp_name, |
| | 1815 | llvm_global_ty, |
| | 1816 | .default, |
| | 1817 | global_index.toConst(), |
| | 1818 | ); |
| | 1819 | break :global alias.ptrConst(&o.builder).global; |
| 1796 | }, | 1820 | }, |
| 1797 | .replaced => unreachable, | 1821 | .replaced => unreachable, // a replaced global would have lost the name `exp_name` |
| 1798 | } | 1822 | } |
| 1799 | } | 1823 | }; |
| 1800 | const alias_index = try o.builder.addAlias( | 1824 | |
| 1801 | .empty, | 1825 | // Now for a bit of setup which |
| 1802 | global_index.typeOf(&o.builder), | 1826 | |
| 1803 | .default, | 1827 | // We need the alias to *not* be `unnamed_addr` to ensure that the alias address equals |
| 1804 | global_index.toConst(), | 1828 | // the address of the original global. |
| 1805 | ); | 1829 | alias_global.setUnnamedAddr(.default, &o.builder); |
| 1806 | try alias_index.rename(exp_name, &o.builder); | 1830 | |
| 1807 | | 1831 | if (comp.config.dll_export_fns and exp.opts.visibility != .hidden) |
| 1808 | const alias_global_index = alias_index.ptrConst(&o.builder).global; | 1832 | alias_global.setDllStorageClass(.dllexport, &o.builder); |
| 1809 | alias_global_index.setUnnamedAddr(.default, &o.builder); | 1833 | alias_global.setLinkage(switch (exp.opts.linkage) { |
| 1810 | if (comp.config.dll_export_fns and first_export.opts.visibility != .hidden) | 1834 | .internal => if (o.builder.strip) .private else .internal, // we still did useful work in replacing an existing symbol if there was one |
| 1811 | alias_global_index.setDllStorageClass(.dllexport, &o.builder); | | |
| 1812 | alias_global_index.setLinkage(switch (first_export.opts.linkage) { | | |
| 1813 | .internal => unreachable, | | |
| 1814 | .strong => .external, | 1835 | .strong => .external, |
| 1815 | .weak => .weak_odr, | 1836 | .weak => .weak_odr, |
| 1816 | .link_once => .linkonce_odr, | 1837 | .link_once => .linkonce_odr, |
| 1817 | }, &o.builder); | 1838 | }, &o.builder); |
| 1818 | alias_global_index.setVisibility(switch (first_export.opts.visibility) { | 1839 | alias_global.setVisibility(switch (exp.opts.visibility) { |
| 1819 | .default => .default, | 1840 | .default => .default, |
| 1820 | .hidden => .hidden, | 1841 | .hidden => .hidden, |
| 1821 | .protected => .protected, | 1842 | .protected => .protected, |
| ... | @@ -1826,7 +1847,10 @@ pub const Object = struct { | ... | @@ -1826,7 +1847,10 @@ pub const Object = struct { |
| 1826 | pub fn updateContainerType(o: *Object, pt: Zcu.PerThread, ty: InternPool.Index, success: bool) Allocator.Error!void { | 1847 | pub fn updateContainerType(o: *Object, pt: Zcu.PerThread, ty: InternPool.Index, success: bool) Allocator.Error!void { |
| 1827 | try o.type_pool.updateContainerType(pt, .{ .llvm = o }, ty, success); | 1848 | try o.type_pool.updateContainerType(pt, .{ .llvm = o }, ty, success); |
| 1828 | if (o.named_enum_map.get(ty)) |function_index| { | 1849 | if (o.named_enum_map.get(ty)) |function_index| { |
| 1829 | try o.updateIsNamedEnumValueFunction(pt, .fromInterned(ty), function_index); | 1850 | try o.updateIsNamedEnumValueFunction(.fromInterned(ty), function_index); |
| | 1851 | } |
| | 1852 | if (o.enum_tag_name_map.get(ty)) |function_index| { |
| | 1853 | try o.updateEnumTagNameFunction(.fromInterned(ty), function_index); |
| 1830 | } | 1854 | } |
| 1831 | } | 1855 | } |
| 1832 | | 1856 | |
| ... | @@ -1834,7 +1858,8 @@ pub const Object = struct { | ... | @@ -1834,7 +1858,8 @@ pub const Object = struct { |
| 1834 | /// | 1858 | /// |
| 1835 | /// `val` is always a type because `o.type_pool` only contains types. | 1859 | /// `val` is always a type because `o.type_pool` only contains types. |
| 1836 | pub fn addConst(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void { | 1860 | pub fn addConst(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void { |
| 1837 | const zcu = pt.zcu; | 1861 | _ = pt; |
| | 1862 | const zcu = o.zcu; |
| 1838 | const gpa = zcu.comp.gpa; | 1863 | const gpa = zcu.comp.gpa; |
| 1839 | assert(zcu.intern_pool.typeOf(val) == .type_type); | 1864 | assert(zcu.intern_pool.typeOf(val) == .type_type); |
| 1840 | | 1865 | |
| ... | @@ -1860,7 +1885,7 @@ pub const Object = struct { | ... | @@ -1860,7 +1885,7 @@ pub const Object = struct { |
| 1860 | /// | 1885 | /// |
| 1861 | /// `val` is always a type because `o.type_pool` only contains types. | 1886 | /// `val` is always a type because `o.type_pool` only contains types. |
| 1862 | pub fn updateConstIncomplete(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void { | 1887 | pub fn updateConstIncomplete(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void { |
| 1863 | const zcu = pt.zcu; | 1888 | const zcu = o.zcu; |
| 1864 | assert(zcu.intern_pool.typeOf(val) == .type_type); | 1889 | assert(zcu.intern_pool.typeOf(val) == .type_type); |
| 1865 | | 1890 | |
| 1866 | const ty: Type = .fromInterned(val); | 1891 | const ty: Type = .fromInterned(val); |
| ... | @@ -1874,7 +1899,12 @@ pub const Object = struct { | ... | @@ -1874,7 +1899,12 @@ pub const Object = struct { |
| 1874 | assert(val != .anyerror_type); | 1899 | assert(val != .anyerror_type); |
| 1875 | const fwd_ref = o.debug_types.items[@intFromEnum(index)]; | 1900 | const fwd_ref = o.debug_types.items[@intFromEnum(index)]; |
| 1876 | const name_str = try o.builder.metadataStringFmt("{f}", .{ty.fmt(pt)}); | 1901 | const name_str = try o.builder.metadataStringFmt("{f}", .{ty.fmt(pt)}); |
| 1877 | const debug_incomplete_type = try o.builder.debugSignedType(name_str, 0); | 1902 | // If `ty` is a function, use a dummy *function* type to prevent existing debug |
| | 1903 | // subprograms from becoming ill-formed. |
| | 1904 | const debug_incomplete_type = switch (ty.zigTypeTag(zcu)) { |
| | 1905 | .@"fn" => try o.builder.debugSubroutineType(null), |
| | 1906 | else => try o.builder.debugSignedType(name_str, 0), |
| | 1907 | }; |
| 1878 | o.builder.resolveDebugForwardReference(fwd_ref, debug_incomplete_type); | 1908 | o.builder.resolveDebugForwardReference(fwd_ref, debug_incomplete_type); |
| 1879 | } | 1909 | } |
| 1880 | } | 1910 | } |
| ... | @@ -1882,7 +1912,7 @@ pub const Object = struct { | ... | @@ -1882,7 +1912,7 @@ pub const Object = struct { |
| 1882 | /// | 1912 | /// |
| 1883 | /// `val` is always a type because `o.type_pool` only contains types. | 1913 | /// `val` is always a type because `o.type_pool` only contains types. |
| 1884 | pub fn updateConst(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void { | 1914 | pub fn updateConst(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void { |
| 1885 | const zcu = pt.zcu; | 1915 | const zcu = o.zcu; |
| 1886 | assert(zcu.intern_pool.typeOf(val) == .type_type); | 1916 | assert(zcu.intern_pool.typeOf(val) == .type_type); |
| 1887 | | 1917 | |
| 1888 | const ty: Type = .fromInterned(val); | 1918 | const ty: Type = .fromInterned(val); |
| ... | @@ -1904,13 +1934,13 @@ pub const Object = struct { | ... | @@ -1904,13 +1934,13 @@ pub const Object = struct { |
| 1904 | } | 1934 | } |
| 1905 | } | 1935 | } |
| 1906 | | 1936 | |
| 1907 | fn getDebugFile(o: *Object, pt: Zcu.PerThread, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata { | 1937 | pub fn getDebugFile(o: *Object, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata { |
| 1908 | const gpa = o.gpa; | 1938 | const gpa = o.gpa; |
| 1909 | const gop = try o.debug_file_map.getOrPut(gpa, file_index); | 1939 | const gop = try o.debug_file_map.getOrPut(gpa, file_index); |
| 1910 | errdefer assert(o.debug_file_map.remove(file_index)); | 1940 | errdefer assert(o.debug_file_map.remove(file_index)); |
| 1911 | if (gop.found_existing) return gop.value_ptr.*; | 1941 | if (gop.found_existing) return gop.value_ptr.*; |
| 1912 | const path = pt.zcu.fileByIndex(file_index).path; | 1942 | const path = o.zcu.fileByIndex(file_index).path; |
| 1913 | const abs_path = try path.toAbsolute(pt.zcu.comp.dirs, gpa); | 1943 | const abs_path = try path.toAbsolute(o.zcu.comp.dirs, gpa); |
| 1914 | defer gpa.free(abs_path); | 1944 | defer gpa.free(abs_path); |
| 1915 | | 1945 | |
| 1916 | gop.value_ptr.* = try o.builder.debugFile( | 1946 | gop.value_ptr.* = try o.builder.debugFile( |
| ... | @@ -1920,7 +1950,7 @@ pub const Object = struct { | ... | @@ -1920,7 +1950,7 @@ pub const Object = struct { |
| 1920 | return gop.value_ptr.*; | 1950 | return gop.value_ptr.*; |
| 1921 | } | 1951 | } |
| 1922 | | 1952 | |
| 1923 | fn getDebugType(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Metadata { | 1953 | pub fn getDebugType(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Metadata { |
| 1924 | assert(!o.builder.strip); | 1954 | assert(!o.builder.strip); |
| 1925 | const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern()); | 1955 | const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern()); |
| 1926 | return o.debug_types.items[@intFromEnum(index)]; | 1956 | return o.debug_types.items[@intFromEnum(index)]; |
| ... | @@ -1937,8 +1967,8 @@ pub const Object = struct { | ... | @@ -1937,8 +1967,8 @@ pub const Object = struct { |
| 1937 | assert(!o.builder.strip); | 1967 | assert(!o.builder.strip); |
| 1938 | | 1968 | |
| 1939 | const gpa = o.gpa; | 1969 | const gpa = o.gpa; |
| 1940 | const target = o.target; | 1970 | const zcu = o.zcu; |
| 1941 | const zcu = pt.zcu; | 1971 | const target = zcu.getTarget(); |
| 1942 | const ip = &zcu.intern_pool; | 1972 | const ip = &zcu.intern_pool; |
| 1943 | | 1973 | |
| 1944 | const name = try o.builder.metadataStringFmt("{f}", .{ty.fmt(pt)}); | 1974 | const name = try o.builder.metadataStringFmt("{f}", .{ty.fmt(pt)}); |
| ... | @@ -2203,7 +2233,9 @@ pub const Object = struct { | ... | @@ -2203,7 +2233,9 @@ pub const Object = struct { |
| 2203 | }, | 2233 | }, |
| 2204 | .@"fn" => { | 2234 | .@"fn" => { |
| 2205 | if (!ty.fnHasRuntimeBits(zcu)) { | 2235 | if (!ty.fnHasRuntimeBits(zcu)) { |
| 2206 | return o.builder.debugSignedType(name, 0); | 2236 | // Use a dummy *function* type to prevent existing debug subprograms from |
| | 2237 | // becoming ill-formed. |
| | 2238 | return o.builder.debugSubroutineType(null); |
| 2207 | } | 2239 | } |
| 2208 | | 2240 | |
| 2209 | const fn_info = zcu.typeToFunc(ty).?; | 2241 | const fn_info = zcu.typeToFunc(ty).?; |
| ... | @@ -2212,13 +2244,14 @@ pub const Object = struct { | ... | @@ -2212,13 +2244,14 @@ pub const Object = struct { |
| 2212 | defer debug_param_types.deinit(gpa); | 2244 | defer debug_param_types.deinit(gpa); |
| 2213 | | 2245 | |
| 2214 | // Return type goes first. | 2246 | // Return type goes first. |
| 2215 | const sret = firstParamSRet(fn_info, zcu, target); | 2247 | if (firstParamSRet(fn_info, zcu, target)) { |
| 2216 | const ret_ty: Type = if (sret) .void else .fromInterned(fn_info.return_type); | 2248 | // Actual return type is void, then first arg is the sret pointer. |
| 2217 | debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ret_ty)); | 2249 | const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type)); |
| 2218 | | 2250 | debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void)); |
| 2219 | if (sret) { | | |
| 2220 | const ptr_ty = try pt.singleMutPtrType(Type.fromInterned(fn_info.return_type)); | | |
| 2221 | debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ptr_ty)); | 2251 | debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ptr_ty)); |
| | 2252 | } else { |
| | 2253 | const ret_ty: Type = .fromInterned(fn_info.return_type); |
| | 2254 | debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ret_ty)); |
| 2222 | } | 2255 | } |
| 2223 | | 2256 | |
| 2224 | if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) { | 2257 | if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) { |
| ... | @@ -2287,7 +2320,7 @@ pub const Object = struct { | ... | @@ -2287,7 +2320,7 @@ pub const Object = struct { |
| 2287 | | 2320 | |
| 2288 | const struct_type = zcu.typeToStruct(ty).?; | 2321 | const struct_type = zcu.typeToStruct(ty).?; |
| 2289 | | 2322 | |
| 2290 | const file = try o.getDebugFile(pt, struct_type.zir_index.resolveFile(ip)); | 2323 | const file = try o.getDebugFile(struct_type.zir_index.resolveFile(ip)); |
| 2291 | const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| | 2324 | const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| |
| 2292 | try o.namespaceToDebugScope(pt, parent_namespace) | 2325 | try o.namespaceToDebugScope(pt, parent_namespace) |
| 2293 | else | 2326 | else |
| ... | @@ -2354,7 +2387,7 @@ pub const Object = struct { | ... | @@ -2354,7 +2387,7 @@ pub const Object = struct { |
| 2354 | .@"union" => { | 2387 | .@"union" => { |
| 2355 | const union_type = ip.loadUnionType(ty.toIntern()); | 2388 | const union_type = ip.loadUnionType(ty.toIntern()); |
| 2356 | | 2389 | |
| 2357 | const file = try o.getDebugFile(pt, union_type.zir_index.resolveFile(ip)); | 2390 | const file = try o.getDebugFile(union_type.zir_index.resolveFile(ip)); |
| 2358 | const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| | 2391 | const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| |
| 2359 | try o.namespaceToDebugScope(pt, parent_namespace) | 2392 | try o.namespaceToDebugScope(pt, parent_namespace) |
| 2360 | else | 2393 | else |
| ... | @@ -2512,7 +2545,7 @@ pub const Object = struct { | ... | @@ -2512,7 +2545,7 @@ pub const Object = struct { |
| 2512 | ); | 2545 | ); |
| 2513 | }, | 2546 | }, |
| 2514 | .@"enum" => { | 2547 | .@"enum" => { |
| 2515 | const file = try o.getDebugFile(pt, ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip)); | 2548 | const file = try o.getDebugFile(ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip)); |
| 2516 | const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| | 2549 | const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| |
| 2517 | try o.namespaceToDebugScope(pt, parent_namespace) | 2550 | try o.namespaceToDebugScope(pt, parent_namespace) |
| 2518 | else | 2551 | else |
| ... | @@ -2573,7 +2606,7 @@ pub const Object = struct { | ... | @@ -2573,7 +2606,7 @@ pub const Object = struct { |
| 2573 | return o.builder.debugSignedType(name, 0); | 2606 | return o.builder.debugSignedType(name, 0); |
| 2574 | } | 2607 | } |
| 2575 | | 2608 | |
| 2576 | const file = try o.getDebugFile(pt, ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip)); | 2609 | const file = try o.getDebugFile(ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip)); |
| 2577 | const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| | 2610 | const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| |
| 2578 | try o.namespaceToDebugScope(pt, parent_namespace) | 2611 | try o.namespaceToDebugScope(pt, parent_namespace) |
| 2579 | else | 2612 | else |
| ... | @@ -2598,8 +2631,8 @@ pub const Object = struct { | ... | @@ -2598,8 +2631,8 @@ pub const Object = struct { |
| 2598 | } | 2631 | } |
| 2599 | | 2632 | |
| 2600 | /// Called in `emit` so that the global error set is fully populated. | 2633 | /// Called in `emit` so that the global error set is fully populated. |
| 2601 | fn lowerDebugAnyerrorType(o: *Object, pt: Zcu.PerThread) Allocator.Error!Builder.Metadata { | 2634 | fn lowerDebugAnyerrorType(o: *Object) Allocator.Error!Builder.Metadata { |
| 2602 | const zcu = pt.zcu; | 2635 | const zcu = o.zcu; |
| 2603 | const ip = &zcu.intern_pool; | 2636 | const ip = &zcu.intern_pool; |
| 2604 | const gpa = zcu.comp.gpa; | 2637 | const gpa = zcu.comp.gpa; |
| 2605 | | 2638 | |
| ... | @@ -2633,7 +2666,7 @@ pub const Object = struct { | ... | @@ -2633,7 +2666,7 @@ pub const Object = struct { |
| 2633 | null, // file | 2666 | null, // file |
| 2634 | o.debug_compile_unit.unwrap().?, // scope | 2667 | o.debug_compile_unit.unwrap().?, // scope |
| 2635 | 0, // line | 2668 | 0, // line |
| 2636 | try o.getDebugType(pt, try pt.intType(.unsigned, error_set_bits)), | 2669 | try o.builder.debugUnsignedType(null, error_set_bits), |
| 2637 | Type.anyerror.abiSize(zcu) * 8, | 2670 | Type.anyerror.abiSize(zcu) * 8, |
| 2638 | Type.anyerror.abiAlignment(zcu).toByteUnits().? * 8, | 2671 | Type.anyerror.abiAlignment(zcu).toByteUnits().? * 8, |
| 2639 | try o.builder.metadataTuple(enumerators), | 2672 | try o.builder.metadataTuple(enumerators), |
| ... | @@ -2643,92 +2676,44 @@ pub const Object = struct { | ... | @@ -2643,92 +2676,44 @@ pub const Object = struct { |
| 2643 | } | 2676 | } |
| 2644 | | 2677 | |
| 2645 | fn namespaceToDebugScope(o: *Object, pt: Zcu.PerThread, namespace_index: InternPool.NamespaceIndex) !Builder.Metadata { | 2678 | fn namespaceToDebugScope(o: *Object, pt: Zcu.PerThread, namespace_index: InternPool.NamespaceIndex) !Builder.Metadata { |
| 2646 | const zcu = pt.zcu; | 2679 | const zcu = o.zcu; |
| 2647 | const namespace = zcu.namespacePtr(namespace_index); | 2680 | const namespace = zcu.namespacePtr(namespace_index); |
| 2648 | if (namespace.parent == .none) return try o.getDebugFile(pt, namespace.file_scope); | 2681 | if (namespace.parent == .none) return try o.getDebugFile(namespace.file_scope); |
| 2649 | return o.getDebugType(pt, .fromInterned(namespace.owner_type)); | 2682 | return o.getDebugType(pt, .fromInterned(namespace.owner_type)); |
| 2650 | } | 2683 | } |
| 2651 | | 2684 | |
| 2652 | fn allocTypeName(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error![:0]const u8 { | 2685 | /// Sets the attributes and callconv of the given `Builder.Function`, which corresponds to the |
| 2653 | var aw: Io.Writer.Allocating = .init(o.gpa); | 2686 | /// given `Nav` (which is a function). |
| 2654 | defer aw.deinit(); | 2687 | fn addLlvmFunctionAttributes( |
| 2655 | ty.print(&aw.writer, pt, null) catch |err| switch (err) { | | |
| 2656 | error.WriteFailed => return error.OutOfMemory, | | |
| 2657 | }; | | |
| 2658 | return aw.toOwnedSliceSentinel(0); | | |
| 2659 | } | | |
| 2660 | | | |
| 2661 | /// If the llvm function does not exist, create it. | | |
| 2662 | /// Note that this can be called before the function's semantic analysis has | | |
| 2663 | /// completed, so if any attributes rely on that, they must be done in updateFunc, not here. | | |
| 2664 | fn resolveLlvmFunction( | | |
| 2665 | o: *Object, | 2688 | o: *Object, |
| 2666 | pt: Zcu.PerThread, | 2689 | pt: Zcu.PerThread, |
| 2667 | nav_index: InternPool.Nav.Index, | 2690 | nav_id: InternPool.Nav.Index, |
| 2668 | ) Allocator.Error!Builder.Function.Index { | 2691 | function_index: Builder.Function.Index, |
| 2669 | const zcu = pt.zcu; | 2692 | ) Allocator.Error!void { |
| | 2693 | const zcu = o.zcu; |
| 2670 | const ip = &zcu.intern_pool; | 2694 | const ip = &zcu.intern_pool; |
| 2671 | const gpa = o.gpa; | 2695 | const nav = ip.getNav(nav_id); |
| 2672 | const nav = ip.getNav(nav_index); | 2696 | const owner_mod = zcu.navFileScope(nav_id).mod.?; |
| 2673 | const owner_mod = zcu.navFileScope(nav_index).mod.?; | | |
| 2674 | const ty: Type = .fromInterned(nav.resolved.?.type); | 2697 | const ty: Type = .fromInterned(nav.resolved.?.type); |
| 2675 | const gop = try o.nav_map.getOrPut(gpa, nav_index); | | |
| 2676 | if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.function; | | |
| 2677 | | 2698 | |
| 2678 | const fn_info = zcu.typeToFunc(ty).?; | 2699 | const fn_info = zcu.typeToFunc(ty).?; |
| 2679 | const target = &owner_mod.resolved_target.result; | 2700 | const target = &owner_mod.resolved_target.result; |
| 2680 | const sret = firstParamSRet(fn_info, zcu, target); | | |
| 2681 | | | |
| 2682 | const is_extern, const lib_name = if (nav.getExtern(ip)) |@"extern"| | | |
| 2683 | .{ true, @"extern".lib_name } | | |
| 2684 | else | | |
| 2685 | .{ false, .none }; | | |
| 2686 | const function_index = try o.builder.addFunction( | | |
| 2687 | try o.lowerType(pt, ty), | | |
| 2688 | try o.builder.strtabString((if (is_extern) nav.name else nav.fqn).toSlice(ip)), | | |
| 2689 | toLlvmAddressSpace(nav.resolved.?.@"addrspace", target), | | |
| 2690 | ); | | |
| 2691 | gop.value_ptr.* = function_index.ptrConst(&o.builder).global; | | |
| 2692 | | 2701 | |
| 2693 | var attributes: Builder.FunctionAttributes.Wip = .{}; | 2702 | var attributes: Builder.FunctionAttributes.Wip = .{}; |
| 2694 | defer attributes.deinit(&o.builder); | 2703 | defer attributes.deinit(&o.builder); |
| 2695 | | 2704 | |
| 2696 | if (!is_extern) { | 2705 | if (target.cpu.arch.isWasm()) if (nav.getExtern(ip)) |@"extern"| { |
| 2697 | function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); | 2706 | try attributes.addFnAttr(.{ .string = .{ |
| 2698 | function_index.setUnnamedAddr(.unnamed_addr, &o.builder); | 2707 | .kind = try o.builder.string("wasm-import-name"), |
| 2699 | } else { | 2708 | .value = try o.builder.string(nav.name.toSlice(ip)), |
| 2700 | if (target.cpu.arch.isWasm()) { | 2709 | } }, &o.builder); |
| 2701 | try attributes.addFnAttr(.{ .string = .{ | 2710 | if (@"extern".lib_name.toSlice(ip)) |lib_name_slice| { |
| 2702 | .kind = try o.builder.string("wasm-import-name"), | 2711 | if (!std.mem.eql(u8, lib_name_slice, "c")) try attributes.addFnAttr(.{ .string = .{ |
| 2703 | .value = try o.builder.string(nav.name.toSlice(ip)), | 2712 | .kind = try o.builder.string("wasm-import-module"), |
| | 2713 | .value = try o.builder.string(lib_name_slice), |
| 2704 | } }, &o.builder); | 2714 | } }, &o.builder); |
| 2705 | if (lib_name.toSlice(ip)) |lib_name_slice| { | | |
| 2706 | if (!std.mem.eql(u8, lib_name_slice, "c")) try attributes.addFnAttr(.{ .string = .{ | | |
| 2707 | .kind = try o.builder.string("wasm-import-module"), | | |
| 2708 | .value = try o.builder.string(lib_name_slice), | | |
| 2709 | } }, &o.builder); | | |
| 2710 | } | | |
| 2711 | } | 2715 | } |
| 2712 | } | 2716 | }; |
| 2713 | | | |
| 2714 | var llvm_arg_i: u32 = 0; | | |
| 2715 | if (sret) { | | |
| 2716 | // Sret pointers must not be address 0 | | |
| 2717 | try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); | | |
| 2718 | try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); | | |
| 2719 | | | |
| 2720 | const raw_llvm_ret_ty = try o.lowerType(pt, Type.fromInterned(fn_info.return_type)); | | |
| 2721 | try attributes.addParamAttr(llvm_arg_i, .{ .sret = raw_llvm_ret_ty }, &o.builder); | | |
| 2722 | | | |
| 2723 | llvm_arg_i += 1; | | |
| 2724 | } | | |
| 2725 | | | |
| 2726 | const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing; | | |
| 2727 | | | |
| 2728 | if (err_return_tracing) { | | |
| 2729 | try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); | | |
| 2730 | llvm_arg_i += 1; | | |
| 2731 | } | | |
| 2732 | | 2717 | |
| 2733 | if (fn_info.cc == .async) { | 2718 | if (fn_info.cc == .async) { |
| 2734 | @panic("TODO: LLVM backend lower async function"); | 2719 | @panic("TODO: LLVM backend lower async function"); |
| ... | @@ -2803,9 +2788,6 @@ pub const Object = struct { | ... | @@ -2803,9 +2788,6 @@ pub const Object = struct { |
| 2803 | } | 2788 | } |
| 2804 | } | 2789 | } |
| 2805 | | 2790 | |
| 2806 | if (nav.resolved.?.@"align" != .none) | | |
| 2807 | function_index.setAlignment(nav.resolved.?.@"align".toLlvm(), &o.builder); | | |
| 2808 | | | |
| 2809 | // Function attributes that are independent of analysis results of the function body. | 2791 | // Function attributes that are independent of analysis results of the function body. |
| 2810 | try o.addCommonFnAttributes( | 2792 | try o.addCommonFnAttributes( |
| 2811 | &attributes, | 2793 | &attributes, |
| ... | @@ -2821,18 +2803,81 @@ pub const Object = struct { | ... | @@ -2821,18 +2803,81 @@ pub const Object = struct { |
| 2821 | | 2803 | |
| 2822 | if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder); | 2804 | if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder); |
| 2823 | | 2805 | |
| 2824 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); | 2806 | var it = iterateParamTypes(o, fn_info); |
| 2825 | return function_index; | 2807 | if (firstParamSRet(fn_info, zcu, target)) { |
| 2826 | } | 2808 | // Sret pointers must not be address 0 |
| | 2809 | try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder); |
| | 2810 | try attributes.addParamAttr(it.llvm_index, .@"noalias", &o.builder); |
| 2827 | | 2811 | |
| 2828 | fn addCommonFnAttributes( | 2812 | const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type)); |
| 2829 | o: *Object, | 2813 | try attributes.addParamAttr(it.llvm_index, .{ .sret = raw_llvm_ret_ty }, &o.builder); |
| 2830 | attributes: *Builder.FunctionAttributes.Wip, | 2814 | it.llvm_index += 1; |
| 2831 | owner_mod: *Package.Module, | 2815 | } |
| 2832 | omit_frame_pointer: bool, | 2816 | const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing; |
| 2833 | ) Allocator.Error!void { | 2817 | if (err_return_tracing) { |
| 2834 | if (!owner_mod.red_zone) { | 2818 | try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder); |
| 2835 | try attributes.addFnAttr(.noredzone, &o.builder); | 2819 | it.llvm_index += 1; |
| | 2820 | } |
| | 2821 | while (try it.next()) |lowering| switch (lowering) { |
| | 2822 | .byval => { |
| | 2823 | const param_index = it.zig_index - 1; |
| | 2824 | const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[param_index]); |
| | 2825 | if (!isByRef(param_ty, zcu)) { |
| | 2826 | try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1); |
| | 2827 | } |
| | 2828 | }, |
| | 2829 | .byref => { |
| | 2830 | const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| | 2831 | const param_llvm_ty = try o.lowerType(param_ty); |
| | 2832 | const alignment = param_ty.abiAlignment(zcu); |
| | 2833 | try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty); |
| | 2834 | }, |
| | 2835 | .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), |
| | 2836 | .slice => { |
| | 2837 | const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| | 2838 | const ptr_info = param_ty.ptrInfo(zcu); |
| | 2839 | const llvm_ptr_index = it.llvm_index - 2; |
| | 2840 | if (std.math.cast(u5, it.zig_index - 1)) |i| { |
| | 2841 | if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { |
| | 2842 | try attributes.addParamAttr(llvm_ptr_index, .@"noalias", &o.builder); |
| | 2843 | } |
| | 2844 | } |
| | 2845 | if (param_ty.zigTypeTag(zcu) != .optional and |
| | 2846 | !ptr_info.flags.is_allowzero and |
| | 2847 | ptr_info.flags.address_space == .generic) |
| | 2848 | { |
| | 2849 | try attributes.addParamAttr(llvm_ptr_index, .nonnull, &o.builder); |
| | 2850 | } |
| | 2851 | if (ptr_info.flags.is_const) { |
| | 2852 | try attributes.addParamAttr(llvm_ptr_index, .readonly, &o.builder); |
| | 2853 | } |
| | 2854 | const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) { |
| | 2855 | else => |a| .wrap(a.toLlvm()), |
| | 2856 | .none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)), |
| | 2857 | }; |
| | 2858 | try attributes.addParamAttr(llvm_ptr_index, .{ .@"align" = elem_align }, &o.builder); |
| | 2859 | }, |
| | 2860 | // No attributes needed for these. |
| | 2861 | .no_bits, |
| | 2862 | .abi_sized_int, |
| | 2863 | .multiple_llvm_types, |
| | 2864 | .float_array, |
| | 2865 | .i32_array, |
| | 2866 | .i64_array, |
| | 2867 | => continue, |
| | 2868 | }; |
| | 2869 | |
| | 2870 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| | 2871 | } |
| | 2872 | |
| | 2873 | fn addCommonFnAttributes( |
| | 2874 | o: *Object, |
| | 2875 | attributes: *Builder.FunctionAttributes.Wip, |
| | 2876 | owner_mod: *Package.Module, |
| | 2877 | omit_frame_pointer: bool, |
| | 2878 | ) Allocator.Error!void { |
| | 2879 | if (!owner_mod.red_zone) { |
| | 2880 | try attributes.addFnAttr(.noredzone, &o.builder); |
| 2836 | } | 2881 | } |
| 2837 | if (omit_frame_pointer) { | 2882 | if (omit_frame_pointer) { |
| 2838 | try attributes.addFnAttr(.{ .string = .{ | 2883 | try attributes.addFnAttr(.{ .string = .{ |
| ... | @@ -2895,109 +2940,16 @@ pub const Object = struct { | ... | @@ -2895,109 +2940,16 @@ pub const Object = struct { |
| 2895 | } | 2940 | } |
| 2896 | } | 2941 | } |
| 2897 | | 2942 | |
| 2898 | fn resolveGlobalUav( | 2943 | pub fn errorIntType(o: *Object) Allocator.Error!Builder.Type { |
| 2899 | o: *Object, | 2944 | return o.builder.intType(o.zcu.errorSetBits()); |
| 2900 | pt: Zcu.PerThread, | | |
| 2901 | uav: InternPool.Index, | | |
| 2902 | llvm_addr_space: Builder.AddrSpace, | | |
| 2903 | alignment: InternPool.Alignment, | | |
| 2904 | ) Allocator.Error!Builder.Variable.Index { | | |
| 2905 | assert(alignment != .none); | | |
| 2906 | // TODO: Add address space to the anon_decl_map | | |
| 2907 | const gop = try o.uav_map.getOrPut(o.gpa, uav); | | |
| 2908 | if (gop.found_existing) { | | |
| 2909 | // Keep the greater of the two alignments. | | |
| 2910 | const variable_index = gop.value_ptr.ptr(&o.builder).kind.variable; | | |
| 2911 | const old_alignment = InternPool.Alignment.fromLlvm(variable_index.getAlignment(&o.builder)); | | |
| 2912 | const max_alignment = old_alignment.maxStrict(alignment); | | |
| 2913 | variable_index.setAlignment(max_alignment.toLlvm(), &o.builder); | | |
| 2914 | return variable_index; | | |
| 2915 | } | | |
| 2916 | errdefer assert(o.uav_map.remove(uav)); | | |
| 2917 | | | |
| 2918 | const zcu = pt.zcu; | | |
| 2919 | const decl_ty = zcu.intern_pool.typeOf(uav); | | |
| 2920 | | | |
| 2921 | const variable_index = try o.builder.addVariable( | | |
| 2922 | try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav)}), | | |
| 2923 | try o.lowerType(pt, Type.fromInterned(decl_ty)), | | |
| 2924 | llvm_addr_space, | | |
| 2925 | ); | | |
| 2926 | gop.value_ptr.* = variable_index.ptrConst(&o.builder).global; | | |
| 2927 | | | |
| 2928 | try variable_index.setInitializer(try o.lowerValue(pt, uav), &o.builder); | | |
| 2929 | variable_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); | | |
| 2930 | variable_index.setMutability(.constant, &o.builder); | | |
| 2931 | variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); | | |
| 2932 | variable_index.setAlignment(alignment.toLlvm(), &o.builder); | | |
| 2933 | return variable_index; | | |
| 2934 | } | | |
| 2935 | | | |
| 2936 | fn resolveGlobalNav( | | |
| 2937 | o: *Object, | | |
| 2938 | pt: Zcu.PerThread, | | |
| 2939 | nav_index: InternPool.Nav.Index, | | |
| 2940 | ) Allocator.Error!Builder.Variable.Index { | | |
| 2941 | const gop = try o.nav_map.getOrPut(o.gpa, nav_index); | | |
| 2942 | if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.variable; | | |
| 2943 | errdefer assert(o.nav_map.remove(nav_index)); | | |
| 2944 | | | |
| 2945 | const zcu = pt.zcu; | | |
| 2946 | const ip = &zcu.intern_pool; | | |
| 2947 | const nav = ip.getNav(nav_index); | | |
| 2948 | const linkage: std.builtin.GlobalLinkage, const visibility: Builder.Visibility, const is_dll_import: bool = switch (nav.resolved.?.value) { | | |
| 2949 | .none => .{ .internal, .default, false }, // this is a source declaration which is *not* marked `extern` | | |
| 2950 | else => |val| switch (ip.indexToKey(val)) { | | |
| 2951 | else => .{ .internal, .default, false }, | | |
| 2952 | .@"extern" => |e| .{ e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import }, | | |
| 2953 | }, | | |
| 2954 | }; | | |
| 2955 | | | |
| 2956 | const variable_index = try o.builder.addVariable( | | |
| 2957 | try o.builder.strtabString(switch (linkage) { | | |
| 2958 | .internal => nav.fqn, | | |
| 2959 | .strong, .weak => nav.name, | | |
| 2960 | .link_once => unreachable, | | |
| 2961 | }.toSlice(ip)), | | |
| 2962 | try o.lowerType(pt, .fromInterned(nav.resolved.?.type)), | | |
| 2963 | toLlvmGlobalAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget()), | | |
| 2964 | ); | | |
| 2965 | gop.value_ptr.* = variable_index.ptrConst(&o.builder).global; | | |
| 2966 | | | |
| 2967 | // This is needed for declarations created by `@extern`. | | |
| 2968 | switch (linkage) { | | |
| 2969 | .internal => { | | |
| 2970 | variable_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); | | |
| 2971 | variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); | | |
| 2972 | }, | | |
| 2973 | .strong, .weak => { | | |
| 2974 | variable_index.setLinkage(switch (linkage) { | | |
| 2975 | .internal => unreachable, | | |
| 2976 | .strong => .external, | | |
| 2977 | .weak => .extern_weak, | | |
| 2978 | .link_once => unreachable, | | |
| 2979 | }, &o.builder); | | |
| 2980 | variable_index.setUnnamedAddr(.default, &o.builder); | | |
| 2981 | if (nav.resolved.?.@"threadlocal" and !zcu.navFileScope(nav_index).mod.?.single_threaded) | | |
| 2982 | variable_index.setThreadLocal(.generaldynamic, &o.builder); | | |
| 2983 | if (is_dll_import) variable_index.setDllStorageClass(.dllimport, &o.builder); | | |
| 2984 | }, | | |
| 2985 | .link_once => unreachable, | | |
| 2986 | } | | |
| 2987 | variable_index.setVisibility(visibility, &o.builder); | | |
| 2988 | return variable_index; | | |
| 2989 | } | | |
| 2990 | | | |
| 2991 | fn errorIntType(o: *Object, pt: Zcu.PerThread) Allocator.Error!Builder.Type { | | |
| 2992 | return o.builder.intType(pt.zcu.errorSetBits()); | | |
| 2993 | } | 2945 | } |
| 2994 | | 2946 | |
| 2995 | fn lowerType(o: *Object, pt: Zcu.PerThread, t: Type) Allocator.Error!Builder.Type { | 2947 | pub fn lowerType(o: *Object, t: Type) Allocator.Error!Builder.Type { |
| 2996 | const zcu = pt.zcu; | 2948 | const zcu = o.zcu; |
| 2997 | const target = zcu.getTarget(); | 2949 | const target = zcu.getTarget(); |
| 2998 | const ip = &zcu.intern_pool; | 2950 | const ip = &zcu.intern_pool; |
| 2999 | return switch (t.toIntern()) { | 2951 | return switch (t.toIntern()) { |
| 3000 | .u0_type, .i0_type => unreachable, | 2952 | .u0_type, .i0_type => unreachable, // no runtime bits |
| 3001 | inline .u1_type, | 2953 | inline .u1_type, |
| 3002 | .u8_type, | 2954 | .u8_type, |
| 3003 | .i8_type, | 2955 | .i8_type, |
| ... | @@ -3046,18 +2998,18 @@ pub const Object = struct { | ... | @@ -3046,18 +2998,18 @@ pub const Object = struct { |
| 3046 | return .i8; | 2998 | return .i8; |
| 3047 | }, | 2999 | }, |
| 3048 | .bool_type => .i1, | 3000 | .bool_type => .i1, |
| 3049 | .void_type => .void, | 3001 | .anyerror_type => try o.errorIntType(), |
| 3050 | .type_type => unreachable, | 3002 | .void_type => unreachable, // no runtime bits |
| 3051 | .anyerror_type => try o.errorIntType(pt), | 3003 | .type_type => unreachable, // no runtime bits |
| 3052 | .comptime_int_type, | 3004 | .comptime_int_type => unreachable, // no runtime bits |
| 3053 | .comptime_float_type, | 3005 | .comptime_float_type => unreachable, // no runtime bits |
| 3054 | .noreturn_type, | 3006 | .noreturn_type => unreachable, // no runtime bits |
| 3055 | => unreachable, | 3007 | .null_type => unreachable, // no runtime bits |
| | 3008 | .undefined_type => unreachable, // no runtime bits |
| | 3009 | .enum_literal_type => unreachable, // no runtime bits |
| | 3010 | .optional_noreturn_type => unreachable, // no runtime bits |
| | 3011 | .empty_tuple_type => unreachable, // no runtime bits |
| 3056 | .anyframe_type => @panic("TODO implement lowerType for AnyFrame types"), | 3012 | .anyframe_type => @panic("TODO implement lowerType for AnyFrame types"), |
| 3057 | .null_type, | | |
| 3058 | .undefined_type, | | |
| 3059 | .enum_literal_type, | | |
| 3060 | => unreachable, | | |
| 3061 | .ptr_usize_type, | 3013 | .ptr_usize_type, |
| 3062 | .ptr_const_comptime_int_type, | 3014 | .ptr_const_comptime_int_type, |
| 3063 | .manyptr_u8_type, | 3015 | .manyptr_u8_type, |
| ... | @@ -3066,14 +3018,11 @@ pub const Object = struct { | ... | @@ -3066,14 +3018,11 @@ pub const Object = struct { |
| 3066 | => .ptr, | 3018 | => .ptr, |
| 3067 | .slice_const_u8_type, | 3019 | .slice_const_u8_type, |
| 3068 | .slice_const_u8_sentinel_0_type, | 3020 | .slice_const_u8_sentinel_0_type, |
| 3069 | => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(pt, Type.usize) }), | 3021 | => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize) }), |
| 3070 | .optional_noreturn_type => unreachable, | | |
| 3071 | .anyerror_void_error_union_type, | 3022 | .anyerror_void_error_union_type, |
| 3072 | .adhoc_inferred_error_set_type, | 3023 | .adhoc_inferred_error_set_type, |
| 3073 | => try o.errorIntType(pt), | 3024 | => try o.errorIntType(), |
| 3074 | .generic_poison_type, | 3025 | .generic_poison_type => unreachable, |
| 3075 | .empty_tuple_type, | | |
| 3076 | => unreachable, | | |
| 3077 | // values, not types | 3026 | // values, not types |
| 3078 | .undef, | 3027 | .undef, |
| 3079 | .undef_bool, | 3028 | .undef_bool, |
| ... | @@ -3107,24 +3056,28 @@ pub const Object = struct { | ... | @@ -3107,24 +3056,28 @@ pub const Object = struct { |
| 3107 | .one, .many, .c => ptr_ty, | 3056 | .one, .many, .c => ptr_ty, |
| 3108 | .slice => try o.builder.structType(.normal, &.{ | 3057 | .slice => try o.builder.structType(.normal, &.{ |
| 3109 | ptr_ty, | 3058 | ptr_ty, |
| 3110 | try o.lowerType(pt, Type.usize), | 3059 | try o.lowerType(.usize), |
| 3111 | }), | 3060 | }), |
| 3112 | }; | 3061 | }; |
| 3113 | }, | 3062 | }, |
| 3114 | .array_type => |array_type| o.builder.arrayType( | 3063 | .array_type => |array_type| o.builder.arrayType( |
| 3115 | array_type.lenIncludingSentinel(), | 3064 | array_type.lenIncludingSentinel(), |
| 3116 | try o.lowerType(pt, Type.fromInterned(array_type.child)), | 3065 | try o.lowerType(.fromInterned(array_type.child)), |
| 3117 | ), | 3066 | ), |
| 3118 | .vector_type => |vector_type| o.builder.vectorType( | 3067 | .vector_type => |vector_type| o.builder.vectorType( |
| 3119 | .normal, | 3068 | .normal, |
| 3120 | vector_type.len, | 3069 | vector_type.len, |
| 3121 | try o.lowerType(pt, Type.fromInterned(vector_type.child)), | 3070 | try o.lowerType(.fromInterned(vector_type.child)), |
| 3122 | ), | 3071 | ), |
| 3123 | .opt_type => |child_ty| { | 3072 | .opt_type => |child_ty| { |
| 3124 | // Must stay in sync with `opt_payload` logic in `lowerPtr`. | 3073 | // Must stay in sync with `opt_payload` logic in `lowerPtr`. |
| 3125 | if (!Type.fromInterned(child_ty).hasRuntimeBits(zcu)) return .i8; | 3074 | switch (Type.fromInterned(child_ty).classify(zcu)) { |
| | 3075 | .no_possible_value, .fully_comptime => unreachable, |
| | 3076 | .one_possible_value => return .i8, |
| | 3077 | .runtime, .partially_comptime => {}, |
| | 3078 | } |
| 3126 | | 3079 | |
| 3127 | const payload_ty = try o.lowerType(pt, Type.fromInterned(child_ty)); | 3080 | const payload_ty = try o.lowerType(.fromInterned(child_ty)); |
| 3128 | if (t.optionalReprIsPayload(zcu)) return payload_ty; | 3081 | if (t.optionalReprIsPayload(zcu)) return payload_ty; |
| 3129 | | 3082 | |
| 3130 | comptime assert(optional_layout_version == 3); | 3083 | comptime assert(optional_layout_version == 3); |
| ... | @@ -3143,10 +3096,15 @@ pub const Object = struct { | ... | @@ -3143,10 +3096,15 @@ pub const Object = struct { |
| 3143 | .error_union_type => |error_union_type| { | 3096 | .error_union_type => |error_union_type| { |
| 3144 | // Must stay in sync with `codegen.errUnionPayloadOffset`. | 3097 | // Must stay in sync with `codegen.errUnionPayloadOffset`. |
| 3145 | // See logic in `lowerPtr`. | 3098 | // See logic in `lowerPtr`. |
| 3146 | const error_type = try o.errorIntType(pt); | 3099 | const error_type = try o.errorIntType(); |
| 3147 | if (!Type.fromInterned(error_union_type.payload_type).hasRuntimeBits(zcu)) | 3100 | |
| 3148 | return error_type; | 3101 | switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) { |
| 3149 | const payload_type = try o.lowerType(pt, Type.fromInterned(error_union_type.payload_type)); | 3102 | .fully_comptime => unreachable, |
| | 3103 | .no_possible_value, .one_possible_value => return error_type, |
| | 3104 | .runtime, .partially_comptime => {}, |
| | 3105 | } |
| | 3106 | |
| | 3107 | const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type)); |
| 3150 | | 3108 | |
| 3151 | const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu); | 3109 | const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu); |
| 3152 | const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits())); | 3110 | const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits())); |
| ... | @@ -3186,17 +3144,18 @@ pub const Object = struct { | ... | @@ -3186,17 +3144,18 @@ pub const Object = struct { |
| 3186 | const struct_type = ip.loadStructType(t.toIntern()); | 3144 | const struct_type = ip.loadStructType(t.toIntern()); |
| 3187 | | 3145 | |
| 3188 | if (struct_type.layout == .@"packed") { | 3146 | if (struct_type.layout == .@"packed") { |
| 3189 | const int_ty = try o.lowerType(pt, .fromInterned(struct_type.packed_backing_int_type)); | 3147 | const int_ty = try o.lowerType(.fromInterned(struct_type.packed_backing_int_type)); |
| 3190 | try o.type_map.put(o.gpa, t.toIntern(), int_ty); | 3148 | try o.type_map.put(o.gpa, t.toIntern(), int_ty); |
| 3191 | return int_ty; | 3149 | return int_ty; |
| 3192 | } | 3150 | } |
| 3193 | | 3151 | |
| | 3152 | assert(struct_type.size > 0); |
| | 3153 | |
| 3194 | var llvm_field_types: std.ArrayList(Builder.Type) = .empty; | 3154 | var llvm_field_types: std.ArrayList(Builder.Type) = .empty; |
| 3195 | defer llvm_field_types.deinit(o.gpa); | 3155 | defer llvm_field_types.deinit(o.gpa); |
| 3196 | // Although we can estimate how much capacity to add, these cannot be | 3156 | // Although we can estimate how much capacity to add, these cannot be |
| 3197 | // relied upon because of the recursive calls to lowerType below. | 3157 | // relied upon because of the recursive calls to lowerType below. |
| 3198 | try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_type.field_types.len); | 3158 | try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_type.field_types.len); |
| 3199 | try o.struct_field_map.ensureUnusedCapacity(o.gpa, struct_type.field_types.len); | | |
| 3200 | | 3159 | |
| 3201 | comptime assert(struct_layout_version == 2); | 3160 | comptime assert(struct_layout_version == 2); |
| 3202 | var offset: u64 = 0; | 3161 | var offset: u64 = 0; |
| ... | @@ -3221,24 +3180,9 @@ pub const Object = struct { | ... | @@ -3221,24 +3180,9 @@ pub const Object = struct { |
| 3221 | try o.builder.arrayType(padding_len, .i8), | 3180 | try o.builder.arrayType(padding_len, .i8), |
| 3222 | ); | 3181 | ); |
| 3223 | | 3182 | |
| 3224 | if (!field_ty.hasRuntimeBits(zcu)) { | 3183 | if (!field_ty.hasRuntimeBits(zcu)) continue; |
| 3225 | // This is a zero-bit field. If there are runtime bits after this field, | | |
| 3226 | // map to the next LLVM field (which we know exists): otherwise, don't | | |
| 3227 | // map the field, indicating it's at the end of the struct. | | |
| 3228 | if (offset != struct_type.size) { | | |
| 3229 | try o.struct_field_map.put(o.gpa, .{ | | |
| 3230 | .struct_ty = t.toIntern(), | | |
| 3231 | .field_index = field_index, | | |
| 3232 | }, @intCast(llvm_field_types.items.len)); | | |
| 3233 | } | | |
| 3234 | continue; | | |
| 3235 | } | | |
| 3236 | | 3184 | |
| 3237 | try o.struct_field_map.put(o.gpa, .{ | 3185 | try llvm_field_types.append(o.gpa, try o.lowerType(field_ty)); |
| 3238 | .struct_ty = t.toIntern(), | | |
| 3239 | .field_index = field_index, | | |
| 3240 | }, @intCast(llvm_field_types.items.len)); | | |
| 3241 | try llvm_field_types.append(o.gpa, try o.lowerType(pt, field_ty)); | | |
| 3242 | | 3186 | |
| 3243 | offset += field_ty.abiSize(zcu); | 3187 | offset += field_ty.abiSize(zcu); |
| 3244 | } | 3188 | } |
| ... | @@ -3270,19 +3214,15 @@ pub const Object = struct { | ... | @@ -3270,19 +3214,15 @@ pub const Object = struct { |
| 3270 | // Although we can estimate how much capacity to add, these cannot be | 3214 | // Although we can estimate how much capacity to add, these cannot be |
| 3271 | // relied upon because of the recursive calls to lowerType below. | 3215 | // relied upon because of the recursive calls to lowerType below. |
| 3272 | try llvm_field_types.ensureUnusedCapacity(o.gpa, tuple_type.types.len); | 3216 | try llvm_field_types.ensureUnusedCapacity(o.gpa, tuple_type.types.len); |
| 3273 | try o.struct_field_map.ensureUnusedCapacity(o.gpa, tuple_type.types.len); | | |
| 3274 | | 3217 | |
| 3275 | comptime assert(struct_layout_version == 2); | 3218 | comptime assert(struct_layout_version == 2); |
| 3276 | var offset: u64 = 0; | 3219 | var offset: u64 = 0; |
| 3277 | var big_align: InternPool.Alignment = .none; | 3220 | var big_align: InternPool.Alignment = .@"1"; |
| 3278 | | | |
| 3279 | const struct_size = t.abiSize(zcu); | | |
| 3280 | | 3221 | |
| 3281 | for ( | 3222 | for ( |
| 3282 | tuple_type.types.get(ip), | 3223 | tuple_type.types.get(ip), |
| 3283 | tuple_type.values.get(ip), | 3224 | tuple_type.values.get(ip), |
| 3284 | 0.., | 3225 | ) |field_ty, field_val| { |
| 3285 | ) |field_ty, field_val, field_index| { | | |
| 3286 | if (field_val != .none) continue; | 3226 | if (field_val != .none) continue; |
| 3287 | | 3227 | |
| 3288 | const field_align = Type.fromInterned(field_ty).abiAlignment(zcu); | 3228 | const field_align = Type.fromInterned(field_ty).abiAlignment(zcu); |
| ... | @@ -3296,22 +3236,9 @@ pub const Object = struct { | ... | @@ -3296,22 +3236,9 @@ pub const Object = struct { |
| 3296 | try o.builder.arrayType(padding_len, .i8), | 3236 | try o.builder.arrayType(padding_len, .i8), |
| 3297 | ); | 3237 | ); |
| 3298 | if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) { | 3238 | if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) { |
| 3299 | // This is a zero-bit field. If there are runtime bits after this field, | | |
| 3300 | // map to the next LLVM field (which we know exists): otherwise, don't | | |
| 3301 | // map the field, indicating it's at the end of the struct. | | |
| 3302 | if (offset != struct_size) { | | |
| 3303 | try o.struct_field_map.put(o.gpa, .{ | | |
| 3304 | .struct_ty = t.toIntern(), | | |
| 3305 | .field_index = @intCast(field_index), | | |
| 3306 | }, @intCast(llvm_field_types.items.len)); | | |
| 3307 | } | | |
| 3308 | continue; | 3239 | continue; |
| 3309 | } | 3240 | } |
| 3310 | try o.struct_field_map.put(o.gpa, .{ | 3241 | try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty))); |
| 3311 | .struct_ty = t.toIntern(), | | |
| 3312 | .field_index = @intCast(field_index), | | |
| 3313 | }, @intCast(llvm_field_types.items.len)); | | |
| 3314 | try llvm_field_types.append(o.gpa, try o.lowerType(pt, Type.fromInterned(field_ty))); | | |
| 3315 | | 3242 | |
| 3316 | offset += Type.fromInterned(field_ty).abiSize(zcu); | 3243 | offset += Type.fromInterned(field_ty).abiSize(zcu); |
| 3317 | } | 3244 | } |
| ... | @@ -3324,6 +3251,7 @@ pub const Object = struct { | ... | @@ -3324,6 +3251,7 @@ pub const Object = struct { |
| 3324 | try o.builder.arrayType(padding_len, .i8), | 3251 | try o.builder.arrayType(padding_len, .i8), |
| 3325 | ); | 3252 | ); |
| 3326 | } | 3253 | } |
| | 3254 | assert(offset > 0); |
| 3327 | return o.builder.structType(.normal, llvm_field_types.items); | 3255 | return o.builder.structType(.normal, llvm_field_types.items); |
| 3328 | }, | 3256 | }, |
| 3329 | .union_type => { | 3257 | .union_type => { |
| ... | @@ -3332,21 +3260,23 @@ pub const Object = struct { | ... | @@ -3332,21 +3260,23 @@ pub const Object = struct { |
| 3332 | const union_obj = ip.loadUnionType(t.toIntern()); | 3260 | const union_obj = ip.loadUnionType(t.toIntern()); |
| 3333 | | 3261 | |
| 3334 | if (union_obj.layout == .@"packed") { | 3262 | if (union_obj.layout == .@"packed") { |
| 3335 | const int_ty = try o.lowerType(pt, .fromInterned(union_obj.packed_backing_int_type)); | 3263 | const int_ty = try o.lowerType(.fromInterned(union_obj.packed_backing_int_type)); |
| 3336 | try o.type_map.put(o.gpa, t.toIntern(), int_ty); | 3264 | try o.type_map.put(o.gpa, t.toIntern(), int_ty); |
| 3337 | return int_ty; | 3265 | return int_ty; |
| 3338 | } | 3266 | } |
| 3339 | | 3267 | |
| | 3268 | assert(union_obj.size > 0); |
| | 3269 | |
| 3340 | const layout = Type.getUnionLayout(union_obj, zcu); | 3270 | const layout = Type.getUnionLayout(union_obj, zcu); |
| 3341 | | 3271 | |
| 3342 | if (layout.payload_size == 0) { | 3272 | if (layout.payload_size == 0) { |
| 3343 | const enum_tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); | 3273 | const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type)); |
| 3344 | try o.type_map.put(o.gpa, t.toIntern(), enum_tag_ty); | 3274 | try o.type_map.put(o.gpa, t.toIntern(), enum_tag_ty); |
| 3345 | return enum_tag_ty; | 3275 | return enum_tag_ty; |
| 3346 | } | 3276 | } |
| 3347 | | 3277 | |
| 3348 | const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]); | 3278 | const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]); |
| 3349 | const aligned_field_llvm_ty = try o.lowerType(pt, aligned_field_ty); | 3279 | const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty); |
| 3350 | | 3280 | |
| 3351 | const payload_ty = ty: { | 3281 | const payload_ty = ty: { |
| 3352 | if (layout.most_aligned_field_size == layout.payload_size) { | 3282 | if (layout.most_aligned_field_size == layout.payload_size) { |
| ... | @@ -3372,7 +3302,7 @@ pub const Object = struct { | ... | @@ -3372,7 +3302,7 @@ pub const Object = struct { |
| 3372 | ); | 3302 | ); |
| 3373 | return ty; | 3303 | return ty; |
| 3374 | } | 3304 | } |
| 3375 | const enum_tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); | 3305 | const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type)); |
| 3376 | | 3306 | |
| 3377 | // Put the tag before or after the payload depending on which one's | 3307 | // Put the tag before or after the payload depending on which one's |
| 3378 | // alignment is greater. | 3308 | // alignment is greater. |
| ... | @@ -3400,16 +3330,10 @@ pub const Object = struct { | ... | @@ -3400,16 +3330,10 @@ pub const Object = struct { |
| 3400 | ); | 3330 | ); |
| 3401 | return ty; | 3331 | return ty; |
| 3402 | }, | 3332 | }, |
| 3403 | .opaque_type => { | 3333 | .opaque_type => unreachable, // no runtime bits |
| 3404 | const gop = try o.type_map.getOrPut(o.gpa, t.toIntern()); | 3334 | .enum_type => try o.lowerType(t.intTagType(zcu)), |
| 3405 | if (!gop.found_existing) { | 3335 | .func_type => |func_type| try o.lowerFnType(t, func_type), |
| 3406 | gop.value_ptr.* = try o.builder.opaqueType(try o.builder.string(t.containerTypeName(ip).toSlice(ip))); | 3336 | .error_set_type, .inferred_error_set_type => try o.errorIntType(), |
| 3407 | } | | |
| 3408 | return gop.value_ptr.*; | | |
| 3409 | }, | | |
| 3410 | .enum_type => try o.lowerType(pt, t.intTagType(zcu)), | | |
| 3411 | .func_type => |func_type| try o.lowerTypeFn(pt, func_type), | | |
| 3412 | .error_set_type, .inferred_error_set_type => try o.errorIntType(pt), | | |
| 3413 | // values, not types | 3337 | // values, not types |
| 3414 | .undef, | 3338 | .undef, |
| 3415 | .simple_value, | 3339 | .simple_value, |
| ... | @@ -3434,11 +3358,14 @@ pub const Object = struct { | ... | @@ -3434,11 +3358,14 @@ pub const Object = struct { |
| 3434 | }; | 3358 | }; |
| 3435 | } | 3359 | } |
| 3436 | | 3360 | |
| 3437 | fn lowerTypeFn(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { | 3361 | fn lowerFnType(o: *Object, fn_ty: Type, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { |
| 3438 | const zcu = pt.zcu; | 3362 | const zcu = o.zcu; |
| 3439 | const ip = &zcu.intern_pool; | 3363 | const ip = &zcu.intern_pool; |
| 3440 | const target = zcu.getTarget(); | 3364 | const target = zcu.getTarget(); |
| 3441 | const ret_ty = try lowerFnRetTy(o, pt, fn_info); | 3365 | |
| | 3366 | assert(fn_ty.fnHasRuntimeBits(zcu)); |
| | 3367 | |
| | 3368 | const ret_ty = try lowerFnRetTy(o, fn_info); |
| 3442 | | 3369 | |
| 3443 | var llvm_params: std.ArrayList(Builder.Type) = .empty; | 3370 | var llvm_params: std.ArrayList(Builder.Type) = .empty; |
| 3444 | defer llvm_params.deinit(o.gpa); | 3371 | defer llvm_params.deinit(o.gpa); |
| ... | @@ -3453,12 +3380,12 @@ pub const Object = struct { | ... | @@ -3453,12 +3380,12 @@ pub const Object = struct { |
| 3453 | try llvm_params.append(o.gpa, llvm_ptr_ty); | 3380 | try llvm_params.append(o.gpa, llvm_ptr_ty); |
| 3454 | } | 3381 | } |
| 3455 | | 3382 | |
| 3456 | var it = iterateParamTypes(o, pt, fn_info); | 3383 | var it = iterateParamTypes(o, fn_info); |
| 3457 | while (try it.next()) |lowering| switch (lowering) { | 3384 | while (try it.next()) |lowering| switch (lowering) { |
| 3458 | .no_bits => continue, | 3385 | .no_bits => continue, |
| 3459 | .byval => { | 3386 | .byval => { |
| 3460 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | 3387 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| 3461 | try llvm_params.append(o.gpa, try o.lowerType(pt, param_ty)); | 3388 | try llvm_params.append(o.gpa, try o.lowerType(param_ty)); |
| 3462 | }, | 3389 | }, |
| 3463 | .byref, .byref_mut => { | 3390 | .byref, .byref_mut => { |
| 3464 | try llvm_params.append(o.gpa, .ptr); | 3391 | try llvm_params.append(o.gpa, .ptr); |
| ... | @@ -3473,7 +3400,7 @@ pub const Object = struct { | ... | @@ -3473,7 +3400,7 @@ pub const Object = struct { |
| 3473 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | 3400 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| 3474 | try llvm_params.appendSlice(o.gpa, &.{ | 3401 | try llvm_params.appendSlice(o.gpa, &.{ |
| 3475 | try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)), | 3402 | try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)), |
| 3476 | try o.lowerType(pt, Type.usize), | 3403 | try o.lowerType(.usize), |
| 3477 | }); | 3404 | }); |
| 3478 | }, | 3405 | }, |
| 3479 | .multiple_llvm_types => { | 3406 | .multiple_llvm_types => { |
| ... | @@ -3481,7 +3408,7 @@ pub const Object = struct { | ... | @@ -3481,7 +3408,7 @@ pub const Object = struct { |
| 3481 | }, | 3408 | }, |
| 3482 | .float_array => |count| { | 3409 | .float_array => |count| { |
| 3483 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | 3410 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); |
| 3484 | const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(param_ty, zcu).?); | 3411 | const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?); |
| 3485 | try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty)); | 3412 | try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty)); |
| 3486 | }, | 3413 | }, |
| 3487 | .i32_array, .i64_array => |arr_len| { | 3414 | .i32_array, .i64_array => |arr_len| { |
| ... | @@ -3500,20 +3427,17 @@ pub const Object = struct { | ... | @@ -3500,20 +3427,17 @@ pub const Object = struct { |
| 3500 | ); | 3427 | ); |
| 3501 | } | 3428 | } |
| 3502 | | 3429 | |
| 3503 | fn lowerValue(o: *Object, pt: Zcu.PerThread, arg_val: InternPool.Index) Allocator.Error!Builder.Constant { | 3430 | pub fn lowerValue(o: *Object, arg_val: InternPool.Index) Allocator.Error!Builder.Constant { |
| 3504 | const zcu = pt.zcu; | 3431 | const zcu = o.zcu; |
| 3505 | const ip = &zcu.intern_pool; | 3432 | const ip = &zcu.intern_pool; |
| 3506 | const target = zcu.getTarget(); | 3433 | const target = zcu.getTarget(); |
| 3507 | | 3434 | |
| 3508 | const val = Value.fromInterned(arg_val); | 3435 | const val: Value = .fromInterned(arg_val); |
| 3509 | const val_key = ip.indexToKey(val.toIntern()); | 3436 | const val_key = ip.indexToKey(val.toIntern()); |
| 3510 | | 3437 | |
| 3511 | if (val.isUndef(zcu)) { | | |
| 3512 | return o.builder.undefConst(try o.lowerType(pt, Type.fromInterned(val_key.typeOf()))); | | |
| 3513 | } | | |
| 3514 | | | |
| 3515 | const ty: Type = .fromInterned(val_key.typeOf()); | 3438 | const ty: Type = .fromInterned(val_key.typeOf()); |
| 3516 | ty.assertHasLayout(zcu); | 3439 | ty.assertHasLayout(zcu); |
| | 3440 | assert(ty.hasRuntimeBits(zcu)); |
| 3517 | | 3441 | |
| 3518 | return switch (val_key) { | 3442 | return switch (val_key) { |
| 3519 | .int_type, | 3443 | .int_type, |
| ... | @@ -3534,7 +3458,7 @@ pub const Object = struct { | ... | @@ -3534,7 +3458,7 @@ pub const Object = struct { |
| 3534 | .inferred_error_set_type, | 3458 | .inferred_error_set_type, |
| 3535 | => unreachable, // types, not values | 3459 | => unreachable, // types, not values |
| 3536 | | 3460 | |
| 3537 | .undef => unreachable, // handled above | 3461 | .undef => return o.builder.undefConst(try o.lowerType(ty)), |
| 3538 | .simple_value => |simple_value| switch (simple_value) { | 3462 | .simple_value => |simple_value| switch (simple_value) { |
| 3539 | .void => unreachable, // non-runtime value | 3463 | .void => unreachable, // non-runtime value |
| 3540 | .null => unreachable, // non-runtime value | 3464 | .null => unreachable, // non-runtime value |
| ... | @@ -3544,46 +3468,40 @@ pub const Object = struct { | ... | @@ -3544,46 +3468,40 @@ pub const Object = struct { |
| 3544 | .true => .true, | 3468 | .true => .true, |
| 3545 | }, | 3469 | }, |
| 3546 | .enum_literal => unreachable, // non-runtime value | 3470 | .enum_literal => unreachable, // non-runtime value |
| 3547 | .@"extern" => |@"extern"| { | 3471 | .@"extern" => unreachable, // non-runtime value |
| 3548 | const function_index = try o.resolveLlvmFunction(pt, @"extern".owner_nav); | 3472 | .func => unreachable, // non-runtime value |
| 3549 | return function_index.ptrConst(&o.builder).global.toConst(); | | |
| 3550 | }, | | |
| 3551 | .func => |func| { | | |
| 3552 | const function_index = try o.resolveLlvmFunction(pt, func.owner_nav); | | |
| 3553 | return function_index.ptrConst(&o.builder).global.toConst(); | | |
| 3554 | }, | | |
| 3555 | .int => { | 3473 | .int => { |
| 3556 | var bigint_space: Value.BigIntSpace = undefined; | 3474 | var bigint_space: Value.BigIntSpace = undefined; |
| 3557 | const bigint = val.toBigInt(&bigint_space, zcu); | 3475 | const bigint = val.toBigInt(&bigint_space, zcu); |
| 3558 | return lowerBigInt(o, pt, ty, bigint); | 3476 | const llvm_int_ty = try o.builder.intType(ty.intInfo(zcu).bits); |
| | 3477 | return o.builder.bigIntConst(llvm_int_ty, bigint); |
| 3559 | }, | 3478 | }, |
| 3560 | .err => |err| { | 3479 | .err => |err| { |
| 3561 | const int = try pt.getErrorValue(err.name); | 3480 | const int = zcu.intern_pool.getErrorValueIfExists(err.name).?; |
| 3562 | const llvm_int = try o.builder.intConst(try o.errorIntType(pt), int); | 3481 | return o.builder.intConst(try o.errorIntType(), int); |
| 3563 | return llvm_int; | | |
| 3564 | }, | 3482 | }, |
| 3565 | .error_union => |error_union| { | 3483 | .error_union => |error_union| { |
| 3566 | const err_val = switch (error_union.val) { | 3484 | const llvm_error_ty = try o.errorIntType(); |
| 3567 | .err_name => |err_name| try pt.intern(.{ .err = .{ | 3485 | const llvm_error_value = switch (error_union.val) { |
| 3568 | .ty = ty.errorUnionSet(zcu).toIntern(), | 3486 | .err_name => |name| try o.builder.intConst( |
| 3569 | .name = err_name, | 3487 | llvm_error_ty, |
| 3570 | } }), | 3488 | zcu.intern_pool.getErrorValueIfExists(name).?, |
| 3571 | .payload => (try pt.intValue(try pt.errorIntType(), 0)).toIntern(), | 3489 | ), |
| | 3490 | .payload => try o.builder.intConst(llvm_error_ty, 0), |
| 3572 | }; | 3491 | }; |
| 3573 | const err_int_ty = try pt.errorIntType(); | 3492 | |
| 3574 | const payload_type = ty.errorUnionPayload(zcu); | 3493 | const payload_type = ty.errorUnionPayload(zcu); |
| 3575 | if (!payload_type.hasRuntimeBits(zcu)) { | 3494 | if (!payload_type.hasRuntimeBits(zcu)) { |
| 3576 | // We use the error type directly as the type. | 3495 | // We use the error type directly as the type. |
| 3577 | return o.lowerValue(pt, err_val); | 3496 | return llvm_error_value; |
| 3578 | } | 3497 | } |
| 3579 | | 3498 | |
| 3580 | const payload_align = payload_type.abiAlignment(zcu); | 3499 | const payload_align = payload_type.abiAlignment(zcu); |
| 3581 | const error_align = err_int_ty.abiAlignment(zcu); | 3500 | const error_align = Type.errorAbiAlignment(zcu); |
| 3582 | const llvm_error_value = try o.lowerValue(pt, err_val); | 3501 | const llvm_payload_value = switch (error_union.val) { |
| 3583 | const llvm_payload_value = try o.lowerValue(pt, switch (error_union.val) { | 3502 | .err_name => try o.builder.undefConst(try o.lowerType(payload_type)), |
| 3584 | .err_name => try pt.intern(.{ .undef = payload_type.toIntern() }), | 3503 | .payload => |payload| try o.lowerValue(payload), |
| 3585 | .payload => |payload| payload, | 3504 | }; |
| 3586 | }); | | |
| 3587 | | 3505 | |
| 3588 | var fields: [3]Builder.Type = undefined; | 3506 | var fields: [3]Builder.Type = undefined; |
| 3589 | var vals: [3]Builder.Constant = undefined; | 3507 | var vals: [3]Builder.Constant = undefined; |
| ... | @@ -3597,7 +3515,7 @@ pub const Object = struct { | ... | @@ -3597,7 +3515,7 @@ pub const Object = struct { |
| 3597 | fields[0] = vals[0].typeOf(&o.builder); | 3515 | fields[0] = vals[0].typeOf(&o.builder); |
| 3598 | fields[1] = vals[1].typeOf(&o.builder); | 3516 | fields[1] = vals[1].typeOf(&o.builder); |
| 3599 | | 3517 | |
| 3600 | const llvm_ty = try o.lowerType(pt, ty); | 3518 | const llvm_ty = try o.lowerType(ty); |
| 3601 | const llvm_ty_fields = llvm_ty.structFields(&o.builder); | 3519 | const llvm_ty_fields = llvm_ty.structFields(&o.builder); |
| 3602 | if (llvm_ty_fields.len > 2) { | 3520 | if (llvm_ty_fields.len > 2) { |
| 3603 | assert(llvm_ty_fields.len == 3); | 3521 | assert(llvm_ty_fields.len == 3); |
| ... | @@ -3609,7 +3527,7 @@ pub const Object = struct { | ... | @@ -3609,7 +3527,7 @@ pub const Object = struct { |
| 3609 | fields[0..llvm_ty_fields.len], | 3527 | fields[0..llvm_ty_fields.len], |
| 3610 | ), vals[0..llvm_ty_fields.len]); | 3528 | ), vals[0..llvm_ty_fields.len]); |
| 3611 | }, | 3529 | }, |
| 3612 | .enum_tag => |enum_tag| o.lowerValue(pt, enum_tag.int), | 3530 | .enum_tag => |enum_tag| o.lowerValue(enum_tag.int), |
| 3613 | .float => switch (ty.floatBits(target)) { | 3531 | .float => switch (ty.floatBits(target)) { |
| 3614 | 16 => if (backendSupportsF16(target)) | 3532 | 16 => if (backendSupportsF16(target)) |
| 3615 | try o.builder.halfConst(val.toFloat(f16, zcu)) | 3533 | try o.builder.halfConst(val.toFloat(f16, zcu)) |
| ... | @@ -3624,10 +3542,10 @@ pub const Object = struct { | ... | @@ -3624,10 +3542,10 @@ pub const Object = struct { |
| 3624 | 128 => try o.builder.fp128Const(val.toFloat(f128, zcu)), | 3542 | 128 => try o.builder.fp128Const(val.toFloat(f128, zcu)), |
| 3625 | else => unreachable, | 3543 | else => unreachable, |
| 3626 | }, | 3544 | }, |
| 3627 | .ptr => try o.lowerPtr(pt, arg_val, 0), | 3545 | .ptr => try o.lowerPtr(arg_val, 0), |
| 3628 | .slice => |slice| return o.builder.structConst(try o.lowerType(pt, ty), &.{ | 3546 | .slice => |slice| return o.builder.structConst(try o.lowerType(ty), &.{ |
| 3629 | try o.lowerValue(pt, slice.ptr), | 3547 | try o.lowerValue(slice.ptr), |
| 3630 | try o.lowerValue(pt, slice.len), | 3548 | try o.lowerValue(slice.len), |
| 3631 | }), | 3549 | }), |
| 3632 | .opt => |opt| { | 3550 | .opt => |opt| { |
| 3633 | comptime assert(optional_layout_version == 3); | 3551 | comptime assert(optional_layout_version == 3); |
| ... | @@ -3637,7 +3555,7 @@ pub const Object = struct { | ... | @@ -3637,7 +3555,7 @@ pub const Object = struct { |
| 3637 | if (!payload_ty.hasRuntimeBits(zcu)) { | 3555 | if (!payload_ty.hasRuntimeBits(zcu)) { |
| 3638 | return non_null_bit; | 3556 | return non_null_bit; |
| 3639 | } | 3557 | } |
| 3640 | const llvm_ty = try o.lowerType(pt, ty); | 3558 | const llvm_ty = try o.lowerType(ty); |
| 3641 | if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) { | 3559 | if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) { |
| 3642 | .none => switch (llvm_ty.tag(&o.builder)) { | 3560 | .none => switch (llvm_ty.tag(&o.builder)) { |
| 3643 | .integer => try o.builder.intConst(llvm_ty, 0), | 3561 | .integer => try o.builder.intConst(llvm_ty, 0), |
| ... | @@ -3645,16 +3563,16 @@ pub const Object = struct { | ... | @@ -3645,16 +3563,16 @@ pub const Object = struct { |
| 3645 | .structure => try o.builder.zeroInitConst(llvm_ty), | 3563 | .structure => try o.builder.zeroInitConst(llvm_ty), |
| 3646 | else => unreachable, | 3564 | else => unreachable, |
| 3647 | }, | 3565 | }, |
| 3648 | else => |payload| try o.lowerValue(pt, payload), | 3566 | else => |payload| try o.lowerValue(payload), |
| 3649 | }; | 3567 | }; |
| 3650 | assert(payload_ty.zigTypeTag(zcu) != .@"fn"); | 3568 | assert(payload_ty.zigTypeTag(zcu) != .@"fn"); |
| 3651 | | 3569 | |
| 3652 | var fields: [3]Builder.Type = undefined; | 3570 | var fields: [3]Builder.Type = undefined; |
| 3653 | var vals: [3]Builder.Constant = undefined; | 3571 | var vals: [3]Builder.Constant = undefined; |
| 3654 | vals[0] = try o.lowerValue(pt, switch (opt.val) { | 3572 | vals[0] = switch (opt.val) { |
| 3655 | .none => try pt.intern(.{ .undef = payload_ty.toIntern() }), | 3573 | .none => try o.builder.undefConst(try o.lowerType(payload_ty)), |
| 3656 | else => |payload| payload, | 3574 | else => |payload| try o.lowerValue(payload), |
| 3657 | }); | 3575 | }; |
| 3658 | vals[1] = non_null_bit; | 3576 | vals[1] = non_null_bit; |
| 3659 | fields[0] = vals[0].typeOf(&o.builder); | 3577 | fields[0] = vals[0].typeOf(&o.builder); |
| 3660 | fields[1] = vals[1].typeOf(&o.builder); | 3578 | fields[1] = vals[1].typeOf(&o.builder); |
| ... | @@ -3670,14 +3588,14 @@ pub const Object = struct { | ... | @@ -3670,14 +3588,14 @@ pub const Object = struct { |
| 3670 | fields[0..llvm_ty_fields.len], | 3588 | fields[0..llvm_ty_fields.len], |
| 3671 | ), vals[0..llvm_ty_fields.len]); | 3589 | ), vals[0..llvm_ty_fields.len]); |
| 3672 | }, | 3590 | }, |
| 3673 | .bitpack => |bitpack| return o.lowerValue(pt, bitpack.backing_int_val), | 3591 | .bitpack => |bitpack| return o.lowerValue(bitpack.backing_int_val), |
| 3674 | .aggregate => |aggregate| switch (ip.indexToKey(ty.toIntern())) { | 3592 | .aggregate => |aggregate| switch (ip.indexToKey(ty.toIntern())) { |
| 3675 | .array_type => |array_type| switch (aggregate.storage) { | 3593 | .array_type => |array_type| switch (aggregate.storage) { |
| 3676 | .bytes => |bytes| try o.builder.stringConst(try o.builder.string( | 3594 | .bytes => |bytes| try o.builder.stringConst(try o.builder.string( |
| 3677 | bytes.toSlice(array_type.lenIncludingSentinel(), ip), | 3595 | bytes.toSlice(array_type.lenIncludingSentinel(), ip), |
| 3678 | )), | 3596 | )), |
| 3679 | .elems => |elems| { | 3597 | .elems => |elems| { |
| 3680 | const array_ty = try o.lowerType(pt, ty); | 3598 | const array_ty = try o.lowerType(ty); |
| 3681 | const elem_ty = array_ty.childType(&o.builder); | 3599 | const elem_ty = array_ty.childType(&o.builder); |
| 3682 | assert(elems.len == array_ty.aggregateLen(&o.builder)); | 3600 | assert(elems.len == array_ty.aggregateLen(&o.builder)); |
| 3683 | | 3601 | |
| ... | @@ -3697,7 +3615,7 @@ pub const Object = struct { | ... | @@ -3697,7 +3615,7 @@ pub const Object = struct { |
| 3697 | | 3615 | |
| 3698 | var need_unnamed = false; | 3616 | var need_unnamed = false; |
| 3699 | for (vals, fields, elems) |*result_val, *result_field, elem| { | 3617 | for (vals, fields, elems) |*result_val, *result_field, elem| { |
| 3700 | result_val.* = try o.lowerValue(pt, elem); | 3618 | result_val.* = try o.lowerValue(elem); |
| 3701 | result_field.* = result_val.typeOf(&o.builder); | 3619 | result_field.* = result_val.typeOf(&o.builder); |
| 3702 | if (result_field.* != elem_ty) need_unnamed = true; | 3620 | if (result_field.* != elem_ty) need_unnamed = true; |
| 3703 | } | 3621 | } |
| ... | @@ -3709,7 +3627,7 @@ pub const Object = struct { | ... | @@ -3709,7 +3627,7 @@ pub const Object = struct { |
| 3709 | .repeated_elem => |elem| { | 3627 | .repeated_elem => |elem| { |
| 3710 | const len: usize = @intCast(array_type.len); | 3628 | const len: usize = @intCast(array_type.len); |
| 3711 | const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel()); | 3629 | const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel()); |
| 3712 | const array_ty = try o.lowerType(pt, ty); | 3630 | const array_ty = try o.lowerType(ty); |
| 3713 | const elem_ty = array_ty.childType(&o.builder); | 3631 | const elem_ty = array_ty.childType(&o.builder); |
| 3714 | | 3632 | |
| 3715 | const ExpectedContents = extern struct { | 3633 | const ExpectedContents = extern struct { |
| ... | @@ -3727,12 +3645,12 @@ pub const Object = struct { | ... | @@ -3727,12 +3645,12 @@ pub const Object = struct { |
| 3727 | defer allocator.free(fields); | 3645 | defer allocator.free(fields); |
| 3728 | | 3646 | |
| 3729 | var need_unnamed = false; | 3647 | var need_unnamed = false; |
| 3730 | @memset(vals[0..len], try o.lowerValue(pt, elem)); | 3648 | @memset(vals[0..len], try o.lowerValue(elem)); |
| 3731 | @memset(fields[0..len], vals[0].typeOf(&o.builder)); | 3649 | @memset(fields[0..len], vals[0].typeOf(&o.builder)); |
| 3732 | if (fields[0] != elem_ty) need_unnamed = true; | 3650 | if (fields[0] != elem_ty) need_unnamed = true; |
| 3733 | | 3651 | |
| 3734 | if (array_type.sentinel != .none) { | 3652 | if (array_type.sentinel != .none) { |
| 3735 | vals[len] = try o.lowerValue(pt, array_type.sentinel); | 3653 | vals[len] = try o.lowerValue(array_type.sentinel); |
| 3736 | fields[len] = vals[len].typeOf(&o.builder); | 3654 | fields[len] = vals[len].typeOf(&o.builder); |
| 3737 | if (fields[len] != elem_ty) need_unnamed = true; | 3655 | if (fields[len] != elem_ty) need_unnamed = true; |
| 3738 | } | 3656 | } |
| ... | @@ -3744,7 +3662,7 @@ pub const Object = struct { | ... | @@ -3744,7 +3662,7 @@ pub const Object = struct { |
| 3744 | }, | 3662 | }, |
| 3745 | }, | 3663 | }, |
| 3746 | .vector_type => |vector_type| { | 3664 | .vector_type => |vector_type| { |
| 3747 | const vector_ty = try o.lowerType(pt, ty); | 3665 | const vector_ty = try o.lowerType(ty); |
| 3748 | switch (aggregate.storage) { | 3666 | switch (aggregate.storage) { |
| 3749 | .bytes, .elems => { | 3667 | .bytes, .elems => { |
| 3750 | const ExpectedContents = [Builder.expected_fields_len]Builder.Constant; | 3668 | const ExpectedContents = [Builder.expected_fields_len]Builder.Constant; |
| ... | @@ -3761,7 +3679,7 @@ pub const Object = struct { | ... | @@ -3761,7 +3679,7 @@ pub const Object = struct { |
| 3761 | result_val.* = try o.builder.intConst(.i8, byte); | 3679 | result_val.* = try o.builder.intConst(.i8, byte); |
| 3762 | }, | 3680 | }, |
| 3763 | .elems => |elems| for (vals, elems) |*result_val, elem| { | 3681 | .elems => |elems| for (vals, elems) |*result_val, elem| { |
| 3764 | result_val.* = try o.lowerValue(pt, elem); | 3682 | result_val.* = try o.lowerValue(elem); |
| 3765 | }, | 3683 | }, |
| 3766 | .repeated_elem => unreachable, | 3684 | .repeated_elem => unreachable, |
| 3767 | } | 3685 | } |
| ... | @@ -3769,12 +3687,12 @@ pub const Object = struct { | ... | @@ -3769,12 +3687,12 @@ pub const Object = struct { |
| 3769 | }, | 3687 | }, |
| 3770 | .repeated_elem => |elem| return o.builder.splatConst( | 3688 | .repeated_elem => |elem| return o.builder.splatConst( |
| 3771 | vector_ty, | 3689 | vector_ty, |
| 3772 | try o.lowerValue(pt, elem), | 3690 | try o.lowerValue(elem), |
| 3773 | ), | 3691 | ), |
| 3774 | } | 3692 | } |
| 3775 | }, | 3693 | }, |
| 3776 | .tuple_type => |tuple| { | 3694 | .tuple_type => |tuple| { |
| 3777 | const struct_ty = try o.lowerType(pt, ty); | 3695 | const struct_ty = try o.lowerType(ty); |
| 3778 | const llvm_len = struct_ty.aggregateLen(&o.builder); | 3696 | const llvm_len = struct_ty.aggregateLen(&o.builder); |
| 3779 | | 3697 | |
| 3780 | const ExpectedContents = extern struct { | 3698 | const ExpectedContents = extern struct { |
| ... | @@ -3794,14 +3712,14 @@ pub const Object = struct { | ... | @@ -3794,14 +3712,14 @@ pub const Object = struct { |
| 3794 | comptime assert(struct_layout_version == 2); | 3712 | comptime assert(struct_layout_version == 2); |
| 3795 | var llvm_index: usize = 0; | 3713 | var llvm_index: usize = 0; |
| 3796 | var offset: u64 = 0; | 3714 | var offset: u64 = 0; |
| 3797 | var big_align: InternPool.Alignment = .none; | 3715 | var big_align: InternPool.Alignment = .@"1"; |
| 3798 | var need_unnamed = false; | 3716 | var need_unnamed = false; |
| 3799 | for ( | 3717 | for ( |
| 3800 | tuple.types.get(ip), | 3718 | tuple.types.get(ip), |
| 3801 | tuple.values.get(ip), | 3719 | tuple.values.get(ip), |
| 3802 | 0.., | 3720 | 0.., |
| 3803 | ) |field_ty, field_val, field_index| { | 3721 | ) |field_ty, field_comptime_val, field_index| { |
| 3804 | if (field_val != .none) continue; | 3722 | if (field_comptime_val != .none) continue; |
| 3805 | if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue; | 3723 | if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue; |
| 3806 | | 3724 | |
| 3807 | const field_align = Type.fromInterned(field_ty).abiAlignment(zcu); | 3725 | const field_align = Type.fromInterned(field_ty).abiAlignment(zcu); |
| ... | @@ -3819,8 +3737,11 @@ pub const Object = struct { | ... | @@ -3819,8 +3737,11 @@ pub const Object = struct { |
| 3819 | llvm_index += 1; | 3737 | llvm_index += 1; |
| 3820 | } | 3738 | } |
| 3821 | | 3739 | |
| 3822 | vals[llvm_index] = | 3740 | vals[llvm_index] = switch (aggregate.storage) { |
| 3823 | try o.lowerValue(pt, (try val.fieldValue(pt, field_index)).toIntern()); | 3741 | .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)), |
| | 3742 | .elems => |elems| try o.lowerValue(elems[field_index]), |
| | 3743 | .repeated_elem => |elem| try o.lowerValue(elem), |
| | 3744 | }; |
| 3824 | fields[llvm_index] = vals[llvm_index].typeOf(&o.builder); | 3745 | fields[llvm_index] = vals[llvm_index].typeOf(&o.builder); |
| 3825 | if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index]) | 3746 | if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index]) |
| 3826 | need_unnamed = true; | 3747 | need_unnamed = true; |
| ... | @@ -3848,7 +3769,7 @@ pub const Object = struct { | ... | @@ -3848,7 +3769,7 @@ pub const Object = struct { |
| 3848 | }, | 3769 | }, |
| 3849 | .struct_type => { | 3770 | .struct_type => { |
| 3850 | const struct_type = ip.loadStructType(ty.toIntern()); | 3771 | const struct_type = ip.loadStructType(ty.toIntern()); |
| 3851 | const struct_ty = try o.lowerType(pt, ty); | 3772 | const struct_ty = try o.lowerType(ty); |
| 3852 | assert(struct_type.layout != .@"packed"); | 3773 | assert(struct_type.layout != .@"packed"); |
| 3853 | const llvm_len = struct_ty.aggregateLen(&o.builder); | 3774 | const llvm_len = struct_ty.aggregateLen(&o.builder); |
| 3854 | | 3775 | |
| ... | @@ -3892,10 +3813,11 @@ pub const Object = struct { | ... | @@ -3892,10 +3813,11 @@ pub const Object = struct { |
| 3892 | continue; | 3813 | continue; |
| 3893 | } | 3814 | } |
| 3894 | | 3815 | |
| 3895 | vals[llvm_index] = try o.lowerValue( | 3816 | vals[llvm_index] = switch (aggregate.storage) { |
| 3896 | pt, | 3817 | .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)), |
| 3897 | (try val.fieldValue(pt, field_index)).toIntern(), | 3818 | .elems => |elems| try o.lowerValue(elems[field_index]), |
| 3898 | ); | 3819 | .repeated_elem => |elem| try o.lowerValue(elem), |
| | 3820 | }; |
| 3899 | fields[llvm_index] = vals[llvm_index].typeOf(&o.builder); | 3821 | fields[llvm_index] = vals[llvm_index].typeOf(&o.builder); |
| 3900 | if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index]) | 3822 | if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index]) |
| 3901 | need_unnamed = true; | 3823 | need_unnamed = true; |
| ... | @@ -3924,9 +3846,9 @@ pub const Object = struct { | ... | @@ -3924,9 +3846,9 @@ pub const Object = struct { |
| 3924 | else => unreachable, | 3846 | else => unreachable, |
| 3925 | }, | 3847 | }, |
| 3926 | .un => |un| { | 3848 | .un => |un| { |
| 3927 | const union_ty = try o.lowerType(pt, ty); | 3849 | const union_ty = try o.lowerType(ty); |
| 3928 | const layout = ty.unionGetLayout(zcu); | 3850 | const layout = ty.unionGetLayout(zcu); |
| 3929 | if (layout.payload_size == 0) return o.lowerValue(pt, un.tag); | 3851 | if (layout.payload_size == 0) return o.lowerValue(un.tag); |
| 3930 | | 3852 | |
| 3931 | const union_obj = zcu.typeToUnion(ty).?; | 3853 | const union_obj = zcu.typeToUnion(ty).?; |
| 3932 | const container_layout = union_obj.layout; | 3854 | const container_layout = union_obj.layout; |
| ... | @@ -3947,7 +3869,7 @@ pub const Object = struct { | ... | @@ -3947,7 +3869,7 @@ pub const Object = struct { |
| 3947 | const padding_len = layout.payload_size; | 3869 | const padding_len = layout.payload_size; |
| 3948 | break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8)); | 3870 | break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8)); |
| 3949 | } | 3871 | } |
| 3950 | const payload = try o.lowerValue(pt, un.val); | 3872 | const payload = try o.lowerValue(un.val); |
| 3951 | const payload_ty = payload.typeOf(&o.builder); | 3873 | const payload_ty = payload.typeOf(&o.builder); |
| 3952 | if (payload_ty != union_ty.structFields(&o.builder)[ | 3874 | if (payload_ty != union_ty.structFields(&o.builder)[ |
| 3953 | @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align)) | 3875 | @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align)) |
| ... | @@ -3962,7 +3884,7 @@ pub const Object = struct { | ... | @@ -3962,7 +3884,7 @@ pub const Object = struct { |
| 3962 | ); | 3884 | ); |
| 3963 | } else p: { | 3885 | } else p: { |
| 3964 | assert(layout.tag_size == 0); | 3886 | assert(layout.tag_size == 0); |
| 3965 | const union_val = try o.lowerValue(pt, un.val); | 3887 | const union_val = try o.lowerValue(un.val); |
| 3966 | need_unnamed = true; | 3888 | need_unnamed = true; |
| 3967 | break :p union_val; | 3889 | break :p union_val; |
| 3968 | }; | 3890 | }; |
| ... | @@ -3972,7 +3894,7 @@ pub const Object = struct { | ... | @@ -3972,7 +3894,7 @@ pub const Object = struct { |
| 3972 | try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty}) | 3894 | try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty}) |
| 3973 | else | 3895 | else |
| 3974 | union_ty, &.{payload}); | 3896 | union_ty, &.{payload}); |
| 3975 | const tag = try o.lowerValue(pt, un.tag); | 3897 | const tag = try o.lowerValue(un.tag); |
| 3976 | const tag_ty = tag.typeOf(&o.builder); | 3898 | const tag_ty = tag.typeOf(&o.builder); |
| 3977 | var fields: [3]Builder.Type = undefined; | 3899 | var fields: [3]Builder.Type = undefined; |
| 3978 | var vals: [3]Builder.Constant = undefined; | 3900 | var vals: [3]Builder.Constant = undefined; |
| ... | @@ -3998,52 +3920,45 @@ pub const Object = struct { | ... | @@ -3998,52 +3920,45 @@ pub const Object = struct { |
| 3998 | }; | 3920 | }; |
| 3999 | } | 3921 | } |
| 4000 | | 3922 | |
| 4001 | fn lowerBigInt( | | |
| 4002 | o: *Object, | | |
| 4003 | pt: Zcu.PerThread, | | |
| 4004 | ty: Type, | | |
| 4005 | bigint: std.math.big.int.Const, | | |
| 4006 | ) Allocator.Error!Builder.Constant { | | |
| 4007 | const zcu = pt.zcu; | | |
| 4008 | return o.builder.bigIntConst(try o.builder.intType(ty.intInfo(zcu).bits), bigint); | | |
| 4009 | } | | |
| 4010 | | | |
| 4011 | fn lowerPtr( | 3923 | fn lowerPtr( |
| 4012 | o: *Object, | 3924 | o: *Object, |
| 4013 | pt: Zcu.PerThread, | | |
| 4014 | ptr_val: InternPool.Index, | 3925 | ptr_val: InternPool.Index, |
| 4015 | prev_offset: u64, | 3926 | prev_offset: u64, |
| 4016 | ) Allocator.Error!Builder.Constant { | 3927 | ) Allocator.Error!Builder.Constant { |
| 4017 | const zcu = pt.zcu; | 3928 | const zcu = o.zcu; |
| 4018 | const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr; | 3929 | const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr; |
| 4019 | const offset: u64 = prev_offset + ptr.byte_offset; | 3930 | const offset: u64 = prev_offset + ptr.byte_offset; |
| 4020 | return switch (ptr.base_addr) { | 3931 | return switch (ptr.base_addr) { |
| 4021 | .nav => |nav| { | 3932 | .nav => |nav| { |
| 4022 | const base_ptr = try o.lowerNavRefValue(pt, nav); | 3933 | const base_ptr = try o.lowerNavRef(nav); |
| 4023 | return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{ | 3934 | return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{ |
| 4024 | try o.builder.intConst(.i64, offset), | 3935 | try o.builder.intConst(.i64, offset), |
| 4025 | }); | 3936 | }); |
| 4026 | }, | 3937 | }, |
| 4027 | .uav => |uav| { | 3938 | .uav => |uav| { |
| 4028 | const base_ptr = try o.lowerUavRef(pt, uav); | 3939 | const orig_ptr_ty: Type = .fromInterned(uav.orig_ty); |
| | 3940 | const base_ptr = try o.lowerUavRef( |
| | 3941 | uav.val, |
| | 3942 | orig_ptr_ty.ptrAlignment(zcu), |
| | 3943 | orig_ptr_ty.ptrAddressSpace(zcu), |
| | 3944 | ); |
| 4029 | return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{ | 3945 | return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{ |
| 4030 | try o.builder.intConst(.i64, offset), | 3946 | try o.builder.intConst(.i64, offset), |
| 4031 | }); | 3947 | }); |
| 4032 | }, | 3948 | }, |
| 4033 | .int => try o.builder.castConst( | 3949 | .int => try o.builder.castConst( |
| 4034 | .inttoptr, | 3950 | .inttoptr, |
| 4035 | try o.builder.intConst(try o.lowerType(pt, Type.usize), offset), | 3951 | try o.builder.intConst(try o.lowerType(.usize), offset), |
| 4036 | try o.lowerType(pt, Type.fromInterned(ptr.ty)), | 3952 | try o.lowerType(.fromInterned(ptr.ty)), |
| 4037 | ), | 3953 | ), |
| 4038 | .eu_payload => |eu_ptr| try o.lowerPtr( | 3954 | .eu_payload => |eu_ptr| try o.lowerPtr( |
| 4039 | pt, | | |
| 4040 | eu_ptr, | 3955 | eu_ptr, |
| 4041 | offset + codegen.errUnionPayloadOffset( | 3956 | offset + codegen.errUnionPayloadOffset( |
| 4042 | Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu), | 3957 | Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu), |
| 4043 | zcu, | 3958 | zcu, |
| 4044 | ), | 3959 | ), |
| 4045 | ), | 3960 | ), |
| 4046 | .opt_payload => |opt_ptr| try o.lowerPtr(pt, opt_ptr, offset), | 3961 | .opt_payload => |opt_ptr| try o.lowerPtr(opt_ptr, offset), |
| 4047 | .field => |field| { | 3962 | .field => |field| { |
| 4048 | const agg_ty = Value.fromInterned(field.base).typeOf(zcu).childType(zcu); | 3963 | const agg_ty = Value.fromInterned(field.base).typeOf(zcu).childType(zcu); |
| 4049 | const field_off: u64 = switch (agg_ty.zigTypeTag(zcu)) { | 3964 | const field_off: u64 = switch (agg_ty.zigTypeTag(zcu)) { |
| ... | @@ -4061,132 +3976,118 @@ pub const Object = struct { | ... | @@ -4061,132 +3976,118 @@ pub const Object = struct { |
| 4061 | }, | 3976 | }, |
| 4062 | else => unreachable, | 3977 | else => unreachable, |
| 4063 | }; | 3978 | }; |
| 4064 | return o.lowerPtr(pt, field.base, offset + field_off); | 3979 | return o.lowerPtr(field.base, offset + field_off); |
| 4065 | }, | 3980 | }, |
| 4066 | .arr_elem => |arr_elem| { | 3981 | .arr_elem => |arr_elem| { |
| 4067 | const base_ptr_ty = Value.fromInterned(arr_elem.base).typeOf(zcu); | 3982 | const base_ptr_ty = Value.fromInterned(arr_elem.base).typeOf(zcu); |
| 4068 | assert(base_ptr_ty.ptrSize(zcu) == .many); | 3983 | assert(base_ptr_ty.ptrSize(zcu) == .many); |
| 4069 | const elem_size = base_ptr_ty.childType(zcu).abiSize(zcu); | 3984 | const elem_size = base_ptr_ty.childType(zcu).abiSize(zcu); |
| 4070 | return o.lowerPtr(pt, arr_elem.base, offset + elem_size * arr_elem.index); | 3985 | return o.lowerPtr(arr_elem.base, offset + elem_size * arr_elem.index); |
| 4071 | }, | 3986 | }, |
| 4072 | .comptime_field => unreachable, | 3987 | .comptime_field => unreachable, |
| 4073 | .comptime_alloc => unreachable, | 3988 | .comptime_alloc => unreachable, |
| 4074 | }; | 3989 | }; |
| 4075 | } | 3990 | } |
| 4076 | | 3991 | |
| 4077 | /// This logic is very similar to `lowerNavRefValue` but for anonymous declarations. | 3992 | pub fn lowerPtrToVoid( |
| 4078 | /// Maybe the logic could be unified. | | |
| 4079 | fn lowerUavRef( | | |
| 4080 | o: *Object, | 3993 | o: *Object, |
| 4081 | pt: Zcu.PerThread, | 3994 | /// Must not be `.none`. |
| 4082 | uav: InternPool.Key.Ptr.BaseAddr.Uav, | 3995 | @"align": InternPool.Alignment, |
| | 3996 | @"addrspace": std.builtin.AddressSpace, |
| 4083 | ) Allocator.Error!Builder.Constant { | 3997 | ) Allocator.Error!Builder.Constant { |
| 4084 | const zcu = pt.zcu; | 3998 | const addr: u64 = @"align".toByteUnits().?; |
| | 3999 | const llvm_usize = try o.lowerType(.usize); |
| | 4000 | const llvm_addr = try o.builder.intConst(llvm_usize, addr); |
| | 4001 | const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget())); |
| | 4002 | return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty); |
| | 4003 | } |
| | 4004 | |
| | 4005 | pub fn lowerUavRef( |
| | 4006 | o: *Object, |
| | 4007 | uav_val: InternPool.Index, |
| | 4008 | /// Must not be `.none`. |
| | 4009 | @"align": InternPool.Alignment, |
| | 4010 | @"addrspace": std.builtin.AddressSpace, |
| | 4011 | ) Allocator.Error!Builder.Constant { |
| | 4012 | assert(@"align" != .none); |
| | 4013 | |
| | 4014 | const zcu = o.zcu; |
| 4085 | const ip = &zcu.intern_pool; | 4015 | const ip = &zcu.intern_pool; |
| 4086 | const uav_val = uav.val; | 4016 | const gpa = zcu.comp.gpa; |
| 4087 | const uav_ty = Type.fromInterned(ip.typeOf(uav_val)); | 4017 | |
| 4088 | const target = zcu.getTarget(); | 4018 | const uav_ty: Type = .fromInterned(ip.typeOf(uav_val)); |
| 4089 | | 4019 | |
| 4090 | switch (ip.indexToKey(uav_val)) { | 4020 | switch (ip.indexToKey(uav_val)) { |
| 4091 | .func => @panic("TODO"), | 4021 | .func => unreachable, // should be using a Nav ref |
| 4092 | .@"extern" => @panic("TODO"), | 4022 | .@"extern" => unreachable, // should be using a Nav ref |
| 4093 | else => {}, | 4023 | else => {}, |
| 4094 | } | 4024 | } |
| 4095 | | 4025 | |
| 4096 | const ptr_ty = Type.fromInterned(uav.orig_ty); | 4026 | if (!uav_ty.hasRuntimeBits(zcu)) { |
| 4097 | | 4027 | return o.lowerPtrToVoid(@"align", @"addrspace"); |
| 4098 | if (!uav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) { | | |
| 4099 | return o.lowerPtrToVoid(pt, ptr_ty); | | |
| 4100 | } | 4028 | } |
| 4101 | | 4029 | |
| 4102 | assert(uav_ty.zigTypeTag(zcu) != .@"fn"); // should be using a Nav ref | 4030 | const llvm_addrspace = toLlvmAddressSpace(@"addrspace", zcu.getTarget()); |
| 4103 | | | |
| 4104 | const llvm_addr_space = toLlvmAddressSpace(ptr_ty.ptrAddressSpace(zcu), target); | | |
| 4105 | const alignment = ptr_ty.ptrAlignment(zcu); | | |
| 4106 | const llvm_global = (try o.resolveGlobalUav(pt, uav.val, llvm_addr_space, alignment)).ptrConst(&o.builder).global; | | |
| 4107 | | | |
| 4108 | const llvm_val = try o.builder.convConst( | | |
| 4109 | llvm_global.toConst(), | | |
| 4110 | try o.builder.ptrType(llvm_addr_space), | | |
| 4111 | ); | | |
| 4112 | | | |
| 4113 | return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty)); | | |
| 4114 | } | | |
| 4115 | | 4031 | |
| 4116 | fn lowerNavRefValue(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) Allocator.Error!Builder.Constant { | 4032 | const gop = try o.uav_map.getOrPut(gpa, .{ .val = uav_val, .@"addrspace" = @"addrspace" }); |
| 4117 | const zcu = pt.zcu; | 4033 | if (gop.found_existing) { |
| | 4034 | // Keep the greater of the two alignments. |
| | 4035 | const llvm_variable = gop.value_ptr.*; |
| | 4036 | const old_align: InternPool.Alignment = .fromLlvm(llvm_variable.getAlignment(&o.builder)); |
| | 4037 | llvm_variable.setAlignment(old_align.maxStrict(@"align").toLlvm(), &o.builder); |
| | 4038 | return llvm_variable.ptrConst(&o.builder).global.toConst(); |
| | 4039 | } |
| | 4040 | errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" })); |
| | 4041 | |
| | 4042 | const llvm_ty = try o.lowerType(uav_ty); |
| | 4043 | const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav_val)}); |
| | 4044 | const llvm_variable = try o.builder.addVariable(llvm_name, llvm_ty, llvm_addrspace); |
| | 4045 | gop.value_ptr.* = llvm_variable; |
| | 4046 | try llvm_variable.setInitializer(try o.lowerValue(uav_val), &o.builder); |
| | 4047 | llvm_variable.setMutability(.constant, &o.builder); |
| | 4048 | llvm_variable.setAlignment(@"align".toLlvm(), &o.builder); |
| | 4049 | const llvm_global = llvm_variable.ptrConst(&o.builder).global; |
| | 4050 | llvm_global.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); |
| | 4051 | llvm_global.setUnnamedAddr(.unnamed_addr, &o.builder); |
| | 4052 | return llvm_global.toConst(); |
| | 4053 | } |
| | 4054 | |
| | 4055 | pub fn lowerNavRef(o: *Object, nav_id: InternPool.Nav.Index) Allocator.Error!Builder.Constant { |
| | 4056 | const zcu = o.zcu; |
| 4118 | const ip = &zcu.intern_pool; | 4057 | const ip = &zcu.intern_pool; |
| | 4058 | const gpa = zcu.comp.gpa; |
| 4119 | | 4059 | |
| 4120 | const nav = ip.getNav(nav_index); | 4060 | const nav = ip.getNav(nav_id); |
| 4121 | | | |
| 4122 | const nav_ty: Type = .fromInterned(nav.resolved.?.type); | 4061 | const nav_ty: Type = .fromInterned(nav.resolved.?.type); |
| 4123 | const ptr_ty = try pt.navPtrType(nav_index); | 4062 | if (!nav_ty.isRuntimeFnOrHasRuntimeBits(zcu) and nav.getExtern(ip) == null) { |
| 4124 | | 4063 | const nav_align = switch (nav.resolved.?.@"align") { |
| 4125 | if (nav.getExtern(ip) == null and !nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) { | 4064 | .none => nav_ty.abiAlignment(zcu), |
| 4126 | return o.lowerPtrToVoid(pt, ptr_ty); | 4065 | else => |a| a, |
| 4127 | } | | |
| 4128 | | | |
| 4129 | const llvm_global = if (nav_ty.zigTypeTag(zcu) == .@"fn") | | |
| 4130 | (try o.resolveLlvmFunction(pt, nav_index)).ptrConst(&o.builder).global | | |
| 4131 | else | | |
| 4132 | (try o.resolveGlobalNav(pt, nav_index)).ptrConst(&o.builder).global; | | |
| 4133 | | | |
| 4134 | const llvm_val = try o.builder.convConst( | | |
| 4135 | llvm_global.toConst(), | | |
| 4136 | try o.builder.ptrType(toLlvmAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget())), | | |
| 4137 | ); | | |
| 4138 | | | |
| 4139 | return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty)); | | |
| 4140 | } | | |
| 4141 | | | |
| 4142 | fn lowerPtrToVoid(o: *Object, pt: Zcu.PerThread, ptr_ty: Type) Allocator.Error!Builder.Constant { | | |
| 4143 | const zcu = pt.zcu; | | |
| 4144 | // Even though we are pointing at something which has zero bits (e.g. `void`), | | |
| 4145 | // Pointers are defined to have bits. So we must return something here. | | |
| 4146 | // The value cannot be undefined, because we use the `nonnull` annotation | | |
| 4147 | // for non-optional pointers. We also need to respect the alignment, even though | | |
| 4148 | // the address will never be dereferenced. | | |
| 4149 | const int: u64 = ptr_ty.ptrInfo(zcu).flags.alignment.toByteUnits() orelse | | |
| 4150 | // Note that these 0xaa values are appropriate even in release-optimized builds | | |
| 4151 | // because we need a well-defined value that is not null, and LLVM does not | | |
| 4152 | // have an "undef_but_not_null" attribute. As an example, if this `alloc` AIR | | |
| 4153 | // instruction is followed by a `wrap_optional`, it will return this value | | |
| 4154 | // verbatim, and the result should test as non-null. | | |
| 4155 | switch (zcu.getTarget().ptrBitWidth()) { | | |
| 4156 | 16 => 0xaaaa, | | |
| 4157 | 32 => 0xaaaaaaaa, | | |
| 4158 | 64 => 0xaaaaaaaa_aaaaaaaa, | | |
| 4159 | else => unreachable, | | |
| 4160 | }; | 4066 | }; |
| 4161 | const llvm_usize = try o.lowerType(pt, Type.usize); | 4067 | return o.lowerPtrToVoid(nav_align, nav.resolved.?.@"addrspace"); |
| 4162 | const llvm_ptr_ty = try o.lowerType(pt, ptr_ty); | 4068 | } |
| 4163 | return o.builder.castConst(.inttoptr, try o.builder.intConst(llvm_usize, int), llvm_ptr_ty); | 4069 | |
| 4164 | } | 4070 | const gop = try o.nav_map.getOrPut(gpa, nav_id); |
| 4165 | | 4071 | if (!gop.found_existing) { |
| 4166 | /// If the operand type of an atomic operation is not byte sized we need to | 4072 | errdefer assert(o.nav_map.remove(nav_id)); |
| 4167 | /// widen it before using it and then truncate the result. | 4073 | // The NAV hasn't been lowered yet, so generate a placeholder global whose details will |
| 4168 | /// RMW exchange of floating-point values is bitcasted to same-sized integer | 4074 | // be filled in later. |
| 4169 | /// types to work around a LLVM deficiency when targeting ARM/AArch64. | 4075 | const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip)); |
| 4170 | fn getAtomicAbiType(o: *Object, pt: Zcu.PerThread, ty: Type, is_rmw_xchg: bool) Allocator.Error!Builder.Type { | 4076 | gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{ |
| 4171 | const zcu = pt.zcu; | 4077 | .type = .void, // placeholder; populated by `updateNav`/`updateFunc` |
| 4172 | switch (ty.zigTypeTag(zcu)) { | 4078 | .kind = .{ .alias = .none }, // placeholder; populated by `updateNav`/`updateFunc` |
| 4173 | .int, .@"enum", .@"struct", .@"union" => {}, | 4079 | }); |
| 4174 | .float => { | | |
| 4175 | if (!is_rmw_xchg) return .none; | | |
| 4176 | return o.builder.intType(@intCast(ty.abiSize(zcu) * 8)); | | |
| 4177 | }, | | |
| 4178 | .bool => return .i8, | | |
| 4179 | else => return .none, | | |
| 4180 | } | | |
| 4181 | const bit_count = ty.bitSize(zcu); | | |
| 4182 | if (!std.math.isPowerOfTwo(bit_count) or (bit_count % 8) != 0) { | | |
| 4183 | return o.builder.intType(@intCast(ty.abiSize(zcu) * 8)); | | |
| 4184 | } else { | | |
| 4185 | return .none; | | |
| 4186 | } | 4080 | } |
| | 4081 | const llvm_global = gop.value_ptr.*; |
| | 4082 | |
| | 4083 | // We need to make sure the global's address space is up to date, because that affects the |
| | 4084 | // type of a pointer to this global. But everything else about the global will be populated |
| | 4085 | // by `updateNav` or `updateFunc`. |
| | 4086 | llvm_global.ptr(&o.builder).addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget()); |
| | 4087 | return llvm_global.toConst(); |
| 4187 | } | 4088 | } |
| 4188 | | 4089 | |
| 4189 | fn addByValParamAttrs( | 4090 | pub fn addByValParamAttrs( |
| 4190 | o: *Object, | 4091 | o: *Object, |
| 4191 | pt: Zcu.PerThread, | 4092 | pt: Zcu.PerThread, |
| 4192 | attributes: *Builder.FunctionAttributes.Wip, | 4093 | attributes: *Builder.FunctionAttributes.Wip, |
| ... | @@ -4195,10 +4096,10 @@ pub const Object = struct { | ... | @@ -4195,10 +4096,10 @@ pub const Object = struct { |
| 4195 | fn_info: InternPool.Key.FuncType, | 4096 | fn_info: InternPool.Key.FuncType, |
| 4196 | llvm_arg_i: u32, | 4097 | llvm_arg_i: u32, |
| 4197 | ) Allocator.Error!void { | 4098 | ) Allocator.Error!void { |
| 4198 | const zcu = pt.zcu; | 4099 | const zcu = o.zcu; |
| 4199 | if (param_ty.isPtrAtRuntime(zcu)) { | 4100 | if (param_ty.isPtrAtRuntime(zcu)) { |
| 4200 | const ptr_info = param_ty.ptrInfo(zcu); | 4101 | const ptr_info = param_ty.ptrInfo(zcu); |
| 4201 | if (math.cast(u5, param_index)) |i| { | 4102 | if (std.math.cast(u5, param_index)) |i| { |
| 4202 | if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { | 4103 | if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { |
| 4203 | try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); | 4104 | try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); |
| 4204 | } | 4105 | } |
| ... | @@ -4214,7 +4115,7 @@ pub const Object = struct { | ... | @@ -4214,7 +4115,7 @@ pub const Object = struct { |
| 4214 | .x86_64_interrupt, | 4115 | .x86_64_interrupt, |
| 4215 | .x86_interrupt, | 4116 | .x86_interrupt, |
| 4216 | => { | 4117 | => { |
| 4217 | const child_type = try lowerType(o, pt, Type.fromInterned(ptr_info.child)); | 4118 | const child_type = try lowerType(o, Type.fromInterned(ptr_info.child)); |
| 4218 | try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder); | 4119 | try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder); |
| 4219 | }, | 4120 | }, |
| 4220 | } | 4121 | } |
| ... | @@ -4232,7 +4133,7 @@ pub const Object = struct { | ... | @@ -4232,7 +4133,7 @@ pub const Object = struct { |
| 4232 | }; | 4133 | }; |
| 4233 | } | 4134 | } |
| 4234 | | 4135 | |
| 4235 | fn addByRefParamAttrs( | 4136 | pub fn addByRefParamAttrs( |
| 4236 | o: *Object, | 4137 | o: *Object, |
| 4237 | attributes: *Builder.FunctionAttributes.Wip, | 4138 | attributes: *Builder.FunctionAttributes.Wip, |
| 4238 | llvm_arg_i: u32, | 4139 | llvm_arg_i: u32, |
| ... | @@ -4246,52 +4147,74 @@ pub const Object = struct { | ... | @@ -4246,52 +4147,74 @@ pub const Object = struct { |
| 4246 | if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder); | 4147 | if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder); |
| 4247 | } | 4148 | } |
| 4248 | | 4149 | |
| 4249 | fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint { | 4150 | pub fn getErrorNameTable(o: *Object) Allocator.Error!Builder.Variable.Index { |
| 4250 | return o.struct_field_map.get(.{ | 4151 | if (o.error_name_table != .none) return o.error_name_table; |
| 4251 | .struct_ty = struct_ty.toIntern(), | | |
| 4252 | .field_index = @intCast(field_index), | | |
| 4253 | }); | | |
| 4254 | } | | |
| 4255 | | 4152 | |
| 4256 | fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index { | 4153 | const name = try o.builder.strtabString("__zig_error_name_table"); |
| 4257 | const name = try o.builder.strtabString(lt_errors_fn_name); | 4154 | // TODO: Address space |
| 4258 | if (o.builder.getGlobal(name)) |llvm_fn| return llvm_fn.ptrConst(&o.builder).kind.function; | 4155 | const variable_index = try o.builder.addVariable(name, .ptr, .default); |
| 4259 | | 4156 | variable_index.setMutability(.constant, &o.builder); |
| 4260 | const zcu = pt.zcu; | 4157 | variable_index.setAlignment( |
| 4261 | const target = &zcu.root_mod.resolved_target.result; | 4158 | Type.slice_const_u8_sentinel_0.abiAlignment(o.zcu).toLlvm(), |
| 4262 | const function_index = try o.builder.addFunction( | 4159 | &o.builder, |
| 4263 | try o.builder.fnType(.i1, &.{try o.errorIntType(pt)}, .normal), | | |
| 4264 | name, | | |
| 4265 | toLlvmAddressSpace(.generic, target), | | |
| 4266 | ); | 4160 | ); |
| | 4161 | const global_index = variable_index.ptrConst(&o.builder).global; |
| | 4162 | global_index.setLinkage(.private, &o.builder); |
| | 4163 | global_index.setUnnamedAddr(.unnamed_addr, &o.builder); |
| 4267 | | 4164 | |
| 4268 | var attributes: Builder.FunctionAttributes.Wip = .{}; | 4165 | o.error_name_table = variable_index; |
| 4269 | defer attributes.deinit(&o.builder); | 4166 | return variable_index; |
| 4270 | try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer); | 4167 | } |
| 4271 | | 4168 | |
| 4272 | function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); | 4169 | pub fn getErrorsLen(o: *Object) Allocator.Error!Builder.Variable.Index { |
| 4273 | function_index.setCallConv(.fastcc, &o.builder); | 4170 | const builder = &o.builder; |
| 4274 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); | 4171 | if (o.errors_len_variable == .none) { |
| 4275 | return function_index; | 4172 | const llvm_err_int_ty = try o.errorIntType(); |
| | 4173 | const name = try builder.strtabString("__zig_errors_len"); |
| | 4174 | const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default); |
| | 4175 | variable_index.setMutability(.constant, builder); |
| | 4176 | variable_index.setAlignment(Type.errorAbiAlignment(o.zcu).toLlvm(), builder); |
| | 4177 | const global_index = variable_index.ptrConst(&o.builder).global; |
| | 4178 | global_index.setLinkage(.private, builder); |
| | 4179 | global_index.setUnnamedAddr(.unnamed_addr, builder); |
| | 4180 | o.errors_len_variable = variable_index; |
| | 4181 | } |
| | 4182 | return o.errors_len_variable; |
| 4276 | } | 4183 | } |
| 4277 | | 4184 | |
| 4278 | fn getEnumTagNameFunction(o: *Object, pt: Zcu.PerThread, enum_ty: Type) !Builder.Function.Index { | 4185 | pub fn getEnumTagNameFunction(o: *Object, enum_ty: Type) Allocator.Error!Builder.Function.Index { |
| 4279 | const zcu = pt.zcu; | 4186 | const zcu = o.zcu; |
| 4280 | const ip = &zcu.intern_pool; | 4187 | const ip = &zcu.intern_pool; |
| 4281 | const enum_type = ip.loadEnumType(enum_ty.toIntern()); | | |
| 4282 | | 4188 | |
| 4283 | const gop = try o.enum_tag_name_map.getOrPut(o.gpa, enum_ty.toIntern()); | 4189 | const gop = try o.enum_tag_name_map.getOrPut(o.gpa, enum_ty.toIntern()); |
| 4284 | if (gop.found_existing) return gop.value_ptr.ptrConst(&o.builder).kind.function; | 4190 | if (gop.found_existing) return gop.value_ptr.*; |
| 4285 | errdefer assert(o.enum_tag_name_map.remove(enum_ty.toIntern())); | 4191 | errdefer assert(o.enum_tag_name_map.remove(enum_ty.toIntern())); |
| 4286 | | | |
| 4287 | const usize_ty = try o.lowerType(pt, Type.usize); | | |
| 4288 | const ret_ty = try o.lowerType(pt, Type.slice_const_u8_sentinel_0); | | |
| 4289 | const target = &zcu.root_mod.resolved_target.result; | | |
| 4290 | const function_index = try o.builder.addFunction( | 4192 | const function_index = try o.builder.addFunction( |
| 4291 | try o.builder.fnType(ret_ty, &.{try o.lowerType(pt, Type.fromInterned(enum_type.int_tag_type))}, .normal), | 4193 | // Dummy function type; `updateEnumTagNameFunction` will replace it with the correct type. |
| 4292 | try o.builder.strtabStringFmt("__zig_tag_name_{f}", .{enum_type.name.fmt(ip)}), | 4194 | // TODO: change the builder API so we don't need to do this. |
| 4293 | toLlvmAddressSpace(.generic, target), | 4195 | try o.builder.fnType(.void, &.{}, .normal), |
| | 4196 | try o.builder.strtabStringFmt("__zig_tag_name_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}), |
| | 4197 | toLlvmAddressSpace(.generic, zcu.getTarget()), |
| 4294 | ); | 4198 | ); |
| | 4199 | gop.value_ptr.* = function_index; |
| | 4200 | try o.updateEnumTagNameFunction(enum_ty, function_index); |
| | 4201 | return function_index; |
| | 4202 | } |
| | 4203 | fn updateEnumTagNameFunction( |
| | 4204 | o: *Object, |
| | 4205 | enum_ty: Type, |
| | 4206 | function_index: Builder.Function.Index, |
| | 4207 | ) Allocator.Error!void { |
| | 4208 | const zcu = o.zcu; |
| | 4209 | const ip = &zcu.intern_pool; |
| | 4210 | const loaded_enum = ip.loadEnumType(enum_ty.toIntern()); |
| | 4211 | |
| | 4212 | const llvm_usize_ty = try o.lowerType(.usize); |
| | 4213 | const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0); |
| | 4214 | const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type)); |
| | 4215 | |
| | 4216 | function_index.ptrConst(&o.builder).global.ptr(&o.builder).type = |
| | 4217 | try o.builder.fnType(llvm_ret_ty, &.{llvm_int_ty}, .normal); |
| 4295 | | 4218 | |
| 4296 | var attributes: Builder.FunctionAttributes.Wip = .{}; | 4219 | var attributes: Builder.FunctionAttributes.Wip = .{}; |
| 4297 | defer attributes.deinit(&o.builder); | 4220 | defer attributes.deinit(&o.builder); |
| ... | @@ -4300,7 +4223,6 @@ pub const Object = struct { | ... | @@ -4300,7 +4223,6 @@ pub const Object = struct { |
| 4300 | function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); | 4223 | function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); |
| 4301 | function_index.setCallConv(.fastcc, &o.builder); | 4224 | function_index.setCallConv(.fastcc, &o.builder); |
| 4302 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); | 4225 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 4303 | gop.value_ptr.* = function_index.ptrConst(&o.builder).global; | | |
| 4304 | | 4226 | |
| 4305 | var wip = try Builder.WipFunction.init(&o.builder, .{ | 4227 | var wip = try Builder.WipFunction.init(&o.builder, .{ |
| 4306 | .function = function_index, | 4228 | .function = function_index, |
| ... | @@ -4314,33 +4236,33 @@ pub const Object = struct { | ... | @@ -4314,33 +4236,33 @@ pub const Object = struct { |
| 4314 | var wip_switch = try wip.@"switch"( | 4236 | var wip_switch = try wip.@"switch"( |
| 4315 | tag_int_value, | 4237 | tag_int_value, |
| 4316 | bad_value_block, | 4238 | bad_value_block, |
| 4317 | @intCast(enum_type.field_names.len), | 4239 | @intCast(loaded_enum.field_names.len), |
| 4318 | .none, | 4240 | .none, |
| 4319 | ); | 4241 | ); |
| 4320 | defer wip_switch.finish(&wip); | 4242 | defer wip_switch.finish(&wip); |
| 4321 | | 4243 | |
| 4322 | for (0..enum_type.field_names.len) |field_index| { | 4244 | for (0..loaded_enum.field_names.len) |field_index| { |
| 4323 | const name = try o.builder.stringNull(enum_type.field_names.get(ip)[field_index].toSlice(ip)); | 4245 | const name = try o.builder.stringNull(loaded_enum.field_names.get(ip)[field_index].toSlice(ip)); |
| 4324 | const name_init = try o.builder.stringConst(name); | 4246 | const name_init = try o.builder.stringConst(name); |
| 4325 | const name_variable_index = | 4247 | const name_variable_index = try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); |
| 4326 | try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); | | |
| 4327 | try name_variable_index.setInitializer(name_init, &o.builder); | 4248 | try name_variable_index.setInitializer(name_init, &o.builder); |
| 4328 | name_variable_index.setLinkage(.private, &o.builder); | | |
| 4329 | name_variable_index.setMutability(.constant, &o.builder); | 4249 | name_variable_index.setMutability(.constant, &o.builder); |
| 4330 | name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); | | |
| 4331 | name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); | 4250 | name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); |
| | 4251 | const name_global_index = name_variable_index.ptrConst(&o.builder).global; |
| | 4252 | name_global_index.setLinkage(.private, &o.builder); |
| | 4253 | name_global_index.setUnnamedAddr(.unnamed_addr, &o.builder); |
| 4332 | | 4254 | |
| 4333 | const name_val = try o.builder.structValue(ret_ty, &.{ | 4255 | const name_val = try o.builder.structValue(llvm_ret_ty, &.{ |
| 4334 | name_variable_index.toConst(&o.builder), | 4256 | name_global_index.toConst(), |
| 4335 | try o.builder.intConst(usize_ty, name.slice(&o.builder).?.len - 1), | 4257 | try o.builder.intConst(llvm_usize_ty, name.slice(&o.builder).?.len - 1), |
| 4336 | }); | 4258 | }); |
| 4337 | | 4259 | |
| 4338 | const return_block = try wip.block(1, "Name"); | 4260 | const return_block = try wip.block(1, "Name"); |
| 4339 | const this_tag_int_value = try o.lowerValue( | 4261 | const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, field_index)) { |
| 4340 | pt, | 4262 | .none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered |
| 4341 | (try pt.enumValueFieldIndex(enum_ty, @intCast(field_index))).toIntern(), | 4263 | else => |tag_val_ip| try o.lowerValue(tag_val_ip), |
| 4342 | ); | 4264 | }; |
| 4343 | try wip_switch.addCase(this_tag_int_value, return_block, &wip); | 4265 | try wip_switch.addCase(llvm_tag_val, return_block, &wip); |
| 4344 | | 4266 | |
| 4345 | wip.cursor = .{ .block = return_block }; | 4267 | wip.cursor = .{ .block = return_block }; |
| 4346 | _ = try wip.ret(name_val); | 4268 | _ = try wip.ret(name_val); |
| ... | @@ -4350,38 +4272,53 @@ pub const Object = struct { | ... | @@ -4350,38 +4272,53 @@ pub const Object = struct { |
| 4350 | _ = try wip.@"unreachable"(); | 4272 | _ = try wip.@"unreachable"(); |
| 4351 | | 4273 | |
| 4352 | try wip.finish(); | 4274 | try wip.finish(); |
| 4353 | return function_index; | | |
| 4354 | } | 4275 | } |
| 4355 | | 4276 | |
| 4356 | fn lazyAbiAlignment(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Alignment.Lazy { | 4277 | pub fn lazyAbiAlignment(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Alignment.Lazy { |
| 4357 | const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern()); | 4278 | const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern()); |
| 4358 | return o.lazy_abi_aligns.items[@intFromEnum(index)]; | 4279 | return o.lazy_abi_aligns.items[@intFromEnum(index)]; |
| 4359 | } | 4280 | } |
| 4360 | | 4281 | |
| | 4282 | pub fn getIsNamedEnumValueFunction(o: *Object, enum_ty: Type) Allocator.Error!Builder.Function.Index { |
| | 4283 | const zcu = o.zcu; |
| | 4284 | const ip = &zcu.intern_pool; |
| | 4285 | |
| | 4286 | const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern()); |
| | 4287 | if (gop.found_existing) return gop.value_ptr.*; |
| | 4288 | errdefer assert(o.named_enum_map.remove(enum_ty.toIntern())); |
| | 4289 | const function_index = try o.builder.addFunction( |
| | 4290 | // Dummy function type; `updateIsNamedEnumValue` will replace it with the correct type. |
| | 4291 | // TODO: change the builder API so we don't need to do this. |
| | 4292 | try o.builder.fnType(.void, &.{}, .normal), |
| | 4293 | try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}), |
| | 4294 | toLlvmAddressSpace(.generic, zcu.getTarget()), |
| | 4295 | ); |
| | 4296 | gop.value_ptr.* = function_index; |
| | 4297 | try o.updateIsNamedEnumValueFunction(enum_ty, function_index); |
| | 4298 | return function_index; |
| | 4299 | } |
| 4361 | fn updateIsNamedEnumValueFunction( | 4300 | fn updateIsNamedEnumValueFunction( |
| 4362 | o: *Object, | 4301 | o: *Object, |
| 4363 | pt: Zcu.PerThread, | | |
| 4364 | enum_ty: Type, | 4302 | enum_ty: Type, |
| 4365 | function_index: Builder.Function.Index, | 4303 | function_index: Builder.Function.Index, |
| 4366 | ) Allocator.Error!void { | 4304 | ) Allocator.Error!void { |
| 4367 | const zcu = pt.zcu; | 4305 | const zcu = o.zcu; |
| 4368 | const builder = &o.builder; | 4306 | const ip = &zcu.intern_pool; |
| 4369 | const loaded_enum = zcu.intern_pool.loadEnumType(enum_ty.toIntern()); | 4307 | const loaded_enum = ip.loadEnumType(enum_ty.toIntern()); |
| 4370 | function_index.ptrConst(builder).global.ptr(builder).type = try builder.fnType( | 4308 | |
| 4371 | .i1, | 4309 | const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type)); |
| 4372 | &.{try o.lowerType(pt, .fromInterned(loaded_enum.int_tag_type))}, | 4310 | function_index.ptrConst(&o.builder).global.ptr(&o.builder).type = |
| 4373 | .normal, | 4311 | try o.builder.fnType(.i1, &.{llvm_int_ty}, .normal); |
| 4374 | ); | | |
| 4375 | | 4312 | |
| 4376 | var attributes: Builder.FunctionAttributes.Wip = .{}; | 4313 | var attributes: Builder.FunctionAttributes.Wip = .{}; |
| 4377 | defer attributes.deinit(builder); | 4314 | defer attributes.deinit(&o.builder); |
| 4378 | try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer); | 4315 | try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer); |
| 4379 | | 4316 | |
| 4380 | function_index.setLinkage(if (o.builder.strip) .private else .internal, builder); | 4317 | function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); |
| 4381 | function_index.setCallConv(.fastcc, builder); | 4318 | function_index.setCallConv(.fastcc, &o.builder); |
| 4382 | function_index.setAttributes(try attributes.finish(builder), builder); | 4319 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 4383 | | 4320 | |
| 4384 | var wip: Builder.WipFunction = try .init(builder, .{ | 4321 | var wip: Builder.WipFunction = try .init(&o.builder, .{ |
| 4385 | .function = function_index, | 4322 | .function = function_index, |
| 4386 | .strip = true, | 4323 | .strip = true, |
| 4387 | }); | 4324 | }); |
| ... | @@ -4394,13 +4331,19 @@ pub const Object = struct { | ... | @@ -4394,13 +4331,19 @@ pub const Object = struct { |
| 4394 | var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(loaded_enum.field_names.len), .none); | 4331 | var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(loaded_enum.field_names.len), .none); |
| 4395 | defer wip_switch.finish(&wip); | 4332 | defer wip_switch.finish(&wip); |
| 4396 | | 4333 | |
| 4397 | for (0..loaded_enum.field_names.len) |field_index| { | 4334 | if (loaded_enum.field_values.len > 0) { |
| 4398 | const this_tag_int_value = try o.lowerValue( | 4335 | for (loaded_enum.field_values.get(ip)) |tag_val_ip| { |
| 4399 | pt, | 4336 | const llvm_tag_val = try o.lowerValue(tag_val_ip); |
| 4400 | (try pt.enumValueFieldIndex(enum_ty, @intCast(field_index))).toIntern(), | 4337 | try wip_switch.addCase(llvm_tag_val, named_block, &wip); |
| 4401 | ); | 4338 | } |
| 4402 | try wip_switch.addCase(this_tag_int_value, named_block, &wip); | 4339 | } else { |
| | 4340 | // Auto-numbered. |
| | 4341 | for (0..loaded_enum.field_names.len) |field_index| { |
| | 4342 | const llvm_tag_val = try o.builder.intConst(llvm_int_ty, field_index); |
| | 4343 | try wip_switch.addCase(llvm_tag_val, named_block, &wip); |
| | 4344 | } |
| 4403 | } | 4345 | } |
| | 4346 | |
| 4404 | wip.cursor = .{ .block = named_block }; | 4347 | wip.cursor = .{ .block = named_block }; |
| 4405 | _ = try wip.ret(.true); | 4348 | _ = try wip.ret(.true); |
| 4406 | | 4349 | |
| ... | @@ -4409,8167 +4352,306 @@ pub const Object = struct { | ... | @@ -4409,8167 +4352,306 @@ pub const Object = struct { |
| 4409 | | 4352 | |
| 4410 | try wip.finish(); | 4353 | try wip.finish(); |
| 4411 | } | 4354 | } |
| 4412 | }; | | |
| 4413 | | | |
| 4414 | pub const NavGen = struct { | | |
| 4415 | object: *Object, | | |
| 4416 | nav_index: InternPool.Nav.Index, | | |
| 4417 | pt: Zcu.PerThread, | | |
| 4418 | err_msg: ?*Zcu.ErrorMsg, | | |
| 4419 | | | |
| 4420 | fn ownerModule(ng: NavGen) *Package.Module { | | |
| 4421 | return ng.pt.zcu.navFileScope(ng.nav_index).mod.?; | | |
| 4422 | } | | |
| 4423 | | | |
| 4424 | fn todo(ng: *NavGen, comptime format: []const u8, args: anytype) Error { | | |
| 4425 | @branchHint(.cold); | | |
| 4426 | assert(ng.err_msg == null); | | |
| 4427 | const o = ng.object; | | |
| 4428 | const gpa = o.gpa; | | |
| 4429 | const src_loc = ng.pt.zcu.navSrcLoc(ng.nav_index); | | |
| 4430 | ng.err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "TODO (LLVM): " ++ format, args); | | |
| 4431 | return error.CodegenFail; | | |
| 4432 | } | | |
| 4433 | | | |
| 4434 | fn genDecl(ng: *NavGen) !void { | | |
| 4435 | const o = ng.object; | | |
| 4436 | const pt = ng.pt; | | |
| 4437 | const zcu = pt.zcu; | | |
| 4438 | const ip = &zcu.intern_pool; | | |
| 4439 | const nav_index = ng.nav_index; | | |
| 4440 | const nav = ip.getNav(nav_index); | | |
| 4441 | const resolved = nav.resolved.?; | | |
| 4442 | | 4355 | |
| 4443 | const lib_name, const linkage, const visibility: Builder.Visibility, const is_dll_import, const init_val, const owner_nav = switch (ip.indexToKey(resolved.value)) { | 4356 | pub fn getLibcFunction( |
| 4444 | else => .{ .none, .internal, .default, false, resolved.value, nav_index }, | 4357 | o: *Object, |
| 4445 | .@"extern" => |e| .{ e.lib_name, e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import, .none, e.owner_nav }, | 4358 | fn_name: Builder.StrtabString, |
| | 4359 | param_types: []const Builder.Type, |
| | 4360 | return_type: Builder.Type, |
| | 4361 | ) Allocator.Error!Builder.Function.Index { |
| | 4362 | if (o.builder.getGlobal(fn_name)) |global| return switch (global.ptrConst(&o.builder).kind) { |
| | 4363 | .alias => |alias| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function, |
| | 4364 | .function => |function| function, |
| | 4365 | .variable, .replaced => unreachable, |
| 4446 | }; | 4366 | }; |
| 4447 | const ty: Type = .fromInterned(nav.resolved.?.type); | 4367 | return o.builder.addFunction( |
| 4448 | | 4368 | try o.builder.fnType(return_type, param_types, .normal), |
| 4449 | if (linkage != .internal and ip.isFunctionType(ty.toIntern())) { | 4369 | fn_name, |
| 4450 | const function_index = try o.resolveLlvmFunction(pt, owner_nav); | 4370 | toLlvmAddressSpace(.generic, o.zcu.getTarget()), |
| 4451 | // Add parameter attributes which weren't set by `resolveLlvmFunction` | 4371 | ); |
| 4452 | const fn_info = zcu.typeToFunc(ty).?; | 4372 | } |
| 4453 | var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); | 4373 | }; |
| 4454 | defer attributes.deinit(&o.builder); | | |
| 4455 | var it = iterateParamTypes(o, pt, fn_info); | | |
| 4456 | if (firstParamSRet(fn_info, zcu, zcu.getTarget())) it.llvm_index += 1; | | |
| 4457 | if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) it.llvm_index += 1; | | |
| 4458 | while (try it.next()) |lowering| switch (lowering) { | | |
| 4459 | .byval => { | | |
| 4460 | const param_index = it.zig_index - 1; | | |
| 4461 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); | | |
| 4462 | if (!isByRef(param_ty, zcu)) { | | |
| 4463 | try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1); | | |
| 4464 | } | | |
| 4465 | }, | | |
| 4466 | .byref => { | | |
| 4467 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | | |
| 4468 | const param_llvm_ty = try o.lowerType(pt, param_ty); | | |
| 4469 | const alignment = param_ty.abiAlignment(zcu); | | |
| 4470 | try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty); | | |
| 4471 | }, | | |
| 4472 | .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), | | |
| 4473 | // No attributes needed for these. | | |
| 4474 | .no_bits, | | |
| 4475 | .abi_sized_int, | | |
| 4476 | .multiple_llvm_types, | | |
| 4477 | .float_array, | | |
| 4478 | .i32_array, | | |
| 4479 | .i64_array, | | |
| 4480 | => continue, | | |
| 4481 | | | |
| 4482 | .slice => unreachable, // extern functions do not support slice types. | | |
| 4483 | }; | | |
| 4484 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); | | |
| 4485 | } else { | | |
| 4486 | const variable_index = try o.resolveGlobalNav(pt, nav_index); | | |
| 4487 | variable_index.setAlignment(zcu.navAlignment(nav_index).toLlvm(), &o.builder); | | |
| 4488 | if (resolved.@"linksection".toSlice(ip)) |section| | | |
| 4489 | variable_index.setSection(try o.builder.string(section), &o.builder); | | |
| 4490 | if (resolved.@"const") variable_index.setMutability(.constant, &o.builder); | | |
| 4491 | try variable_index.setInitializer(switch (init_val) { | | |
| 4492 | .none => .no_init, | | |
| 4493 | else => try o.lowerValue(pt, init_val), | | |
| 4494 | }, &o.builder); | | |
| 4495 | variable_index.setVisibility(visibility, &o.builder); | | |
| 4496 | | | |
| 4497 | const file_scope = zcu.navFileScopeIndex(nav_index); | | |
| 4498 | const mod = zcu.fileByIndex(file_scope).mod.?; | | |
| 4499 | if (resolved.@"threadlocal" and !mod.single_threaded) | | |
| 4500 | variable_index.setThreadLocal(.generaldynamic, &o.builder); | | |
| 4501 | | 4374 | |
| 4502 | const line_number = zcu.navSrcLine(nav_index) + 1; | 4375 | const CallingConventionInfo = struct { |
| | 4376 | /// The LLVM calling convention to use. |
| | 4377 | llvm_cc: Builder.CallConv, |
| | 4378 | /// Whether to use an `alignstack` attribute to forcibly re-align the stack pointer in the function's prologue. |
| | 4379 | align_stack: bool, |
| | 4380 | /// Whether the function needs a `naked` attribute. |
| | 4381 | naked: bool, |
| | 4382 | /// How many leading parameters to apply the `inreg` attribute to. |
| | 4383 | inreg_param_count: u2 = 0, |
| | 4384 | }; |
| 4503 | | 4385 | |
| 4504 | if (!mod.strip) { | 4386 | pub fn toLlvmCallConv(cc: std.builtin.CallingConvention, target: *const std.Target) ?CallingConventionInfo { |
| 4505 | const debug_file = try o.getDebugFile(pt, file_scope); | 4387 | const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null; |
| 4506 | | 4388 | const incoming_stack_alignment: ?u64, const register_params: u2 = switch (cc) { |
| 4507 | const debug_global_var = try o.builder.debugGlobalVar( | 4389 | inline else => |pl| switch (@TypeOf(pl)) { |
| 4508 | try o.builder.metadataString(nav.name.toSlice(ip)), // Name | 4390 | void => .{ null, 0 }, |
| 4509 | try o.builder.metadataStringFromStrtabString(variable_index.name(&o.builder)), // Linkage name | 4391 | std.builtin.CallingConvention.ArcInterruptOptions, |
| 4510 | debug_file, // File | 4392 | std.builtin.CallingConvention.ArmInterruptOptions, |
| 4511 | debug_file, // Scope | 4393 | std.builtin.CallingConvention.RiscvInterruptOptions, |
| 4512 | line_number, | 4394 | std.builtin.CallingConvention.ShInterruptOptions, |
| 4513 | try o.getDebugType(pt, ty), | 4395 | std.builtin.CallingConvention.MicroblazeInterruptOptions, |
| 4514 | variable_index, | 4396 | std.builtin.CallingConvention.MipsInterruptOptions, |
| 4515 | .{ .local = linkage == .internal }, | 4397 | std.builtin.CallingConvention.CommonOptions, |
| 4516 | ); | 4398 | => .{ pl.incoming_stack_alignment, 0 }, |
| 4517 | | 4399 | std.builtin.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params }, |
| 4518 | const debug_expression = try o.builder.debugExpression(&.{}); | 4400 | else => @compileError("TODO: toLlvmCallConv" ++ @tagName(pl)), |
| 4519 | | | |
| 4520 | const debug_global_var_expression = try o.builder.debugGlobalVarExpression( | | |
| 4521 | debug_global_var, | | |
| 4522 | debug_expression, | | |
| 4523 | ); | | |
| 4524 | | | |
| 4525 | variable_index.setGlobalVariableExpression(debug_global_var_expression, &o.builder); | | |
| 4526 | try o.debug_globals.append(o.gpa, debug_global_var_expression); | | |
| 4527 | } | | |
| 4528 | } | | |
| 4529 | | | |
| 4530 | switch (linkage) { | | |
| 4531 | .internal => {}, | | |
| 4532 | .strong, .weak => { | | |
| 4533 | const global_index = o.nav_map.get(nav_index).?; | | |
| 4534 | | | |
| 4535 | const decl_name = decl_name: { | | |
| 4536 | if (zcu.getTarget().cpu.arch.isWasm() and ty.zigTypeTag(zcu) == .@"fn") { | | |
| 4537 | if (lib_name.toSlice(ip)) |lib_name_slice| { | | |
| 4538 | if (!std.mem.eql(u8, lib_name_slice, "c")) { | | |
| 4539 | break :decl_name try o.builder.strtabStringFmt("{f}|{s}", .{ nav.name.fmt(ip), lib_name_slice }); | | |
| 4540 | } | | |
| 4541 | } | | |
| 4542 | } | | |
| 4543 | break :decl_name try o.builder.strtabString(nav.name.toSlice(ip)); | | |
| 4544 | }; | | |
| 4545 | | | |
| 4546 | if (o.builder.getGlobal(decl_name)) |other_global| { | | |
| 4547 | if (other_global != global_index) { | | |
| 4548 | // Another global already has this name; just use it in place of this global. | | |
| 4549 | try global_index.replace(other_global, &o.builder); | | |
| 4550 | return; | | |
| 4551 | } | | |
| 4552 | } | | |
| 4553 | | | |
| 4554 | try global_index.rename(decl_name, &o.builder); | | |
| 4555 | global_index.setUnnamedAddr(.default, &o.builder); | | |
| 4556 | if (is_dll_import) { | | |
| 4557 | global_index.setDllStorageClass(.dllimport, &o.builder); | | |
| 4558 | } else if (zcu.comp.config.dll_export_fns) { | | |
| 4559 | global_index.setDllStorageClass(.default, &o.builder); | | |
| 4560 | } | | |
| 4561 | | | |
| 4562 | global_index.setLinkage(switch (linkage) { | | |
| 4563 | .internal => unreachable, | | |
| 4564 | .strong => .external, | | |
| 4565 | .weak => .extern_weak, | | |
| 4566 | .link_once => unreachable, | | |
| 4567 | }, &o.builder); | | |
| 4568 | global_index.setVisibility(visibility, &o.builder); | | |
| 4569 | }, | | |
| 4570 | .link_once => unreachable, | | |
| 4571 | } | | |
| 4572 | } | | |
| 4573 | }; | | |
| 4574 | | | |
| 4575 | pub const FuncGen = struct { | | |
| 4576 | gpa: Allocator, | | |
| 4577 | ng: *NavGen, | | |
| 4578 | air: Air, | | |
| 4579 | liveness: Air.Liveness, | | |
| 4580 | wip: Builder.WipFunction, | | |
| 4581 | is_naked: bool, | | |
| 4582 | fuzz: ?Fuzz, | | |
| 4583 | | | |
| 4584 | file: Builder.Metadata, | | |
| 4585 | scope: Builder.Metadata, | | |
| 4586 | | | |
| 4587 | inlined_at: Builder.Metadata.Optional = .none, | | |
| 4588 | | | |
| 4589 | base_line: u32, | | |
| 4590 | prev_dbg_line: c_uint, | | |
| 4591 | prev_dbg_column: c_uint, | | |
| 4592 | | | |
| 4593 | /// This stores the LLVM values used in a function, such that they can be referred to | | |
| 4594 | /// in other instructions. This table is cleared before every function is generated. | | |
| 4595 | func_inst_table: std.AutoHashMapUnmanaged(Air.Inst.Ref, Builder.Value), | | |
| 4596 | | | |
| 4597 | /// If the return type is sret, this is the result pointer. Otherwise null. | | |
| 4598 | /// Note that this can disagree with isByRef for the return type in the case | | |
| 4599 | /// of C ABI functions. | | |
| 4600 | ret_ptr: Builder.Value, | | |
| 4601 | /// Any function that needs to perform Valgrind client requests needs an array alloca | | |
| 4602 | /// instruction, however a maximum of one per function is needed. | | |
| 4603 | valgrind_client_request_array: Builder.Value = .none, | | |
| 4604 | /// These fields are used to refer to the LLVM value of the function parameters | | |
| 4605 | /// in an Arg instruction. | | |
| 4606 | /// This list may be shorter than the list according to the zig type system; | | |
| 4607 | /// it omits 0-bit types. If the function uses sret as the first parameter, | | |
| 4608 | /// this slice does not include it. | | |
| 4609 | args: []const Builder.Value, | | |
| 4610 | arg_index: u32, | | |
| 4611 | arg_inline_index: u32, | | |
| 4612 | | | |
| 4613 | err_ret_trace: Builder.Value = .none, | | |
| 4614 | | | |
| 4615 | /// This data structure is used to implement breaking to blocks. | | |
| 4616 | blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { | | |
| 4617 | parent_bb: Builder.Function.Block.Index, | | |
| 4618 | breaks: *BreakList, | | |
| 4619 | }), | | |
| 4620 | | | |
| 4621 | /// Maps `loop` instructions to the bb to branch to to repeat the loop. | | |
| 4622 | loops: std.AutoHashMapUnmanaged(Air.Inst.Index, Builder.Function.Block.Index), | | |
| 4623 | | | |
| 4624 | /// Maps `loop_switch_br` instructions to the information required to lower | | |
| 4625 | /// dispatches (`switch_dispatch` instructions). | | |
| 4626 | switch_dispatch_info: std.AutoHashMapUnmanaged(Air.Inst.Index, SwitchDispatchInfo), | | |
| 4627 | | | |
| 4628 | sync_scope: Builder.SyncScope, | | |
| 4629 | | | |
| 4630 | disable_intrinsics: bool, | | |
| 4631 | | | |
| 4632 | /// Have we seen loads or stores involving `allowzero` pointers? | | |
| 4633 | allowzero_access: bool = false, | | |
| 4634 | | | |
| 4635 | fn maybeMarkAllowZeroAccess(self: *FuncGen, info: InternPool.Key.PtrType) void { | | |
| 4636 | // LLVM already considers null pointers to be valid in non-generic address spaces, so avoid | | |
| 4637 | // pessimizing optimization for functions with accesses to such pointers. | | |
| 4638 | if (info.flags.address_space == .generic and info.flags.is_allowzero) self.allowzero_access = true; | | |
| 4639 | } | | |
| 4640 | | | |
| 4641 | const Fuzz = struct { | | |
| 4642 | counters_variable: Builder.Variable.Index, | | |
| 4643 | pcs: std.ArrayList(Builder.Constant), | | |
| 4644 | | | |
| 4645 | fn deinit(f: *Fuzz, gpa: Allocator) void { | | |
| 4646 | f.pcs.deinit(gpa); | | |
| 4647 | f.* = undefined; | | |
| 4648 | } | | |
| 4649 | }; | | |
| 4650 | | | |
| 4651 | const SwitchDispatchInfo = struct { | | |
| 4652 | /// These are the blocks corresponding to each switch case. | | |
| 4653 | /// The final element corresponds to the `else` case. | | |
| 4654 | /// Slices allocated into `gpa`. | | |
| 4655 | case_blocks: []Builder.Function.Block.Index, | | |
| 4656 | /// This is `.none` if `jmp_table` is set, since we won't use a `switch` instruction to dispatch. | | |
| 4657 | switch_weights: Builder.Function.Instruction.BrCond.Weights, | | |
| 4658 | /// If not `null`, we have manually constructed a jump table to reach the desired block. | | |
| 4659 | /// `table` can be used if the value is between `min` and `max` inclusive. | | |
| 4660 | /// We perform this lowering manually to avoid some questionable behavior from LLVM. | | |
| 4661 | /// See `airSwitchBr` for details. | | |
| 4662 | jmp_table: ?JmpTable, | | |
| 4663 | | | |
| 4664 | const JmpTable = struct { | | |
| 4665 | min: Builder.Constant, | | |
| 4666 | max: Builder.Constant, | | |
| 4667 | in_bounds_hint: enum { none, unpredictable, likely, unlikely }, | | |
| 4668 | /// Pointer to the jump table itself, to be used with `indirectbr`. | | |
| 4669 | /// The index into the jump table is the dispatch condition minus `min`. | | |
| 4670 | /// The table values are `blockaddress` constants corresponding to blocks in `case_blocks`. | | |
| 4671 | table: Builder.Constant, | | |
| 4672 | /// `true` if `table` conatins a reference to the `else` block. | | |
| 4673 | /// In this case, the `indirectbr` must include the `else` block in its target list. | | |
| 4674 | table_includes_else: bool, | | |
| 4675 | }; | | |
| 4676 | }; | | |
| 4677 | | | |
| 4678 | const BreakList = union { | | |
| 4679 | list: std.MultiArrayList(struct { | | |
| 4680 | bb: Builder.Function.Block.Index, | | |
| 4681 | val: Builder.Value, | | |
| 4682 | }), | | |
| 4683 | len: usize, | | |
| 4684 | }; | | |
| 4685 | | | |
| 4686 | fn deinit(self: *FuncGen) void { | | |
| 4687 | const gpa = self.gpa; | | |
| 4688 | if (self.fuzz) |*f| f.deinit(self.gpa); | | |
| 4689 | self.wip.deinit(); | | |
| 4690 | self.func_inst_table.deinit(gpa); | | |
| 4691 | self.blocks.deinit(gpa); | | |
| 4692 | self.loops.deinit(gpa); | | |
| 4693 | var it = self.switch_dispatch_info.valueIterator(); | | |
| 4694 | while (it.next()) |info| { | | |
| 4695 | self.gpa.free(info.case_blocks); | | |
| 4696 | } | | |
| 4697 | self.switch_dispatch_info.deinit(gpa); | | |
| 4698 | } | | |
| 4699 | | | |
| 4700 | fn todo(self: *FuncGen, comptime format: []const u8, args: anytype) Error { | | |
| 4701 | @branchHint(.cold); | | |
| 4702 | return self.ng.todo(format, args); | | |
| 4703 | } | | |
| 4704 | | | |
| 4705 | fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value { | | |
| 4706 | const gpa = self.gpa; | | |
| 4707 | const gop = try self.func_inst_table.getOrPut(gpa, inst); | | |
| 4708 | if (gop.found_existing) return gop.value_ptr.*; | | |
| 4709 | | | |
| 4710 | const llvm_val = try self.resolveValue((try self.air.value(inst, self.ng.pt)).?); | | |
| 4711 | gop.value_ptr.* = llvm_val.toValue(); | | |
| 4712 | return llvm_val.toValue(); | | |
| 4713 | } | | |
| 4714 | | | |
| 4715 | fn resolveValue(self: *FuncGen, val: Value) Error!Builder.Constant { | | |
| 4716 | const o = self.ng.object; | | |
| 4717 | const pt = self.ng.pt; | | |
| 4718 | const zcu = pt.zcu; | | |
| 4719 | const ty = val.typeOf(zcu); | | |
| 4720 | const llvm_val = try o.lowerValue(pt, val.toIntern()); | | |
| 4721 | if (!isByRef(ty, zcu)) return llvm_val; | | |
| 4722 | | | |
| 4723 | // We have an LLVM value but we need to create a global constant and | | |
| 4724 | // set the value as its initializer, and then return a pointer to the global. | | |
| 4725 | const target = zcu.getTarget(); | | |
| 4726 | const variable_index = try o.builder.addVariable( | | |
| 4727 | .empty, | | |
| 4728 | llvm_val.typeOf(&o.builder), | | |
| 4729 | toLlvmGlobalAddressSpace(.generic, target), | | |
| 4730 | ); | | |
| 4731 | try variable_index.setInitializer(llvm_val, &o.builder); | | |
| 4732 | variable_index.setLinkage(.private, &o.builder); | | |
| 4733 | variable_index.setMutability(.constant, &o.builder); | | |
| 4734 | variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); | | |
| 4735 | variable_index.setAlignment(ty.abiAlignment(zcu).toLlvm(), &o.builder); | | |
| 4736 | return o.builder.convConst( | | |
| 4737 | variable_index.toConst(&o.builder), | | |
| 4738 | try o.builder.ptrType(toLlvmAddressSpace(.generic, target)), | | |
| 4739 | ); | | |
| 4740 | } | | |
| 4741 | | | |
| 4742 | fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) Error!void { | | |
| 4743 | const o = self.ng.object; | | |
| 4744 | const zcu = self.ng.pt.zcu; | | |
| 4745 | const ip = &zcu.intern_pool; | | |
| 4746 | const air_tags = self.air.instructions.items(.tag); | | |
| 4747 | switch (coverage_point) { | | |
| 4748 | .none => {}, | | |
| 4749 | .poi => if (self.fuzz) |*fuzz| { | | |
| 4750 | const poi_index = fuzz.pcs.items.len; | | |
| 4751 | const base_ptr = fuzz.counters_variable.toValue(&o.builder); | | |
| 4752 | const ptr = if (poi_index == 0) base_ptr else try self.wip.gep(.inbounds, .i8, base_ptr, &.{ | | |
| 4753 | try o.builder.intValue(.i32, poi_index), | | |
| 4754 | }, ""); | | |
| 4755 | const one = try o.builder.intValue(.i8, 1); | | |
| 4756 | _ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, ""); | | |
| 4757 | | | |
| 4758 | // LLVM does not allow blockaddress on the entry block. | | |
| 4759 | const pc = if (self.wip.cursor.block == .entry) | | |
| 4760 | self.wip.function.toConst(&o.builder) | | |
| 4761 | else | | |
| 4762 | try o.builder.blockAddrConst(self.wip.function, self.wip.cursor.block); | | |
| 4763 | const gpa = self.gpa; | | |
| 4764 | try fuzz.pcs.append(gpa, pc); | | |
| 4765 | }, | | |
| 4766 | } | | |
| 4767 | for (body, 0..) |inst, i| { | | |
| 4768 | if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue; | | |
| 4769 | | | |
| 4770 | const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) { | | |
| 4771 | // zig fmt: off | | |
| 4772 | | | |
| 4773 | // No "scalarize" legalizations are enabled, so these instructions never appear. | | |
| 4774 | .legalize_vec_elem_val => unreachable, | | |
| 4775 | .legalize_vec_store_elem => unreachable, | | |
| 4776 | // No soft float legalizations are enabled. | | |
| 4777 | .legalize_compiler_rt_call => unreachable, | | |
| 4778 | | | |
| 4779 | .add => try self.airAdd(inst, .normal), | | |
| 4780 | .add_optimized => try self.airAdd(inst, .fast), | | |
| 4781 | .add_wrap => try self.airAddWrap(inst), | | |
| 4782 | .add_sat => try self.airAddSat(inst), | | |
| 4783 | | | |
| 4784 | .sub => try self.airSub(inst, .normal), | | |
| 4785 | .sub_optimized => try self.airSub(inst, .fast), | | |
| 4786 | .sub_wrap => try self.airSubWrap(inst), | | |
| 4787 | .sub_sat => try self.airSubSat(inst), | | |
| 4788 | | | |
| 4789 | .mul => try self.airMul(inst, .normal), | | |
| 4790 | .mul_optimized => try self.airMul(inst, .fast), | | |
| 4791 | .mul_wrap => try self.airMulWrap(inst), | | |
| 4792 | .mul_sat => try self.airMulSat(inst), | | |
| 4793 | | | |
| 4794 | .add_safe => try self.airSafeArithmetic(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"), | | |
| 4795 | .sub_safe => try self.airSafeArithmetic(inst, .@"ssub.with.overflow", .@"usub.with.overflow"), | | |
| 4796 | .mul_safe => try self.airSafeArithmetic(inst, .@"smul.with.overflow", .@"umul.with.overflow"), | | |
| 4797 | | | |
| 4798 | .div_float => try self.airDivFloat(inst, .normal), | | |
| 4799 | .div_trunc => try self.airDivTrunc(inst, .normal), | | |
| 4800 | .div_floor => try self.airDivFloor(inst, .normal), | | |
| 4801 | .div_exact => try self.airDivExact(inst, .normal), | | |
| 4802 | .rem => try self.airRem(inst, .normal), | | |
| 4803 | .mod => try self.airMod(inst, .normal), | | |
| 4804 | .abs => try self.airAbs(inst), | | |
| 4805 | .ptr_add => try self.airPtrAdd(inst), | | |
| 4806 | .ptr_sub => try self.airPtrSub(inst), | | |
| 4807 | .shl => try self.airShl(inst), | | |
| 4808 | .shl_sat => try self.airShlSat(inst), | | |
| 4809 | .shl_exact => try self.airShlExact(inst), | | |
| 4810 | .min => try self.airMin(inst), | | |
| 4811 | .max => try self.airMax(inst), | | |
| 4812 | .slice => try self.airSlice(inst), | | |
| 4813 | .mul_add => try self.airMulAdd(inst), | | |
| 4814 | | | |
| 4815 | .div_float_optimized => try self.airDivFloat(inst, .fast), | | |
| 4816 | .div_trunc_optimized => try self.airDivTrunc(inst, .fast), | | |
| 4817 | .div_floor_optimized => try self.airDivFloor(inst, .fast), | | |
| 4818 | .div_exact_optimized => try self.airDivExact(inst, .fast), | | |
| 4819 | .rem_optimized => try self.airRem(inst, .fast), | | |
| 4820 | .mod_optimized => try self.airMod(inst, .fast), | | |
| 4821 | | | |
| 4822 | .add_with_overflow => try self.airOverflow(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"), | | |
| 4823 | .sub_with_overflow => try self.airOverflow(inst, .@"ssub.with.overflow", .@"usub.with.overflow"), | | |
| 4824 | .mul_with_overflow => try self.airOverflow(inst, .@"smul.with.overflow", .@"umul.with.overflow"), | | |
| 4825 | .shl_with_overflow => try self.airShlWithOverflow(inst), | | |
| 4826 | | | |
| 4827 | .bit_and, .bool_and => try self.airAnd(inst), | | |
| 4828 | .bit_or, .bool_or => try self.airOr(inst), | | |
| 4829 | .xor => try self.airXor(inst), | | |
| 4830 | .shr => try self.airShr(inst, false), | | |
| 4831 | .shr_exact => try self.airShr(inst, true), | | |
| 4832 | | | |
| 4833 | .sqrt => try self.airUnaryOp(inst, .sqrt), | | |
| 4834 | .sin => try self.airUnaryOp(inst, .sin), | | |
| 4835 | .cos => try self.airUnaryOp(inst, .cos), | | |
| 4836 | .tan => try self.airUnaryOp(inst, .tan), | | |
| 4837 | .exp => try self.airUnaryOp(inst, .exp), | | |
| 4838 | .exp2 => try self.airUnaryOp(inst, .exp2), | | |
| 4839 | .log => try self.airUnaryOp(inst, .log), | | |
| 4840 | .log2 => try self.airUnaryOp(inst, .log2), | | |
| 4841 | .log10 => try self.airUnaryOp(inst, .log10), | | |
| 4842 | .floor => try self.airUnaryOp(inst, .floor), | | |
| 4843 | .ceil => try self.airUnaryOp(inst, .ceil), | | |
| 4844 | .round => try self.airUnaryOp(inst, .round), | | |
| 4845 | .trunc_float => try self.airUnaryOp(inst, .trunc), | | |
| 4846 | | | |
| 4847 | .neg => try self.airNeg(inst, .normal), | | |
| 4848 | .neg_optimized => try self.airNeg(inst, .fast), | | |
| 4849 | | | |
| 4850 | .cmp_eq => try self.airCmp(inst, .eq, .normal), | | |
| 4851 | .cmp_gt => try self.airCmp(inst, .gt, .normal), | | |
| 4852 | .cmp_gte => try self.airCmp(inst, .gte, .normal), | | |
| 4853 | .cmp_lt => try self.airCmp(inst, .lt, .normal), | | |
| 4854 | .cmp_lte => try self.airCmp(inst, .lte, .normal), | | |
| 4855 | .cmp_neq => try self.airCmp(inst, .neq, .normal), | | |
| 4856 | | | |
| 4857 | .cmp_eq_optimized => try self.airCmp(inst, .eq, .fast), | | |
| 4858 | .cmp_gt_optimized => try self.airCmp(inst, .gt, .fast), | | |
| 4859 | .cmp_gte_optimized => try self.airCmp(inst, .gte, .fast), | | |
| 4860 | .cmp_lt_optimized => try self.airCmp(inst, .lt, .fast), | | |
| 4861 | .cmp_lte_optimized => try self.airCmp(inst, .lte, .fast), | | |
| 4862 | .cmp_neq_optimized => try self.airCmp(inst, .neq, .fast), | | |
| 4863 | | | |
| 4864 | .cmp_vector => try self.airCmpVector(inst, .normal), | | |
| 4865 | .cmp_vector_optimized => try self.airCmpVector(inst, .fast), | | |
| 4866 | .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), | | |
| 4867 | | | |
| 4868 | .is_non_null => try self.airIsNonNull(inst, false, .ne), | | |
| 4869 | .is_non_null_ptr => try self.airIsNonNull(inst, true , .ne), | | |
| 4870 | .is_null => try self.airIsNonNull(inst, false, .eq), | | |
| 4871 | .is_null_ptr => try self.airIsNonNull(inst, true , .eq), | | |
| 4872 | | | |
| 4873 | .is_non_err => try self.airIsErr(inst, .eq, false), | | |
| 4874 | .is_non_err_ptr => try self.airIsErr(inst, .eq, true), | | |
| 4875 | .is_err => try self.airIsErr(inst, .ne, false), | | |
| 4876 | .is_err_ptr => try self.airIsErr(inst, .ne, true), | | |
| 4877 | | | |
| 4878 | .alloc => try self.airAlloc(inst), | | |
| 4879 | .ret_ptr => try self.airRetPtr(inst), | | |
| 4880 | .arg => try self.airArg(inst), | | |
| 4881 | .bitcast => try self.airBitCast(inst), | | |
| 4882 | .breakpoint => try self.airBreakpoint(inst), | | |
| 4883 | .ret_addr => try self.airRetAddr(inst), | | |
| 4884 | .frame_addr => try self.airFrameAddress(inst), | | |
| 4885 | .@"try" => try self.airTry(inst, false), | | |
| 4886 | .try_cold => try self.airTry(inst, true), | | |
| 4887 | .try_ptr => try self.airTryPtr(inst, false), | | |
| 4888 | .try_ptr_cold => try self.airTryPtr(inst, true), | | |
| 4889 | .intcast => try self.airIntCast(inst, false), | | |
| 4890 | .intcast_safe => try self.airIntCast(inst, true), | | |
| 4891 | .trunc => try self.airTrunc(inst), | | |
| 4892 | .fptrunc => try self.airFptrunc(inst), | | |
| 4893 | .fpext => try self.airFpext(inst), | | |
| 4894 | .load => try self.airLoad(inst), | | |
| 4895 | .not => try self.airNot(inst), | | |
| 4896 | .store => try self.airStore(inst, false), | | |
| 4897 | .store_safe => try self.airStore(inst, true), | | |
| 4898 | .assembly => try self.airAssembly(inst), | | |
| 4899 | .slice_ptr => try self.airSliceField(inst, 0), | | |
| 4900 | .slice_len => try self.airSliceField(inst, 1), | | |
| 4901 | | | |
| 4902 | .ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0), | | |
| 4903 | .ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1), | | |
| 4904 | | | |
| 4905 | .int_from_float => try self.airIntFromFloat(inst, .normal), | | |
| 4906 | .int_from_float_optimized => try self.airIntFromFloat(inst, .fast), | | |
| 4907 | .int_from_float_safe => unreachable, // handled by `legalizeFeatures` | | |
| 4908 | .int_from_float_optimized_safe => unreachable, // handled by `legalizeFeatures` | | |
| 4909 | | | |
| 4910 | .array_to_slice => try self.airArrayToSlice(inst), | | |
| 4911 | .float_from_int => try self.airFloatFromInt(inst), | | |
| 4912 | .cmpxchg_weak => try self.airCmpxchg(inst, .weak), | | |
| 4913 | .cmpxchg_strong => try self.airCmpxchg(inst, .strong), | | |
| 4914 | .atomic_rmw => try self.airAtomicRmw(inst), | | |
| 4915 | .atomic_load => try self.airAtomicLoad(inst), | | |
| 4916 | .memset => try self.airMemset(inst, false), | | |
| 4917 | .memset_safe => try self.airMemset(inst, true), | | |
| 4918 | .memcpy => try self.airMemcpy(inst), | | |
| 4919 | .memmove => try self.airMemmove(inst), | | |
| 4920 | .set_union_tag => try self.airSetUnionTag(inst), | | |
| 4921 | .get_union_tag => try self.airGetUnionTag(inst), | | |
| 4922 | .clz => try self.airClzCtz(inst, .ctlz), | | |
| 4923 | .ctz => try self.airClzCtz(inst, .cttz), | | |
| 4924 | .popcount => try self.airBitOp(inst, .ctpop), | | |
| 4925 | .byte_swap => try self.airByteSwap(inst), | | |
| 4926 | .bit_reverse => try self.airBitOp(inst, .bitreverse), | | |
| 4927 | .tag_name => try self.airTagName(inst), | | |
| 4928 | .error_name => try self.airErrorName(inst), | | |
| 4929 | .splat => try self.airSplat(inst), | | |
| 4930 | .select => try self.airSelect(inst), | | |
| 4931 | .shuffle_one => try self.airShuffleOne(inst), | | |
| 4932 | .shuffle_two => try self.airShuffleTwo(inst), | | |
| 4933 | .aggregate_init => try self.airAggregateInit(inst), | | |
| 4934 | .union_init => try self.airUnionInit(inst), | | |
| 4935 | .prefetch => try self.airPrefetch(inst), | | |
| 4936 | .addrspace_cast => try self.airAddrSpaceCast(inst), | | |
| 4937 | | | |
| 4938 | .is_named_enum_value => try self.airIsNamedEnumValue(inst), | | |
| 4939 | .error_set_has_value => try self.airErrorSetHasValue(inst), | | |
| 4940 | | | |
| 4941 | .reduce => try self.airReduce(inst, .normal), | | |
| 4942 | .reduce_optimized => try self.airReduce(inst, .fast), | | |
| 4943 | | | |
| 4944 | .atomic_store_unordered => try self.airAtomicStore(inst, .unordered), | | |
| 4945 | .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic), | | |
| 4946 | .atomic_store_release => try self.airAtomicStore(inst, .release), | | |
| 4947 | .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst), | | |
| 4948 | | | |
| 4949 | .struct_field_ptr => try self.airStructFieldPtr(inst), | | |
| 4950 | .struct_field_val => try self.airStructFieldVal(inst), | | |
| 4951 | | | |
| 4952 | .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0), | | |
| 4953 | .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1), | | |
| 4954 | .struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2), | | |
| 4955 | .struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3), | | |
| 4956 | | | |
| 4957 | .field_parent_ptr => try self.airFieldParentPtr(inst), | | |
| 4958 | | | |
| 4959 | .array_elem_val => try self.airArrayElemVal(inst), | | |
| 4960 | .slice_elem_val => try self.airSliceElemVal(inst), | | |
| 4961 | .slice_elem_ptr => try self.airSliceElemPtr(inst), | | |
| 4962 | .ptr_elem_val => try self.airPtrElemVal(inst), | | |
| 4963 | .ptr_elem_ptr => try self.airPtrElemPtr(inst), | | |
| 4964 | | | |
| 4965 | .optional_payload => try self.airOptionalPayload(inst), | | |
| 4966 | .optional_payload_ptr => try self.airOptionalPayloadPtr(inst), | | |
| 4967 | .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst), | | |
| 4968 | | | |
| 4969 | .unwrap_errunion_payload => try self.airErrUnionPayload(inst, false), | | |
| 4970 | .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true), | | |
| 4971 | .unwrap_errunion_err => try self.airErrUnionErr(inst, false), | | |
| 4972 | .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true), | | |
| 4973 | .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst), | | |
| 4974 | .err_return_trace => try self.airErrReturnTrace(inst), | | |
| 4975 | .set_err_return_trace => try self.airSetErrReturnTrace(inst), | | |
| 4976 | .save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst), | | |
| 4977 | | | |
| 4978 | .wrap_optional => try self.airWrapOptional(body[i..]), | | |
| 4979 | .wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]), | | |
| 4980 | .wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]), | | |
| 4981 | | | |
| 4982 | .wasm_memory_size => try self.airWasmMemorySize(inst), | | |
| 4983 | .wasm_memory_grow => try self.airWasmMemoryGrow(inst), | | |
| 4984 | | | |
| 4985 | .runtime_nav_ptr => try self.airRuntimeNavPtr(inst), | | |
| 4986 | | | |
| 4987 | .inferred_alloc, .inferred_alloc_comptime => unreachable, | | |
| 4988 | | | |
| 4989 | .dbg_stmt => try self.airDbgStmt(inst), | | |
| 4990 | .dbg_empty_stmt => try self.airDbgEmptyStmt(inst), | | |
| 4991 | .dbg_var_ptr => try self.airDbgVarPtr(inst), | | |
| 4992 | .dbg_var_val => try self.airDbgVarVal(inst, false), | | |
| 4993 | .dbg_arg_inline => try self.airDbgVarVal(inst, true), | | |
| 4994 | | | |
| 4995 | .c_va_arg => try self.airCVaArg(inst), | | |
| 4996 | .c_va_copy => try self.airCVaCopy(inst), | | |
| 4997 | .c_va_end => try self.airCVaEnd(inst), | | |
| 4998 | .c_va_start => try self.airCVaStart(inst), | | |
| 4999 | | | |
| 5000 | .work_item_id => try self.airWorkItemId(inst), | | |
| 5001 | .work_group_size => try self.airWorkGroupSize(inst), | | |
| 5002 | .work_group_id => try self.airWorkGroupId(inst), | | |
| 5003 | | | |
| 5004 | // Instructions that are known to always be `noreturn` based on their tag. | | |
| 5005 | .br => return self.airBr(inst), | | |
| 5006 | .repeat => return self.airRepeat(inst), | | |
| 5007 | .switch_dispatch => return self.airSwitchDispatch(inst), | | |
| 5008 | .cond_br => return self.airCondBr(inst), | | |
| 5009 | .switch_br => return self.airSwitchBr(inst, false), | | |
| 5010 | .loop_switch_br => return self.airSwitchBr(inst, true), | | |
| 5011 | .loop => return self.airLoop(inst), | | |
| 5012 | .ret => return self.airRet(inst, false), | | |
| 5013 | .ret_safe => return self.airRet(inst, true), | | |
| 5014 | .ret_load => return self.airRetLoad(inst), | | |
| 5015 | .trap => return self.airTrap(inst), | | |
| 5016 | .unreach => return self.airUnreach(inst), | | |
| 5017 | | | |
| 5018 | // Instructions which may be `noreturn`. | | |
| 5019 | .block => res: { | | |
| 5020 | const res = try self.airBlock(inst); | | |
| 5021 | if (self.typeOfIndex(inst).isNoReturn(zcu)) return; | | |
| 5022 | break :res res; | | |
| 5023 | }, | | |
| 5024 | .dbg_inline_block => res: { | | |
| 5025 | const res = try self.airDbgInlineBlock(inst); | | |
| 5026 | if (self.typeOfIndex(inst).isNoReturn(zcu)) return; | | |
| 5027 | break :res res; | | |
| 5028 | }, | | |
| 5029 | .call, .call_always_tail, .call_never_tail, .call_never_inline => |tag| res: { | | |
| 5030 | const res = try self.airCall(inst, switch (tag) { | | |
| 5031 | .call => .auto, | | |
| 5032 | .call_always_tail => .always_tail, | | |
| 5033 | .call_never_tail => .never_tail, | | |
| 5034 | .call_never_inline => .never_inline, | | |
| 5035 | else => unreachable, | | |
| 5036 | }); | | |
| 5037 | // TODO: the AIR we emit for calls is a bit weird - the instruction has | | |
| 5038 | // type `noreturn`, but there are instructions (and maybe a safety check) following | | |
| 5039 | // nonetheless. The `unreachable` or safety check should be emitted by backends instead. | | |
| 5040 | //if (self.typeOfIndex(inst).isNoReturn(mod)) return; | | |
| 5041 | break :res res; | | |
| 5042 | }, | | |
| 5043 | | | |
| 5044 | // zig fmt: on | | |
| 5045 | }; | | |
| 5046 | if (val != .none) try self.func_inst_table.putNoClobber(self.gpa, inst.toRef(), val); | | |
| 5047 | } | | |
| 5048 | unreachable; | | |
| 5049 | } | | |
| 5050 | | | |
| 5051 | fn genBodyDebugScope( | | |
| 5052 | self: *FuncGen, | | |
| 5053 | maybe_inline_func: ?InternPool.Index, | | |
| 5054 | body: []const Air.Inst.Index, | | |
| 5055 | coverage_point: Air.CoveragePoint, | | |
| 5056 | ) Error!void { | | |
| 5057 | if (self.wip.strip) return self.genBody(body, coverage_point); | | |
| 5058 | | | |
| 5059 | const old_debug_location = self.wip.debug_location; | | |
| 5060 | const old_file = self.file; | | |
| 5061 | const old_inlined_at = self.inlined_at; | | |
| 5062 | const old_base_line = self.base_line; | | |
| 5063 | defer if (maybe_inline_func) |_| { | | |
| 5064 | self.wip.debug_location = old_debug_location; | | |
| 5065 | self.file = old_file; | | |
| 5066 | self.inlined_at = old_inlined_at; | | |
| 5067 | self.base_line = old_base_line; | | |
| 5068 | }; | | |
| 5069 | | | |
| 5070 | const old_scope = self.scope; | | |
| 5071 | defer self.scope = old_scope; | | |
| 5072 | | | |
| 5073 | if (maybe_inline_func) |inline_func| { | | |
| 5074 | const o = self.ng.object; | | |
| 5075 | const pt = self.ng.pt; | | |
| 5076 | const zcu = pt.zcu; | | |
| 5077 | const ip = &zcu.intern_pool; | | |
| 5078 | | | |
| 5079 | const func = zcu.funcInfo(inline_func); | | |
| 5080 | const nav = ip.getNav(func.owner_nav); | | |
| 5081 | const file_scope = zcu.navFileScopeIndex(func.owner_nav); | | |
| 5082 | const mod = zcu.fileByIndex(file_scope).mod.?; | | |
| 5083 | | | |
| 5084 | self.file = try o.getDebugFile(pt, file_scope); | | |
| 5085 | | | |
| 5086 | self.base_line = zcu.navSrcLine(func.owner_nav); | | |
| 5087 | const line_number = self.base_line + 1; | | |
| 5088 | self.inlined_at = try self.wip.debug_location.toMetadata(&o.builder); | | |
| 5089 | | | |
| 5090 | const fn_ty = try pt.funcType(.{ | | |
| 5091 | .param_types = &.{}, | | |
| 5092 | .return_type = .void_type, | | |
| 5093 | }); | | |
| 5094 | | | |
| 5095 | self.scope = try o.builder.debugSubprogram( | | |
| 5096 | self.file, | | |
| 5097 | try o.builder.metadataString(nav.name.toSlice(&zcu.intern_pool)), | | |
| 5098 | try o.builder.metadataString(nav.fqn.toSlice(&zcu.intern_pool)), | | |
| 5099 | line_number, | | |
| 5100 | line_number + func.lbrace_line, | | |
| 5101 | try o.getDebugType(pt, fn_ty), | | |
| 5102 | .{ | | |
| 5103 | .di_flags = .{ .StaticMember = true }, | | |
| 5104 | .sp_flags = .{ | | |
| 5105 | .Optimized = mod.optimize_mode != .Debug, | | |
| 5106 | .Definition = true, | | |
| 5107 | .LocalToUnit = true, // inline functions cannot be exported | | |
| 5108 | }, | | |
| 5109 | }, | | |
| 5110 | o.debug_compile_unit.unwrap().?, | | |
| 5111 | ); | | |
| 5112 | } | | |
| 5113 | | | |
| 5114 | self.scope = try self.ng.object.builder.debugLexicalBlock( | | |
| 5115 | self.scope, | | |
| 5116 | self.file, | | |
| 5117 | self.prev_dbg_line, | | |
| 5118 | self.prev_dbg_column, | | |
| 5119 | ); | | |
| 5120 | self.wip.debug_location = .{ .location = .{ | | |
| 5121 | .line = self.prev_dbg_line, | | |
| 5122 | .column = self.prev_dbg_column, | | |
| 5123 | .scope = self.scope.toOptional(), | | |
| 5124 | .inlined_at = self.inlined_at, | | |
| 5125 | } }; | | |
| 5126 | | | |
| 5127 | try self.genBody(body, coverage_point); | | |
| 5128 | } | | |
| 5129 | | | |
| 5130 | pub const CallAttr = enum { | | |
| 5131 | Auto, | | |
| 5132 | NeverTail, | | |
| 5133 | NeverInline, | | |
| 5134 | AlwaysTail, | | |
| 5135 | AlwaysInline, | | |
| 5136 | }; | | |
| 5137 | | | |
| 5138 | fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value { | | |
| 5139 | const air_call = self.air.unwrapCall(inst); | | |
| 5140 | const args = air_call.args; | | |
| 5141 | const o = self.ng.object; | | |
| 5142 | const pt = self.ng.pt; | | |
| 5143 | const zcu = pt.zcu; | | |
| 5144 | const ip = &zcu.intern_pool; | | |
| 5145 | const callee_ty = self.typeOf(air_call.callee); | | |
| 5146 | const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) { | | |
| 5147 | .@"fn" => callee_ty, | | |
| 5148 | .pointer => callee_ty.childType(zcu), | | |
| 5149 | else => unreachable, | | |
| 5150 | }; | | |
| 5151 | const fn_info = zcu.typeToFunc(zig_fn_ty).?; | | |
| 5152 | const return_type = Type.fromInterned(fn_info.return_type); | | |
| 5153 | const llvm_fn = try self.resolveInst(air_call.callee); | | |
| 5154 | const target = zcu.getTarget(); | | |
| 5155 | const sret = firstParamSRet(fn_info, zcu, target); | | |
| 5156 | | | |
| 5157 | var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa); | | |
| 5158 | defer llvm_args.deinit(); | | |
| 5159 | | | |
| 5160 | var attributes: Builder.FunctionAttributes.Wip = .{}; | | |
| 5161 | defer attributes.deinit(&o.builder); | | |
| 5162 | | | |
| 5163 | if (self.disable_intrinsics) { | | |
| 5164 | try attributes.addFnAttr(.nobuiltin, &o.builder); | | |
| 5165 | } | | |
| 5166 | | | |
| 5167 | switch (modifier) { | | |
| 5168 | .auto, .always_tail => {}, | | |
| 5169 | .never_tail, .never_inline => try attributes.addFnAttr(.@"noinline", &o.builder), | | |
| 5170 | .no_suspend, .always_inline, .compile_time => unreachable, | | |
| 5171 | } | | |
| 5172 | | | |
| 5173 | const ret_ptr = if (!sret) null else blk: { | | |
| 5174 | const llvm_ret_ty = try o.lowerType(pt, return_type); | | |
| 5175 | try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder); | | |
| 5176 | | | |
| 5177 | const alignment = return_type.abiAlignment(zcu).toLlvm(); | | |
| 5178 | const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment); | | |
| 5179 | try llvm_args.append(ret_ptr); | | |
| 5180 | break :blk ret_ptr; | | |
| 5181 | }; | | |
| 5182 | | | |
| 5183 | const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing; | | |
| 5184 | if (err_return_tracing) { | | |
| 5185 | assert(self.err_ret_trace != .none); | | |
| 5186 | try llvm_args.append(self.err_ret_trace); | | |
| 5187 | } | | |
| 5188 | | | |
| 5189 | var it = iterateParamTypes(o, pt, fn_info); | | |
| 5190 | while (try it.nextCall(self, args)) |lowering| switch (lowering) { | | |
| 5191 | .no_bits => continue, | | |
| 5192 | .byval => { | | |
| 5193 | const arg = args[it.zig_index - 1]; | | |
| 5194 | const param_ty = self.typeOf(arg); | | |
| 5195 | const llvm_arg = try self.resolveInst(arg); | | |
| 5196 | const llvm_param_ty = try o.lowerType(pt, param_ty); | | |
| 5197 | if (isByRef(param_ty, zcu)) { | | |
| 5198 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | | |
| 5199 | const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, ""); | | |
| 5200 | try llvm_args.append(loaded); | | |
| 5201 | } else { | | |
| 5202 | try llvm_args.append(llvm_arg); | | |
| 5203 | } | | |
| 5204 | }, | | |
| 5205 | .byref => { | | |
| 5206 | const arg = args[it.zig_index - 1]; | | |
| 5207 | const param_ty = self.typeOf(arg); | | |
| 5208 | const llvm_arg = try self.resolveInst(arg); | | |
| 5209 | if (isByRef(param_ty, zcu)) { | | |
| 5210 | try llvm_args.append(llvm_arg); | | |
| 5211 | } else { | | |
| 5212 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | | |
| 5213 | const param_llvm_ty = llvm_arg.typeOfWip(&self.wip); | | |
| 5214 | const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment); | | |
| 5215 | _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment); | | |
| 5216 | try llvm_args.append(arg_ptr); | | |
| 5217 | } | | |
| 5218 | }, | | |
| 5219 | .byref_mut => { | | |
| 5220 | const arg = args[it.zig_index - 1]; | | |
| 5221 | const param_ty = self.typeOf(arg); | | |
| 5222 | const llvm_arg = try self.resolveInst(arg); | | |
| 5223 | | | |
| 5224 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | | |
| 5225 | const param_llvm_ty = try o.lowerType(pt, param_ty); | | |
| 5226 | const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment); | | |
| 5227 | if (isByRef(param_ty, zcu)) { | | |
| 5228 | const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, ""); | | |
| 5229 | _ = try self.wip.store(.normal, loaded, arg_ptr, alignment); | | |
| 5230 | } else { | | |
| 5231 | _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment); | | |
| 5232 | } | | |
| 5233 | try llvm_args.append(arg_ptr); | | |
| 5234 | }, | | |
| 5235 | .abi_sized_int => { | | |
| 5236 | const arg = args[it.zig_index - 1]; | | |
| 5237 | const param_ty = self.typeOf(arg); | | |
| 5238 | const llvm_arg = try self.resolveInst(arg); | | |
| 5239 | const int_llvm_ty = try o.builder.intType(@intCast(param_ty.abiSize(zcu) * 8)); | | |
| 5240 | | | |
| 5241 | if (isByRef(param_ty, zcu)) { | | |
| 5242 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | | |
| 5243 | const loaded = try self.wip.load(.normal, int_llvm_ty, llvm_arg, alignment, ""); | | |
| 5244 | try llvm_args.append(loaded); | | |
| 5245 | } else { | | |
| 5246 | // LLVM does not allow bitcasting structs so we must allocate | | |
| 5247 | // a local, store as one type, and then load as another type. | | |
| 5248 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | | |
| 5249 | const int_ptr = try self.buildAlloca(int_llvm_ty, alignment); | | |
| 5250 | _ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment); | | |
| 5251 | const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, ""); | | |
| 5252 | try llvm_args.append(loaded); | | |
| 5253 | } | | |
| 5254 | }, | | |
| 5255 | .slice => { | | |
| 5256 | const arg = args[it.zig_index - 1]; | | |
| 5257 | const llvm_arg = try self.resolveInst(arg); | | |
| 5258 | const ptr = try self.wip.extractValue(llvm_arg, &.{0}, ""); | | |
| 5259 | const len = try self.wip.extractValue(llvm_arg, &.{1}, ""); | | |
| 5260 | try llvm_args.appendSlice(&.{ ptr, len }); | | |
| 5261 | }, | | |
| 5262 | .multiple_llvm_types => { | | |
| 5263 | const arg = args[it.zig_index - 1]; | | |
| 5264 | const param_ty = self.typeOf(arg); | | |
| 5265 | const llvm_types = it.types_buffer[0..it.types_len]; | | |
| 5266 | const llvm_arg = try self.resolveInst(arg); | | |
| 5267 | const is_by_ref = isByRef(param_ty, zcu); | | |
| 5268 | const arg_ptr = if (is_by_ref) llvm_arg else ptr: { | | |
| 5269 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | | |
| 5270 | const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment); | | |
| 5271 | _ = try self.wip.store(.normal, llvm_arg, ptr, alignment); | | |
| 5272 | break :ptr ptr; | | |
| 5273 | }; | | |
| 5274 | | | |
| 5275 | const llvm_ty = try o.builder.structType(.normal, llvm_types); | | |
| 5276 | try llvm_args.ensureUnusedCapacity(it.types_len); | | |
| 5277 | for (llvm_types, 0..) |field_ty, i| { | | |
| 5278 | const alignment = | | |
| 5279 | Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8)); | | |
| 5280 | const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, ""); | | |
| 5281 | const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, ""); | | |
| 5282 | llvm_args.appendAssumeCapacity(loaded); | | |
| 5283 | } | | |
| 5284 | }, | | |
| 5285 | .float_array => |count| { | | |
| 5286 | const arg = args[it.zig_index - 1]; | | |
| 5287 | const arg_ty = self.typeOf(arg); | | |
| 5288 | var llvm_arg = try self.resolveInst(arg); | | |
| 5289 | const alignment = arg_ty.abiAlignment(zcu).toLlvm(); | | |
| 5290 | if (!isByRef(arg_ty, zcu)) { | | |
| 5291 | const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment); | | |
| 5292 | _ = try self.wip.store(.normal, llvm_arg, ptr, alignment); | | |
| 5293 | llvm_arg = ptr; | | |
| 5294 | } | | |
| 5295 | | | |
| 5296 | const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?); | | |
| 5297 | const array_ty = try o.builder.arrayType(count, float_ty); | | |
| 5298 | | | |
| 5299 | const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, ""); | | |
| 5300 | try llvm_args.append(loaded); | | |
| 5301 | }, | | |
| 5302 | .i32_array, .i64_array => |arr_len| { | | |
| 5303 | const elem_size: u8 = if (lowering == .i32_array) 32 else 64; | | |
| 5304 | const arg = args[it.zig_index - 1]; | | |
| 5305 | const arg_ty = self.typeOf(arg); | | |
| 5306 | var llvm_arg = try self.resolveInst(arg); | | |
| 5307 | const alignment = arg_ty.abiAlignment(zcu).toLlvm(); | | |
| 5308 | if (!isByRef(arg_ty, zcu)) { | | |
| 5309 | const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment); | | |
| 5310 | _ = try self.wip.store(.normal, llvm_arg, ptr, alignment); | | |
| 5311 | llvm_arg = ptr; | | |
| 5312 | } | | |
| 5313 | | | |
| 5314 | const array_ty = | | |
| 5315 | try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size))); | | |
| 5316 | const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, ""); | | |
| 5317 | try llvm_args.append(loaded); | | |
| 5318 | }, | | |
| 5319 | }; | | |
| 5320 | | | |
| 5321 | { | | |
| 5322 | // Add argument attributes. | | |
| 5323 | it = iterateParamTypes(o, pt, fn_info); | | |
| 5324 | it.llvm_index += @intFromBool(sret); | | |
| 5325 | it.llvm_index += @intFromBool(err_return_tracing); | | |
| 5326 | while (try it.next()) |lowering| switch (lowering) { | | |
| 5327 | .byval => { | | |
| 5328 | const param_index = it.zig_index - 1; | | |
| 5329 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); | | |
| 5330 | if (!isByRef(param_ty, zcu)) { | | |
| 5331 | try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1); | | |
| 5332 | } | | |
| 5333 | }, | | |
| 5334 | .byref => { | | |
| 5335 | const param_index = it.zig_index - 1; | | |
| 5336 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); | | |
| 5337 | const param_llvm_ty = try o.lowerType(pt, param_ty); | | |
| 5338 | const alignment = param_ty.abiAlignment(zcu).toLlvm(); | | |
| 5339 | try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty); | | |
| 5340 | }, | | |
| 5341 | .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), | | |
| 5342 | // No attributes needed for these. | | |
| 5343 | .no_bits, | | |
| 5344 | .abi_sized_int, | | |
| 5345 | .multiple_llvm_types, | | |
| 5346 | .float_array, | | |
| 5347 | .i32_array, | | |
| 5348 | .i64_array, | | |
| 5349 | => continue, | | |
| 5350 | | | |
| 5351 | .slice => { | | |
| 5352 | assert(!it.byval_attr); | | |
| 5353 | const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); | | |
| 5354 | const ptr_info = param_ty.ptrInfo(zcu); | | |
| 5355 | const llvm_arg_i = it.llvm_index - 2; | | |
| 5356 | | | |
| 5357 | if (math.cast(u5, it.zig_index - 1)) |i| { | | |
| 5358 | if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { | | |
| 5359 | try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); | | |
| 5360 | } | | |
| 5361 | } | | |
| 5362 | if (param_ty.zigTypeTag(zcu) != .optional and | | |
| 5363 | !ptr_info.flags.is_allowzero and | | |
| 5364 | ptr_info.flags.address_space == .generic) | | |
| 5365 | { | | |
| 5366 | try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); | | |
| 5367 | } | | |
| 5368 | if (ptr_info.flags.is_const) { | | |
| 5369 | try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder); | | |
| 5370 | } | | |
| 5371 | const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) { | | |
| 5372 | else => |a| .wrap(a.toLlvm()), | | |
| 5373 | .none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)), | | |
| 5374 | }; | | |
| 5375 | try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder); | | |
| 5376 | }, | | |
| 5377 | }; | | |
| 5378 | } | | |
| 5379 | | | |
| 5380 | const call = try self.wip.call( | | |
| 5381 | switch (modifier) { | | |
| 5382 | .auto, .never_inline => .normal, | | |
| 5383 | .never_tail => .notail, | | |
| 5384 | .always_tail => .musttail, | | |
| 5385 | .no_suspend, .always_inline, .compile_time => unreachable, | | |
| 5386 | }, | | |
| 5387 | toLlvmCallConvTag(fn_info.cc, target).?, | | |
| 5388 | try attributes.finish(&o.builder), | | |
| 5389 | try o.lowerType(pt, zig_fn_ty), | | |
| 5390 | llvm_fn, | | |
| 5391 | llvm_args.items, | | |
| 5392 | "", | | |
| 5393 | ); | | |
| 5394 | | | |
| 5395 | if (fn_info.return_type == .noreturn_type and modifier != .always_tail) { | | |
| 5396 | return .none; | | |
| 5397 | } | | |
| 5398 | | | |
| 5399 | if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBits(zcu)) { | | |
| 5400 | return .none; | | |
| 5401 | } | | |
| 5402 | | | |
| 5403 | const llvm_ret_ty = try o.lowerType(pt, return_type); | | |
| 5404 | if (ret_ptr) |rp| { | | |
| 5405 | if (isByRef(return_type, zcu)) { | | |
| 5406 | return rp; | | |
| 5407 | } else { | | |
| 5408 | // our by-ref status disagrees with sret so we must load. | | |
| 5409 | const return_alignment = return_type.abiAlignment(zcu).toLlvm(); | | |
| 5410 | return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, ""); | | |
| 5411 | } | | |
| 5412 | } | | |
| 5413 | | | |
| 5414 | const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); | | |
| 5415 | | | |
| 5416 | if (abi_ret_ty != llvm_ret_ty) { | | |
| 5417 | // In this case the function return type is honoring the calling convention by having | | |
| 5418 | // a different LLVM type than the usual one. We solve this here at the callsite | | |
| 5419 | // by using our canonical type, then loading it if necessary. | | |
| 5420 | const alignment = return_type.abiAlignment(zcu).toLlvm(); | | |
| 5421 | const rp = try self.buildAlloca(abi_ret_ty, alignment); | | |
| 5422 | _ = try self.wip.store(.normal, call, rp, alignment); | | |
| 5423 | return if (isByRef(return_type, zcu)) | | |
| 5424 | rp | | |
| 5425 | else | | |
| 5426 | try self.wip.load(.normal, llvm_ret_ty, rp, alignment, ""); | | |
| 5427 | } | | |
| 5428 | | | |
| 5429 | if (isByRef(return_type, zcu)) { | | |
| 5430 | // our by-ref status disagrees with sret so we must allocate, store, | | |
| 5431 | // and return the allocation pointer. | | |
| 5432 | const alignment = return_type.abiAlignment(zcu).toLlvm(); | | |
| 5433 | const rp = try self.buildAlloca(llvm_ret_ty, alignment); | | |
| 5434 | _ = try self.wip.store(.normal, call, rp, alignment); | | |
| 5435 | return rp; | | |
| 5436 | } else { | | |
| 5437 | return call; | | |
| 5438 | } | | |
| 5439 | } | | |
| 5440 | | | |
| 5441 | fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) !void { | | |
| 5442 | const o = fg.ng.object; | | |
| 5443 | const pt = fg.ng.pt; | | |
| 5444 | const zcu = pt.zcu; | | |
| 5445 | const target = zcu.getTarget(); | | |
| 5446 | const panic_func = zcu.funcInfo(zcu.builtin_decl_values.get(panic_id.toBuiltin())); | | |
| 5447 | const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?; | | |
| 5448 | const panic_global = try o.resolveLlvmFunction(pt, panic_func.owner_nav); | | |
| 5449 | | | |
| 5450 | const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto; | | |
| 5451 | if (has_err_trace) assert(fg.err_ret_trace != .none); | | |
| 5452 | _ = try fg.wip.callIntrinsicAssumeCold(); | | |
| 5453 | _ = try fg.wip.call( | | |
| 5454 | .normal, | | |
| 5455 | toLlvmCallConvTag(fn_info.cc, target).?, | | |
| 5456 | .none, | | |
| 5457 | panic_global.typeOf(&o.builder), | | |
| 5458 | panic_global.toValue(&o.builder), | | |
| 5459 | if (has_err_trace) &.{fg.err_ret_trace} else &.{}, | | |
| 5460 | "", | | |
| 5461 | ); | | |
| 5462 | _ = try fg.wip.@"unreachable"(); | | |
| 5463 | } | | |
| 5464 | | | |
| 5465 | fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void { | | |
| 5466 | const o = self.ng.object; | | |
| 5467 | const pt = self.ng.pt; | | |
| 5468 | const zcu = pt.zcu; | | |
| 5469 | const ip = &zcu.intern_pool; | | |
| 5470 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 5471 | const ret_ty = self.typeOf(un_op); | | |
| 5472 | | | |
| 5473 | if (self.ret_ptr != .none) { | | |
| 5474 | const ptr_ty = try pt.singleMutPtrType(ret_ty); | | |
| 5475 | | | |
| 5476 | const operand = try self.resolveInst(un_op); | | |
| 5477 | const val_is_undef = if (try self.air.value(un_op, pt)) |val| val.isUndef(zcu) else false; | | |
| 5478 | if (val_is_undef and safety) undef: { | | |
| 5479 | const ptr_info = ptr_ty.ptrInfo(zcu); | | |
| 5480 | const needs_bitmask = (ptr_info.packed_offset.host_size != 0); | | |
| 5481 | if (needs_bitmask) { | | |
| 5482 | // TODO: only some bits are to be undef, we cannot write with a simple memset. | | |
| 5483 | // meanwhile, ignore the write rather than stomping over valid bits. | | |
| 5484 | // https://github.com/ziglang/zig/issues/15337 | | |
| 5485 | break :undef; | | |
| 5486 | } | | |
| 5487 | const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu)); | | |
| 5488 | _ = try self.wip.callMemSet( | | |
| 5489 | self.ret_ptr, | | |
| 5490 | ptr_ty.ptrAlignment(zcu).toLlvm(), | | |
| 5491 | try o.builder.intValue(.i8, 0xaa), | | |
| 5492 | len, | | |
| 5493 | .normal, | | |
| 5494 | self.disable_intrinsics, | | |
| 5495 | ); | | |
| 5496 | const owner_mod = self.ng.ownerModule(); | | |
| 5497 | if (owner_mod.valgrind) { | | |
| 5498 | try self.valgrindMarkUndef(self.ret_ptr, len); | | |
| 5499 | } | | |
| 5500 | _ = try self.wip.retVoid(); | | |
| 5501 | return; | | |
| 5502 | } | | |
| 5503 | | | |
| 5504 | const unwrapped_operand = operand.unwrap(); | | |
| 5505 | const unwrapped_ret = self.ret_ptr.unwrap(); | | |
| 5506 | | | |
| 5507 | // Return value was stored previously | | |
| 5508 | if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) { | | |
| 5509 | _ = try self.wip.retVoid(); | | |
| 5510 | return; | | |
| 5511 | } | | |
| 5512 | | | |
| 5513 | try self.store(self.ret_ptr, ptr_ty, operand, .none); | | |
| 5514 | _ = try self.wip.retVoid(); | | |
| 5515 | return; | | |
| 5516 | } | | |
| 5517 | const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.ng.nav_index).resolved.?.type)).?; | | |
| 5518 | if (!ret_ty.hasRuntimeBits(zcu)) { | | |
| 5519 | if (Type.fromInterned(fn_info.return_type).isError(zcu)) { | | |
| 5520 | // Functions with an empty error set are emitted with an error code | | |
| 5521 | // return type and return zero so they can be function pointers coerced | | |
| 5522 | // to functions that return anyerror. | | |
| 5523 | _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0)); | | |
| 5524 | } else { | | |
| 5525 | _ = try self.wip.retVoid(); | | |
| 5526 | } | | |
| 5527 | return; | | |
| 5528 | } | | |
| 5529 | | | |
| 5530 | const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); | | |
| 5531 | const operand = try self.resolveInst(un_op); | | |
| 5532 | const val_is_undef = if (try self.air.value(un_op, pt)) |val| val.isUndef(zcu) else false; | | |
| 5533 | const alignment = ret_ty.abiAlignment(zcu).toLlvm(); | | |
| 5534 | | | |
| 5535 | if (val_is_undef and safety) { | | |
| 5536 | const llvm_ret_ty = operand.typeOfWip(&self.wip); | | |
| 5537 | const rp = try self.buildAlloca(llvm_ret_ty, alignment); | | |
| 5538 | const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu)); | | |
| 5539 | _ = try self.wip.callMemSet( | | |
| 5540 | rp, | | |
| 5541 | alignment, | | |
| 5542 | try o.builder.intValue(.i8, 0xaa), | | |
| 5543 | len, | | |
| 5544 | .normal, | | |
| 5545 | self.disable_intrinsics, | | |
| 5546 | ); | | |
| 5547 | const owner_mod = self.ng.ownerModule(); | | |
| 5548 | if (owner_mod.valgrind) { | | |
| 5549 | try self.valgrindMarkUndef(rp, len); | | |
| 5550 | } | | |
| 5551 | _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, "")); | | |
| 5552 | return; | | |
| 5553 | } | | |
| 5554 | | | |
| 5555 | if (isByRef(ret_ty, zcu)) { | | |
| 5556 | // operand is a pointer however self.ret_ptr is null so that means | | |
| 5557 | // we need to return a value. | | |
| 5558 | _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, "")); | | |
| 5559 | return; | | |
| 5560 | } | | |
| 5561 | | | |
| 5562 | const llvm_ret_ty = operand.typeOfWip(&self.wip); | | |
| 5563 | if (abi_ret_ty == llvm_ret_ty) { | | |
| 5564 | _ = try self.wip.ret(operand); | | |
| 5565 | return; | | |
| 5566 | } | | |
| 5567 | | | |
| 5568 | const rp = try self.buildAlloca(llvm_ret_ty, alignment); | | |
| 5569 | _ = try self.wip.store(.normal, operand, rp, alignment); | | |
| 5570 | _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, "")); | | |
| 5571 | return; | | |
| 5572 | } | | |
| 5573 | | | |
| 5574 | fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !void { | | |
| 5575 | const o = self.ng.object; | | |
| 5576 | const pt = self.ng.pt; | | |
| 5577 | const zcu = pt.zcu; | | |
| 5578 | const ip = &zcu.intern_pool; | | |
| 5579 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 5580 | const ptr_ty = self.typeOf(un_op); | | |
| 5581 | const ret_ty = ptr_ty.childType(zcu); | | |
| 5582 | const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.ng.nav_index).resolved.?.type)).?; | | |
| 5583 | if (!ret_ty.hasRuntimeBits(zcu)) { | | |
| 5584 | if (Type.fromInterned(fn_info.return_type).isError(zcu)) { | | |
| 5585 | // Functions with an empty error set are emitted with an error code | | |
| 5586 | // return type and return zero so they can be function pointers coerced | | |
| 5587 | // to functions that return anyerror. | | |
| 5588 | _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0)); | | |
| 5589 | } else { | | |
| 5590 | _ = try self.wip.retVoid(); | | |
| 5591 | } | | |
| 5592 | return; | | |
| 5593 | } | | |
| 5594 | if (self.ret_ptr != .none) { | | |
| 5595 | _ = try self.wip.retVoid(); | | |
| 5596 | return; | | |
| 5597 | } | | |
| 5598 | const ptr = try self.resolveInst(un_op); | | |
| 5599 | const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); | | |
| 5600 | const alignment = ret_ty.abiAlignment(zcu).toLlvm(); | | |
| 5601 | _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, "")); | | |
| 5602 | return; | | |
| 5603 | } | | |
| 5604 | | | |
| 5605 | fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 5606 | const o = self.ng.object; | | |
| 5607 | const pt = self.ng.pt; | | |
| 5608 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 5609 | const list = try self.resolveInst(ty_op.operand); | | |
| 5610 | const arg_ty = ty_op.ty.toType(); | | |
| 5611 | const llvm_arg_ty = try o.lowerType(pt, arg_ty); | | |
| 5612 | | | |
| 5613 | return self.wip.vaArg(list, llvm_arg_ty, ""); | | |
| 5614 | } | | |
| 5615 | | | |
| 5616 | fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 5617 | const o = self.ng.object; | | |
| 5618 | const pt = self.ng.pt; | | |
| 5619 | const zcu = pt.zcu; | | |
| 5620 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 5621 | const src_list = try self.resolveInst(ty_op.operand); | | |
| 5622 | const va_list_ty = ty_op.ty.toType(); | | |
| 5623 | const llvm_va_list_ty = try o.lowerType(pt, va_list_ty); | | |
| 5624 | | | |
| 5625 | const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); | | |
| 5626 | const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); | | |
| 5627 | | | |
| 5628 | _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, ""); | | |
| 5629 | return if (isByRef(va_list_ty, zcu)) | | |
| 5630 | dest_list | | |
| 5631 | else | | |
| 5632 | try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, ""); | | |
| 5633 | } | | |
| 5634 | | | |
| 5635 | fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 5636 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 5637 | const src_list = try self.resolveInst(un_op); | | |
| 5638 | | | |
| 5639 | _ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, ""); | | |
| 5640 | return .none; | | |
| 5641 | } | | |
| 5642 | | | |
| 5643 | fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 5644 | const o = self.ng.object; | | |
| 5645 | const pt = self.ng.pt; | | |
| 5646 | const zcu = pt.zcu; | | |
| 5647 | const va_list_ty = self.typeOfIndex(inst); | | |
| 5648 | const llvm_va_list_ty = try o.lowerType(pt, va_list_ty); | | |
| 5649 | | | |
| 5650 | const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); | | |
| 5651 | const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); | | |
| 5652 | | | |
| 5653 | _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, ""); | | |
| 5654 | return if (isByRef(va_list_ty, zcu)) | | |
| 5655 | dest_list | | |
| 5656 | else | | |
| 5657 | try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, ""); | | |
| 5658 | } | | |
| 5659 | | | |
| 5660 | fn airCmp( | | |
| 5661 | self: *FuncGen, | | |
| 5662 | inst: Air.Inst.Index, | | |
| 5663 | op: math.CompareOperator, | | |
| 5664 | fast: Builder.FastMathKind, | | |
| 5665 | ) !Builder.Value { | | |
| 5666 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 5667 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 5668 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 5669 | const operand_ty = self.typeOf(bin_op.lhs); | | |
| 5670 | | | |
| 5671 | return self.cmp(fast, op, operand_ty, lhs, rhs); | | |
| 5672 | } | | |
| 5673 | | | |
| 5674 | fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 5675 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 5676 | const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data; | | |
| 5677 | | | |
| 5678 | const lhs = try self.resolveInst(extra.lhs); | | |
| 5679 | const rhs = try self.resolveInst(extra.rhs); | | |
| 5680 | const vec_ty = self.typeOf(extra.lhs); | | |
| 5681 | const cmp_op = extra.compareOperator(); | | |
| 5682 | | | |
| 5683 | return self.cmp(fast, cmp_op, vec_ty, lhs, rhs); | | |
| 5684 | } | | |
| 5685 | | | |
| 5686 | fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 5687 | const o = self.ng.object; | | |
| 5688 | const pt = self.ng.pt; | | |
| 5689 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 5690 | const operand = try self.resolveInst(un_op); | | |
| 5691 | const llvm_fn = try o.getCmpLtErrorsLenFunction(pt); | | |
| 5692 | return self.wip.call( | | |
| 5693 | .normal, | | |
| 5694 | .fastcc, | | |
| 5695 | .none, | | |
| 5696 | llvm_fn.typeOf(&o.builder), | | |
| 5697 | llvm_fn.toValue(&o.builder), | | |
| 5698 | &.{operand}, | | |
| 5699 | "", | | |
| 5700 | ); | | |
| 5701 | } | | |
| 5702 | | | |
| 5703 | fn cmp( | | |
| 5704 | self: *FuncGen, | | |
| 5705 | fast: Builder.FastMathKind, | | |
| 5706 | op: math.CompareOperator, | | |
| 5707 | operand_ty: Type, | | |
| 5708 | lhs: Builder.Value, | | |
| 5709 | rhs: Builder.Value, | | |
| 5710 | ) Allocator.Error!Builder.Value { | | |
| 5711 | const o = self.ng.object; | | |
| 5712 | const pt = self.ng.pt; | | |
| 5713 | const zcu = pt.zcu; | | |
| 5714 | const scalar_ty = operand_ty.scalarType(zcu); | | |
| 5715 | const int_ty = switch (scalar_ty.zigTypeTag(zcu)) { | | |
| 5716 | .@"enum" => scalar_ty.intTagType(zcu), | | |
| 5717 | .int, .bool, .pointer, .error_set => scalar_ty, | | |
| 5718 | .optional => blk: { | | |
| 5719 | const payload_ty = operand_ty.optionalChild(zcu); | | |
| 5720 | if (!payload_ty.hasRuntimeBits(zcu) or | | |
| 5721 | operand_ty.optionalReprIsPayload(zcu)) | | |
| 5722 | { | | |
| 5723 | break :blk operand_ty; | | |
| 5724 | } | | |
| 5725 | // We need to emit instructions to check for equality/inequality | | |
| 5726 | // of optionals that are not pointers. | | |
| 5727 | const is_by_ref = isByRef(scalar_ty, zcu); | | |
| 5728 | const opt_llvm_ty = try o.lowerType(pt, scalar_ty); | | |
| 5729 | const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref, .normal); | | |
| 5730 | const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref, .normal); | | |
| 5731 | const llvm_i2 = try o.builder.intType(2); | | |
| 5732 | const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, ""); | | |
| 5733 | const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, ""); | | |
| 5734 | const lhs_shifted = try self.wip.bin(.shl, lhs_non_null_i2, try o.builder.intValue(llvm_i2, 1), ""); | | |
| 5735 | const lhs_rhs_ored = try self.wip.bin(.@"or", lhs_shifted, rhs_non_null_i2, ""); | | |
| 5736 | const both_null_block = try self.wip.block(1, "BothNull"); | | |
| 5737 | const mixed_block = try self.wip.block(1, "Mixed"); | | |
| 5738 | const both_pl_block = try self.wip.block(1, "BothNonNull"); | | |
| 5739 | const end_block = try self.wip.block(3, "End"); | | |
| 5740 | var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none); | | |
| 5741 | defer wip_switch.finish(&self.wip); | | |
| 5742 | try wip_switch.addCase( | | |
| 5743 | try o.builder.intConst(llvm_i2, 0b00), | | |
| 5744 | both_null_block, | | |
| 5745 | &self.wip, | | |
| 5746 | ); | | |
| 5747 | try wip_switch.addCase( | | |
| 5748 | try o.builder.intConst(llvm_i2, 0b11), | | |
| 5749 | both_pl_block, | | |
| 5750 | &self.wip, | | |
| 5751 | ); | | |
| 5752 | | | |
| 5753 | self.wip.cursor = .{ .block = both_null_block }; | | |
| 5754 | _ = try self.wip.br(end_block); | | |
| 5755 | | | |
| 5756 | self.wip.cursor = .{ .block = mixed_block }; | | |
| 5757 | _ = try self.wip.br(end_block); | | |
| 5758 | | | |
| 5759 | self.wip.cursor = .{ .block = both_pl_block }; | | |
| 5760 | const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true); | | |
| 5761 | const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true); | | |
| 5762 | const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload); | | |
| 5763 | _ = try self.wip.br(end_block); | | |
| 5764 | const both_pl_block_end = self.wip.cursor.block; | | |
| 5765 | | | |
| 5766 | self.wip.cursor = .{ .block = end_block }; | | |
| 5767 | const llvm_i1_0 = Builder.Value.false; | | |
| 5768 | const llvm_i1_1 = Builder.Value.true; | | |
| 5769 | const incoming_values: [3]Builder.Value = .{ | | |
| 5770 | switch (op) { | | |
| 5771 | .eq => llvm_i1_1, | | |
| 5772 | .neq => llvm_i1_0, | | |
| 5773 | else => unreachable, | | |
| 5774 | }, | | |
| 5775 | switch (op) { | | |
| 5776 | .eq => llvm_i1_0, | | |
| 5777 | .neq => llvm_i1_1, | | |
| 5778 | else => unreachable, | | |
| 5779 | }, | | |
| 5780 | payload_cmp, | | |
| 5781 | }; | | |
| 5782 | | | |
| 5783 | const phi = try self.wip.phi(.i1, ""); | | |
| 5784 | phi.finish( | | |
| 5785 | &incoming_values, | | |
| 5786 | &.{ both_null_block, mixed_block, both_pl_block_end }, | | |
| 5787 | &self.wip, | | |
| 5788 | ); | | |
| 5789 | return phi.toValue(); | | |
| 5790 | }, | | |
| 5791 | .float => return self.buildFloatCmp(fast, op, operand_ty, .{ lhs, rhs }), | | |
| 5792 | .@"struct", .@"union" => scalar_ty.bitpackBackingInt(zcu), | | |
| 5793 | else => unreachable, | | |
| 5794 | }; | | |
| 5795 | const is_signed = int_ty.isSignedInt(zcu); | | |
| 5796 | const cond: Builder.IntegerCondition = switch (op) { | | |
| 5797 | .eq => .eq, | | |
| 5798 | .neq => .ne, | | |
| 5799 | .lt => if (is_signed) .slt else .ult, | | |
| 5800 | .lte => if (is_signed) .sle else .ule, | | |
| 5801 | .gt => if (is_signed) .sgt else .ugt, | | |
| 5802 | .gte => if (is_signed) .sge else .uge, | | |
| 5803 | }; | | |
| 5804 | return self.wip.icmp(cond, lhs, rhs, ""); | | |
| 5805 | } | | |
| 5806 | | | |
| 5807 | fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 5808 | const block = self.air.unwrapBlock(inst); | | |
| 5809 | return self.lowerBlock(inst, null, block.body); | | |
| 5810 | } | | |
| 5811 | | | |
| 5812 | fn lowerBlock( | | |
| 5813 | self: *FuncGen, | | |
| 5814 | inst: Air.Inst.Index, | | |
| 5815 | maybe_inline_func: ?InternPool.Index, | | |
| 5816 | body: []const Air.Inst.Index, | | |
| 5817 | ) !Builder.Value { | | |
| 5818 | const o = self.ng.object; | | |
| 5819 | const pt = self.ng.pt; | | |
| 5820 | const zcu = pt.zcu; | | |
| 5821 | const inst_ty = self.typeOfIndex(inst); | | |
| 5822 | | | |
| 5823 | if (inst_ty.isNoReturn(zcu)) { | | |
| 5824 | try self.genBodyDebugScope(maybe_inline_func, body, .none); | | |
| 5825 | return .none; | | |
| 5826 | } | | |
| 5827 | | | |
| 5828 | const have_block_result = inst_ty.hasRuntimeBits(zcu); | | |
| 5829 | | | |
| 5830 | var breaks: BreakList = if (have_block_result) .{ .list = .{} } else .{ .len = 0 }; | | |
| 5831 | defer if (have_block_result) breaks.list.deinit(self.gpa); | | |
| 5832 | | | |
| 5833 | const parent_bb = try self.wip.block(0, "Block"); | | |
| 5834 | try self.blocks.putNoClobber(self.gpa, inst, .{ | | |
| 5835 | .parent_bb = parent_bb, | | |
| 5836 | .breaks = &breaks, | | |
| 5837 | }); | | |
| 5838 | defer assert(self.blocks.remove(inst)); | | |
| 5839 | | | |
| 5840 | try self.genBodyDebugScope(maybe_inline_func, body, .none); | | |
| 5841 | | | |
| 5842 | self.wip.cursor = .{ .block = parent_bb }; | | |
| 5843 | | | |
| 5844 | // Create a phi node only if the block returns a value. | | |
| 5845 | if (have_block_result) { | | |
| 5846 | const raw_llvm_ty = try o.lowerType(pt, inst_ty); | | |
| 5847 | const llvm_ty: Builder.Type = ty: { | | |
| 5848 | // If the zig tag type is a function, this represents an actual function body; not | | |
| 5849 | // a pointer to it. LLVM IR allows the call instruction to use function bodies instead | | |
| 5850 | // of function pointers, however the phi makes it a runtime value and therefore | | |
| 5851 | // the LLVM type has to be wrapped in a pointer. | | |
| 5852 | if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) { | | |
| 5853 | break :ty .ptr; | | |
| 5854 | } | | |
| 5855 | break :ty raw_llvm_ty; | | |
| 5856 | }; | | |
| 5857 | | | |
| 5858 | parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len); | | |
| 5859 | const phi = try self.wip.phi(llvm_ty, ""); | | |
| 5860 | phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip); | | |
| 5861 | return phi.toValue(); | | |
| 5862 | } else { | | |
| 5863 | parent_bb.ptr(&self.wip).incoming = @intCast(breaks.len); | | |
| 5864 | return .none; | | |
| 5865 | } | | |
| 5866 | } | | |
| 5867 | | | |
| 5868 | fn airBr(self: *FuncGen, inst: Air.Inst.Index) !void { | | |
| 5869 | const zcu = self.ng.pt.zcu; | | |
| 5870 | const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br; | | |
| 5871 | const block = self.blocks.get(branch.block_inst).?; | | |
| 5872 | | | |
| 5873 | // Add the values to the lists only if the break provides a value. | | |
| 5874 | const operand_ty = self.typeOf(branch.operand); | | |
| 5875 | if (operand_ty.hasRuntimeBits(zcu)) { | | |
| 5876 | const val = try self.resolveInst(branch.operand); | | |
| 5877 | | | |
| 5878 | // For the phi node, we need the basic blocks and the values of the | | |
| 5879 | // break instructions. | | |
| 5880 | try block.breaks.list.append(self.gpa, .{ .bb = self.wip.cursor.block, .val = val }); | | |
| 5881 | } else block.breaks.len += 1; | | |
| 5882 | _ = try self.wip.br(block.parent_bb); | | |
| 5883 | } | | |
| 5884 | | | |
| 5885 | fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) !void { | | |
| 5886 | const repeat = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat; | | |
| 5887 | const loop_bb = self.loops.get(repeat.loop_inst).?; | | |
| 5888 | loop_bb.ptr(&self.wip).incoming += 1; | | |
| 5889 | _ = try self.wip.br(loop_bb); | | |
| 5890 | } | | |
| 5891 | | | |
| 5892 | fn lowerSwitchDispatch( | | |
| 5893 | self: *FuncGen, | | |
| 5894 | switch_inst: Air.Inst.Index, | | |
| 5895 | cond_ref: Air.Inst.Ref, | | |
| 5896 | dispatch_info: SwitchDispatchInfo, | | |
| 5897 | ) !void { | | |
| 5898 | const o = self.ng.object; | | |
| 5899 | const pt = self.ng.pt; | | |
| 5900 | const zcu = pt.zcu; | | |
| 5901 | const cond_ty = self.typeOf(cond_ref); | | |
| 5902 | const switch_br = self.air.unwrapSwitch(switch_inst); | | |
| 5903 | | | |
| 5904 | if (try self.air.value(cond_ref, pt)) |cond_val| { | | |
| 5905 | // Comptime-known dispatch. Iterate the cases to find the correct | | |
| 5906 | // one, and branch to the corresponding element of `case_blocks`. | | |
| 5907 | var it = switch_br.iterateCases(); | | |
| 5908 | const target_case_idx = target: while (it.next()) |case| { | | |
| 5909 | for (case.items) |item| { | | |
| 5910 | const val = Value.fromInterned(item.toInterned().?); | | |
| 5911 | if (cond_val.compareHetero(.eq, val, zcu)) break :target case.idx; | | |
| 5912 | } | | |
| 5913 | for (case.ranges) |range| { | | |
| 5914 | const low = Value.fromInterned(range[0].toInterned().?); | | |
| 5915 | const high = Value.fromInterned(range[1].toInterned().?); | | |
| 5916 | if (cond_val.compareHetero(.gte, low, zcu) and | | |
| 5917 | cond_val.compareHetero(.lte, high, zcu)) | | |
| 5918 | { | | |
| 5919 | break :target case.idx; | | |
| 5920 | } | | |
| 5921 | } | | |
| 5922 | } else dispatch_info.case_blocks.len - 1; | | |
| 5923 | const target_block = dispatch_info.case_blocks[target_case_idx]; | | |
| 5924 | target_block.ptr(&self.wip).incoming += 1; | | |
| 5925 | _ = try self.wip.br(target_block); | | |
| 5926 | return; | | |
| 5927 | } | | |
| 5928 | | | |
| 5929 | // Runtime-known dispatch. | | |
| 5930 | const cond = try self.resolveInst(cond_ref); | | |
| 5931 | | | |
| 5932 | if (dispatch_info.jmp_table) |jmp_table| { | | |
| 5933 | // We should use the constructed jump table. | | |
| 5934 | // First, check the bounds to branch to the `else` case if needed. | | |
| 5935 | const inbounds = try self.wip.bin( | | |
| 5936 | .@"and", | | |
| 5937 | try self.cmp(.normal, .gte, cond_ty, cond, jmp_table.min.toValue()), | | |
| 5938 | try self.cmp(.normal, .lte, cond_ty, cond, jmp_table.max.toValue()), | | |
| 5939 | "", | | |
| 5940 | ); | | |
| 5941 | const jmp_table_block = try self.wip.block(1, "Then"); | | |
| 5942 | const else_block = dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]; | | |
| 5943 | else_block.ptr(&self.wip).incoming += 1; | | |
| 5944 | _ = try self.wip.brCond(inbounds, jmp_table_block, else_block, switch (jmp_table.in_bounds_hint) { | | |
| 5945 | .none => .none, | | |
| 5946 | .unpredictable => .unpredictable, | | |
| 5947 | .likely => .then_likely, | | |
| 5948 | .unlikely => .else_likely, | | |
| 5949 | }); | | |
| 5950 | | | |
| 5951 | self.wip.cursor = .{ .block = jmp_table_block }; | | |
| 5952 | | | |
| 5953 | // Figure out the list of blocks we might branch to. | | |
| 5954 | // This includes all case blocks, but it might not include the `else` block if | | |
| 5955 | // the table is dense. | | |
| 5956 | const target_blocks_len = dispatch_info.case_blocks.len - @intFromBool(!jmp_table.table_includes_else); | | |
| 5957 | const target_blocks = dispatch_info.case_blocks[0..target_blocks_len]; | | |
| 5958 | | | |
| 5959 | // Make sure to cast the index to a usize so it's not treated as negative! | | |
| 5960 | const table_index = try self.wip.conv( | | |
| 5961 | .unsigned, | | |
| 5962 | try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""), | | |
| 5963 | try o.lowerType(pt, .usize), | | |
| 5964 | "", | | |
| 5965 | ); | | |
| 5966 | const target_ptr_ptr = try self.wip.gep( | | |
| 5967 | .inbounds, | | |
| 5968 | .ptr, | | |
| 5969 | jmp_table.table.toValue(), | | |
| 5970 | &.{table_index}, | | |
| 5971 | "", | | |
| 5972 | ); | | |
| 5973 | const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, ""); | | |
| 5974 | | | |
| 5975 | // Do the branch! | | |
| 5976 | _ = try self.wip.indirectbr(target_ptr, target_blocks); | | |
| 5977 | | | |
| 5978 | // Mark all target blocks as having one more incoming branch. | | |
| 5979 | for (target_blocks) |case_block| { | | |
| 5980 | case_block.ptr(&self.wip).incoming += 1; | | |
| 5981 | } | | |
| 5982 | | | |
| 5983 | return; | | |
| 5984 | } | | |
| 5985 | | | |
| 5986 | // We must lower to an actual LLVM `switch` instruction. | | |
| 5987 | // The switch prongs will correspond to our scalar cases. Ranges will | | |
| 5988 | // be handled by conditional branches in the `else` prong. | | |
| 5989 | | | |
| 5990 | const llvm_usize = try o.lowerType(pt, Type.usize); | | |
| 5991 | const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder)) | | |
| 5992 | try self.wip.cast(.ptrtoint, cond, llvm_usize, "") | | |
| 5993 | else | | |
| 5994 | cond; | | |
| 5995 | | | |
| 5996 | const llvm_cases_len, const last_range_case = info: { | | |
| 5997 | var llvm_cases_len: u32 = 0; | | |
| 5998 | var last_range_case: ?u32 = null; | | |
| 5999 | var it = switch_br.iterateCases(); | | |
| 6000 | while (it.next()) |case| { | | |
| 6001 | if (case.ranges.len > 0) last_range_case = case.idx; | | |
| 6002 | llvm_cases_len += @intCast(case.items.len); | | |
| 6003 | } | | |
| 6004 | break :info .{ llvm_cases_len, last_range_case }; | | |
| 6005 | }; | | |
| 6006 | | | |
| 6007 | // The `else` of the LLVM `switch` is the actual `else` prong only | | |
| 6008 | // if there are no ranges. Otherwise, the `else` will have a | | |
| 6009 | // conditional chain before the "true" `else` prong. | | |
| 6010 | const llvm_else_block = if (last_range_case == null) | | |
| 6011 | dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1] | | |
| 6012 | else | | |
| 6013 | try self.wip.block(0, "RangeTest"); | | |
| 6014 | | | |
| 6015 | llvm_else_block.ptr(&self.wip).incoming += 1; | | |
| 6016 | | | |
| 6017 | var wip_switch = try self.wip.@"switch"(cond_int, llvm_else_block, llvm_cases_len, dispatch_info.switch_weights); | | |
| 6018 | defer wip_switch.finish(&self.wip); | | |
| 6019 | | | |
| 6020 | // Construct the actual cases. Set the cursor to the `else` block so | | |
| 6021 | // we can construct ranges at the same time as scalar cases. | | |
| 6022 | self.wip.cursor = .{ .block = llvm_else_block }; | | |
| 6023 | | | |
| 6024 | var it = switch_br.iterateCases(); | | |
| 6025 | while (it.next()) |case| { | | |
| 6026 | const case_block = dispatch_info.case_blocks[case.idx]; | | |
| 6027 | | | |
| 6028 | for (case.items) |item| { | | |
| 6029 | const llvm_item = (try self.resolveInst(item)).toConst().?; | | |
| 6030 | const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder)) | | |
| 6031 | try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize) | | |
| 6032 | else | | |
| 6033 | llvm_item; | | |
| 6034 | try wip_switch.addCase(llvm_int_item, case_block, &self.wip); | | |
| 6035 | } | | |
| 6036 | case_block.ptr(&self.wip).incoming += @intCast(case.items.len); | | |
| 6037 | | | |
| 6038 | if (case.ranges.len == 0) continue; | | |
| 6039 | | | |
| 6040 | // Add a conditional for the ranges, directing to the relevant bb. | | |
| 6041 | // We don't need to consider `cold` branch hints since that information is stored | | |
| 6042 | // in the target bb body, but we do care about likely/unlikely/unpredictable. | | |
| 6043 | | | |
| 6044 | const hint = switch_br.getHint(case.idx); | | |
| 6045 | | | |
| 6046 | var range_cond: ?Builder.Value = null; | | |
| 6047 | for (case.ranges) |range| { | | |
| 6048 | const llvm_min = try self.resolveInst(range[0]); | | |
| 6049 | const llvm_max = try self.resolveInst(range[1]); | | |
| 6050 | const cond_part = try self.wip.bin( | | |
| 6051 | .@"and", | | |
| 6052 | try self.cmp(.normal, .gte, cond_ty, cond, llvm_min), | | |
| 6053 | try self.cmp(.normal, .lte, cond_ty, cond, llvm_max), | | |
| 6054 | "", | | |
| 6055 | ); | | |
| 6056 | if (range_cond) |prev| { | | |
| 6057 | range_cond = try self.wip.bin(.@"or", prev, cond_part, ""); | | |
| 6058 | } else range_cond = cond_part; | | |
| 6059 | } | | |
| 6060 | | | |
| 6061 | // If the check fails, we either branch to the "true" `else` case, | | |
| 6062 | // or to the next range condition. | | |
| 6063 | const range_else_block = if (case.idx == last_range_case.?) | | |
| 6064 | dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1] | | |
| 6065 | else | | |
| 6066 | try self.wip.block(0, "RangeTest"); | | |
| 6067 | | | |
| 6068 | _ = try self.wip.brCond(range_cond.?, case_block, range_else_block, switch (hint) { | | |
| 6069 | .none, .cold => .none, | | |
| 6070 | .unpredictable => .unpredictable, | | |
| 6071 | .likely => .then_likely, | | |
| 6072 | .unlikely => .else_likely, | | |
| 6073 | }); | | |
| 6074 | case_block.ptr(&self.wip).incoming += 1; | | |
| 6075 | range_else_block.ptr(&self.wip).incoming += 1; | | |
| 6076 | | | |
| 6077 | // Construct the next range conditional (if any) in the false branch. | | |
| 6078 | self.wip.cursor = .{ .block = range_else_block }; | | |
| 6079 | } | | |
| 6080 | } | | |
| 6081 | | | |
| 6082 | fn airSwitchDispatch(self: *FuncGen, inst: Air.Inst.Index) !void { | | |
| 6083 | const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br; | | |
| 6084 | const dispatch_info = self.switch_dispatch_info.get(br.block_inst).?; | | |
| 6085 | return self.lowerSwitchDispatch(br.block_inst, br.operand, dispatch_info); | | |
| 6086 | } | | |
| 6087 | | | |
| 6088 | fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !void { | | |
| 6089 | const cond_br = self.air.unwrapCondBr(inst); | | |
| 6090 | const cond = try self.resolveInst(cond_br.condition); | | |
| 6091 | const then_body = cond_br.then_body; | | |
| 6092 | const else_body = cond_br.else_body; | | |
| 6093 | | | |
| 6094 | const Hint = enum { | | |
| 6095 | none, | | |
| 6096 | unpredictable, | | |
| 6097 | then_likely, | | |
| 6098 | else_likely, | | |
| 6099 | then_cold, | | |
| 6100 | else_cold, | | |
| 6101 | }; | | |
| 6102 | const hint: Hint = switch (cond_br.branch_hints.true) { | | |
| 6103 | .none => switch (cond_br.branch_hints.false) { | | |
| 6104 | .none => .none, | | |
| 6105 | .likely => .else_likely, | | |
| 6106 | .unlikely => .then_likely, | | |
| 6107 | .cold => .else_cold, | | |
| 6108 | .unpredictable => .unpredictable, | | |
| 6109 | }, | | |
| 6110 | .likely => switch (cond_br.branch_hints.false) { | | |
| 6111 | .none => .then_likely, | | |
| 6112 | .likely => .unpredictable, | | |
| 6113 | .unlikely => .then_likely, | | |
| 6114 | .cold => .else_cold, | | |
| 6115 | .unpredictable => .unpredictable, | | |
| 6116 | }, | | |
| 6117 | .unlikely => switch (cond_br.branch_hints.false) { | | |
| 6118 | .none => .else_likely, | | |
| 6119 | .likely => .else_likely, | | |
| 6120 | .unlikely => .unpredictable, | | |
| 6121 | .cold => .else_cold, | | |
| 6122 | .unpredictable => .unpredictable, | | |
| 6123 | }, | | |
| 6124 | .cold => .then_cold, | | |
| 6125 | .unpredictable => .unpredictable, | | |
| 6126 | }; | | |
| 6127 | | | |
| 6128 | const then_block = try self.wip.block(1, "Then"); | | |
| 6129 | const else_block = try self.wip.block(1, "Else"); | | |
| 6130 | _ = try self.wip.brCond(cond, then_block, else_block, switch (hint) { | | |
| 6131 | .none, .then_cold, .else_cold => .none, | | |
| 6132 | .unpredictable => .unpredictable, | | |
| 6133 | .then_likely => .then_likely, | | |
| 6134 | .else_likely => .else_likely, | | |
| 6135 | }); | | |
| 6136 | | | |
| 6137 | self.wip.cursor = .{ .block = then_block }; | | |
| 6138 | if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold(); | | |
| 6139 | try self.genBodyDebugScope(null, then_body, cond_br.branch_hints.then_cov); | | |
| 6140 | | | |
| 6141 | self.wip.cursor = .{ .block = else_block }; | | |
| 6142 | if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold(); | | |
| 6143 | try self.genBodyDebugScope(null, else_body, cond_br.branch_hints.else_cov); | | |
| 6144 | | | |
| 6145 | // No need to reset the insert cursor since this instruction is noreturn. | | |
| 6146 | } | | |
| 6147 | | | |
| 6148 | fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { | | |
| 6149 | const unwrapped_try = self.air.unwrapTry(inst); | | |
| 6150 | const err_union = try self.resolveInst(unwrapped_try.error_union); | | |
| 6151 | const body = unwrapped_try.else_body; | | |
| 6152 | const err_union_ty = self.typeOf(unwrapped_try.error_union); | | |
| 6153 | const is_unused = self.liveness.isUnused(inst); | | |
| 6154 | return lowerTry(self, err_union, body, err_union_ty, false, .none, false, is_unused, err_cold); | | |
| 6155 | } | | |
| 6156 | | | |
| 6157 | fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { | | |
| 6158 | const zcu = self.ng.pt.zcu; | | |
| 6159 | const unwrapped_try = self.air.unwrapTryPtr(inst); | | |
| 6160 | const err_union_ptr = try self.resolveInst(unwrapped_try.error_union_ptr); | | |
| 6161 | const body = unwrapped_try.else_body; | | |
| 6162 | const err_union_ptr_ty = self.typeOf(unwrapped_try.error_union_ptr); | | |
| 6163 | const err_union_ty = err_union_ptr_ty.childType(zcu); | | |
| 6164 | const is_unused = self.liveness.isUnused(inst); | | |
| 6165 | | | |
| 6166 | self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu)); | | |
| 6167 | | | |
| 6168 | return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), true, is_unused, err_cold); | | |
| 6169 | } | | |
| 6170 | | | |
| 6171 | fn lowerTry( | | |
| 6172 | fg: *FuncGen, | | |
| 6173 | err_union: Builder.Value, | | |
| 6174 | body: []const Air.Inst.Index, | | |
| 6175 | err_union_ty: Type, | | |
| 6176 | operand_is_ptr: bool, | | |
| 6177 | operand_ptr_align: InternPool.Alignment, | | |
| 6178 | can_elide_load: bool, | | |
| 6179 | is_unused: bool, | | |
| 6180 | err_cold: bool, | | |
| 6181 | ) !Builder.Value { | | |
| 6182 | const o = fg.ng.object; | | |
| 6183 | const pt = fg.ng.pt; | | |
| 6184 | const zcu = pt.zcu; | | |
| 6185 | const payload_ty = err_union_ty.errorUnionPayload(zcu); | | |
| 6186 | const payload_has_bits = payload_ty.hasRuntimeBits(zcu); | | |
| 6187 | const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); | | |
| 6188 | const error_type = try o.errorIntType(pt); | | |
| 6189 | | | |
| 6190 | const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{ | | |
| 6191 | operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)), | | |
| 6192 | operand_ptr_align.minStrict(payload_ty.abiAlignment(zcu)), | | |
| 6193 | } else .{ .none, .none }; | | |
| 6194 | | | |
| 6195 | if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { | | |
| 6196 | const loaded = loaded: { | | |
| 6197 | const access_kind: Builder.MemoryAccessKind = | | |
| 6198 | if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | | |
| 6199 | | | |
| 6200 | if (!payload_has_bits) { | | |
| 6201 | break :loaded if (operand_is_ptr) | | |
| 6202 | try fg.wip.load(access_kind, error_type, err_union, err_set_align.toLlvm(), "") | | |
| 6203 | else | | |
| 6204 | err_union; | | |
| 6205 | } | | |
| 6206 | const err_field_index = try errUnionErrorOffset(payload_ty, pt); | | |
| 6207 | if (operand_is_ptr or isByRef(err_union_ty, zcu)) { | | |
| 6208 | const err_field_ptr = | | |
| 6209 | try fg.wip.gepStruct(err_union_llvm_ty, err_union, err_field_index, ""); | | |
| 6210 | break :loaded try fg.wip.load( | | |
| 6211 | if (operand_is_ptr) access_kind else .normal, | | |
| 6212 | error_type, | | |
| 6213 | err_field_ptr, | | |
| 6214 | err_set_align.toLlvm(), | | |
| 6215 | "", | | |
| 6216 | ); | | |
| 6217 | } | | |
| 6218 | break :loaded try fg.wip.extractValue(err_union, &.{err_field_index}, ""); | | |
| 6219 | }; | | |
| 6220 | const zero = try o.builder.intValue(error_type, 0); | | |
| 6221 | const is_err = try fg.wip.icmp(.ne, loaded, zero, ""); | | |
| 6222 | | | |
| 6223 | const return_block = try fg.wip.block(1, "TryRet"); | | |
| 6224 | const continue_block = try fg.wip.block(1, "TryCont"); | | |
| 6225 | _ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely); | | |
| 6226 | | | |
| 6227 | fg.wip.cursor = .{ .block = return_block }; | | |
| 6228 | if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold(); | | |
| 6229 | try fg.genBodyDebugScope(null, body, .poi); | | |
| 6230 | | | |
| 6231 | fg.wip.cursor = .{ .block = continue_block }; | | |
| 6232 | } | | |
| 6233 | if (is_unused) return .none; | | |
| 6234 | if (!payload_has_bits) return if (operand_is_ptr) err_union else .none; | | |
| 6235 | const offset = try errUnionPayloadOffset(payload_ty, pt); | | |
| 6236 | if (operand_is_ptr) { | | |
| 6237 | return fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, ""); | | |
| 6238 | } else if (isByRef(err_union_ty, zcu)) { | | |
| 6239 | const payload_ptr = try fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, ""); | | |
| 6240 | if (isByRef(payload_ty, zcu)) { | | |
| 6241 | if (can_elide_load) | | |
| 6242 | return payload_ptr; | | |
| 6243 | | | |
| 6244 | return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal); | | |
| 6245 | } | | |
| 6246 | const load_ty = err_union_llvm_ty.structFields(&o.builder)[offset]; | | |
| 6247 | return fg.wip.load(.normal, load_ty, payload_ptr, payload_align.toLlvm(), ""); | | |
| 6248 | } | | |
| 6249 | return fg.wip.extractValue(err_union, &.{offset}, ""); | | |
| 6250 | } | | |
| 6251 | | | |
| 6252 | fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void { | | |
| 6253 | const o = self.ng.object; | | |
| 6254 | const pt = self.ng.pt; | | |
| 6255 | const zcu = pt.zcu; | | |
| 6256 | | | |
| 6257 | const switch_br = self.air.unwrapSwitch(inst); | | |
| 6258 | | | |
| 6259 | // For `loop_switch_br`, we need these BBs prepared ahead of time to generate dispatches. | | |
| 6260 | // For `switch_br`, they allow us to sometimes generate better IR by sharing a BB between | | |
| 6261 | // scalar and range cases in the same prong. | | |
| 6262 | // +1 for `else` case. This is not the same as the LLVM `else` prong, as that may first contain | | |
| 6263 | // conditionals to handle ranges. | | |
| 6264 | const case_blocks = try self.gpa.alloc(Builder.Function.Block.Index, switch_br.cases_len + 1); | | |
| 6265 | defer self.gpa.free(case_blocks); | | |
| 6266 | // We set incoming as 0 for now, and increment it as we construct dispatches. | | |
| 6267 | for (case_blocks[0 .. case_blocks.len - 1]) |*b| b.* = try self.wip.block(0, "Case"); | | |
| 6268 | case_blocks[case_blocks.len - 1] = try self.wip.block(0, "Default"); | | |
| 6269 | | | |
| 6270 | // There's a special case here to manually generate a jump table in some cases. | | |
| 6271 | // | | |
| 6272 | // Labeled switch in Zig is intended to follow the "direct threading" pattern. We would ideally use a jump | | |
| 6273 | // table, and each `continue` has its own indirect `jmp`, to allow the branch predictor to more accurately | | |
| 6274 | // use data patterns to predict future dispatches. The problem, however, is that LLVM emits fascinatingly | | |
| 6275 | // bad asm for this. Not only does it not share the jump table -- which we really need it to do to prevent | | |
| 6276 | // destroying the cache -- but it also actually generates slightly different jump tables for each case, | | |
| 6277 | // and *a separate conditional branch beforehand* to handle dispatching back to the case we're currently | | |
| 6278 | // within(!!). | | |
| 6279 | // | | |
| 6280 | // This asm is really, really, not what we want. As such, we will construct the jump table manually where | | |
| 6281 | // appropriate (the values are dense and relatively few), and use it when lowering dispatches. | | |
| 6282 | | | |
| 6283 | const jmp_table: ?SwitchDispatchInfo.JmpTable = jmp_table: { | | |
| 6284 | if (!is_dispatch_loop) break :jmp_table null; | | |
| 6285 | | | |
| 6286 | // Workaround for: | | |
| 6287 | // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/lib/MC/WasmObjectWriter.cpp#L560 | | |
| 6288 | // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll | | |
| 6289 | if (zcu.comp.getTarget().cpu.arch.isWasm()) break :jmp_table null; | | |
| 6290 | | | |
| 6291 | // On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just | | |
| 6292 | // about acceptable - it won't fill L1d cache on most CPUs. | | |
| 6293 | const max_table_len = 1024; | | |
| 6294 | | | |
| 6295 | const cond_ty = self.typeOf(switch_br.operand); | | |
| 6296 | switch (cond_ty.zigTypeTag(zcu)) { | | |
| 6297 | .bool, .pointer => break :jmp_table null, | | |
| 6298 | .@"enum", .int, .error_set, .@"struct", .@"union" => {}, | | |
| 6299 | else => unreachable, | | |
| 6300 | } | | |
| 6301 | | | |
| 6302 | if (cond_ty.intInfo(zcu).signedness == .signed) break :jmp_table null; | | |
| 6303 | | | |
| 6304 | // Don't worry about the size of the type -- it's irrelevant, because the prong values could be fairly dense. | | |
| 6305 | // If they are, then we will construct a jump table. | | |
| 6306 | const min, const max = self.switchCaseItemRange(switch_br) orelse break :jmp_table null; | | |
| 6307 | const min_int = min.getUnsignedInt(zcu) orelse break :jmp_table null; | | |
| 6308 | const max_int = max.getUnsignedInt(zcu) orelse break :jmp_table null; | | |
| 6309 | const table_len = max_int - min_int + 1; | | |
| 6310 | if (table_len > max_table_len) break :jmp_table null; | | |
| 6311 | | | |
| 6312 | const table_elems = try self.gpa.alloc(Builder.Constant, @intCast(table_len)); | | |
| 6313 | defer self.gpa.free(table_elems); | | |
| 6314 | | | |
| 6315 | // Set them all to the `else` branch, then iterate over the AIR switch | | |
| 6316 | // and replace all values which correspond to other prongs. | | |
| 6317 | @memset(table_elems, try o.builder.blockAddrConst( | | |
| 6318 | self.wip.function, | | |
| 6319 | case_blocks[case_blocks.len - 1], | | |
| 6320 | )); | | |
| 6321 | var item_count: u32 = 0; | | |
| 6322 | var it = switch_br.iterateCases(); | | |
| 6323 | while (it.next()) |case| { | | |
| 6324 | const case_block = case_blocks[case.idx]; | | |
| 6325 | const case_block_addr = try o.builder.blockAddrConst( | | |
| 6326 | self.wip.function, | | |
| 6327 | case_block, | | |
| 6328 | ); | | |
| 6329 | for (case.items) |item| { | | |
| 6330 | const val = Value.fromInterned(item.toInterned().?); | | |
| 6331 | const table_idx = val.toUnsignedInt(zcu) - min_int; | | |
| 6332 | table_elems[@intCast(table_idx)] = case_block_addr; | | |
| 6333 | item_count += 1; | | |
| 6334 | } | | |
| 6335 | for (case.ranges) |range| { | | |
| 6336 | const low = Value.fromInterned(range[0].toInterned().?); | | |
| 6337 | const high = Value.fromInterned(range[1].toInterned().?); | | |
| 6338 | const low_idx = low.toUnsignedInt(zcu) - min_int; | | |
| 6339 | const high_idx = high.toUnsignedInt(zcu) - min_int; | | |
| 6340 | @memset(table_elems[@intCast(low_idx)..@intCast(high_idx + 1)], case_block_addr); | | |
| 6341 | item_count += @intCast(high_idx + 1 - low_idx); | | |
| 6342 | } | | |
| 6343 | } | | |
| 6344 | | | |
| 6345 | const table_llvm_ty = try o.builder.arrayType(table_elems.len, .ptr); | | |
| 6346 | const table_val = try o.builder.arrayConst(table_llvm_ty, table_elems); | | |
| 6347 | | | |
| 6348 | const table_variable = try o.builder.addVariable( | | |
| 6349 | try o.builder.strtabStringFmt("__jmptab_{d}", .{@intFromEnum(inst)}), | | |
| 6350 | table_llvm_ty, | | |
| 6351 | .default, | | |
| 6352 | ); | | |
| 6353 | try table_variable.setInitializer(table_val, &o.builder); | | |
| 6354 | table_variable.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); | | |
| 6355 | table_variable.setUnnamedAddr(.unnamed_addr, &o.builder); | | |
| 6356 | | | |
| 6357 | const table_includes_else = item_count != table_len; | | |
| 6358 | | | |
| 6359 | break :jmp_table .{ | | |
| 6360 | .min = try o.lowerValue(pt, min.toIntern()), | | |
| 6361 | .max = try o.lowerValue(pt, max.toIntern()), | | |
| 6362 | .in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) { | | |
| 6363 | .none, .cold => .none, | | |
| 6364 | .unpredictable => .unpredictable, | | |
| 6365 | .likely => .likely, | | |
| 6366 | .unlikely => .unlikely, | | |
| 6367 | }, | | |
| 6368 | .table = table_variable.toConst(&o.builder), | | |
| 6369 | .table_includes_else = table_includes_else, | | |
| 6370 | }; | | |
| 6371 | }; | | |
| 6372 | | | |
| 6373 | const weights: Builder.Function.Instruction.BrCond.Weights = weights: { | | |
| 6374 | if (jmp_table != null) break :weights .none; // not used | | |
| 6375 | | | |
| 6376 | // First pass. If any weights are `.unpredictable`, unpredictable. | | |
| 6377 | // If all are `.none` or `.cold`, none. | | |
| 6378 | var any_likely = false; | | |
| 6379 | for (0..switch_br.cases_len) |case_idx| { | | |
| 6380 | switch (switch_br.getHint(@intCast(case_idx))) { | | |
| 6381 | .none, .cold => {}, | | |
| 6382 | .likely, .unlikely => any_likely = true, | | |
| 6383 | .unpredictable => break :weights .unpredictable, | | |
| 6384 | } | | |
| 6385 | } | | |
| 6386 | switch (switch_br.getElseHint()) { | | |
| 6387 | .none, .cold => {}, | | |
| 6388 | .likely, .unlikely => any_likely = true, | | |
| 6389 | .unpredictable => break :weights .unpredictable, | | |
| 6390 | } | | |
| 6391 | if (!any_likely) break :weights .none; | | |
| 6392 | | | |
| 6393 | const llvm_cases_len = llvm_cases_len: { | | |
| 6394 | var len: u32 = 0; | | |
| 6395 | var it = switch_br.iterateCases(); | | |
| 6396 | while (it.next()) |case| len += @intCast(case.items.len); | | |
| 6397 | break :llvm_cases_len len; | | |
| 6398 | }; | | |
| 6399 | | | |
| 6400 | var weights = try self.gpa.alloc(Builder.Metadata, 1 + llvm_cases_len + 1); | | |
| 6401 | defer self.gpa.free(weights); | | |
| 6402 | var weight_idx: usize = 0; | | |
| 6403 | | | |
| 6404 | const branch_weights_str = try o.builder.metadataString("branch_weights"); | | |
| 6405 | weights[weight_idx] = branch_weights_str.toMetadata(); | | |
| 6406 | weight_idx += 1; | | |
| 6407 | | | |
| 6408 | const else_weight: u32 = switch (switch_br.getElseHint()) { | | |
| 6409 | .unpredictable => unreachable, | | |
| 6410 | .none, .cold => 1000, | | |
| 6411 | .likely => 2000, | | |
| 6412 | .unlikely => 1, | | |
| 6413 | }; | | |
| 6414 | weights[weight_idx] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight)); | | |
| 6415 | weight_idx += 1; | | |
| 6416 | | | |
| 6417 | var it = switch_br.iterateCases(); | | |
| 6418 | while (it.next()) |case| { | | |
| 6419 | const weight_val: u32 = switch (switch_br.getHint(case.idx)) { | | |
| 6420 | .unpredictable => unreachable, | | |
| 6421 | .none, .cold => 1000, | | |
| 6422 | .likely => 2000, | | |
| 6423 | .unlikely => 1, | | |
| 6424 | }; | | |
| 6425 | const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val)); | | |
| 6426 | @memset(weights[weight_idx..][0..case.items.len], weight_meta); | | |
| 6427 | weight_idx += case.items.len; | | |
| 6428 | } | | |
| 6429 | | | |
| 6430 | assert(weight_idx == weights.len); | | |
| 6431 | break :weights .fromMetadata(try o.builder.metadataTuple(weights)); | | |
| 6432 | }; | | |
| 6433 | | | |
| 6434 | const dispatch_info: SwitchDispatchInfo = .{ | | |
| 6435 | .case_blocks = case_blocks, | | |
| 6436 | .switch_weights = weights, | | |
| 6437 | .jmp_table = jmp_table, | | |
| 6438 | }; | | |
| 6439 | | | |
| 6440 | if (is_dispatch_loop) { | | |
| 6441 | try self.switch_dispatch_info.putNoClobber(self.gpa, inst, dispatch_info); | | |
| 6442 | } | | |
| 6443 | defer if (is_dispatch_loop) { | | |
| 6444 | assert(self.switch_dispatch_info.remove(inst)); | | |
| 6445 | }; | | |
| 6446 | | | |
| 6447 | // Generate the initial dispatch. | | |
| 6448 | // If this is a simple `switch_br`, this is the only dispatch. | | |
| 6449 | try self.lowerSwitchDispatch(inst, switch_br.operand, dispatch_info); | | |
| 6450 | | | |
| 6451 | // Iterate the cases and generate their bodies. | | |
| 6452 | var it = switch_br.iterateCases(); | | |
| 6453 | while (it.next()) |case| { | | |
| 6454 | const case_block = case_blocks[case.idx]; | | |
| 6455 | self.wip.cursor = .{ .block = case_block }; | | |
| 6456 | if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold(); | | |
| 6457 | try self.genBodyDebugScope(null, case.body, .none); | | |
| 6458 | } | | |
| 6459 | self.wip.cursor = .{ .block = case_blocks[case_blocks.len - 1] }; | | |
| 6460 | const else_body = it.elseBody(); | | |
| 6461 | if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold(); | | |
| 6462 | if (else_body.len > 0) { | | |
| 6463 | try self.genBodyDebugScope(null, it.elseBody(), .none); | | |
| 6464 | } else { | | |
| 6465 | _ = try self.wip.@"unreachable"(); | | |
| 6466 | } | | |
| 6467 | } | | |
| 6468 | | | |
| 6469 | fn switchCaseItemRange(self: *FuncGen, switch_br: Air.UnwrappedSwitch) ?[2]Value { | | |
| 6470 | const zcu = self.ng.pt.zcu; | | |
| 6471 | var it = switch_br.iterateCases(); | | |
| 6472 | var min: ?Value = null; | | |
| 6473 | var max: ?Value = null; | | |
| 6474 | while (it.next()) |case| { | | |
| 6475 | for (case.items) |item| { | | |
| 6476 | const val = Value.fromInterned(item.toInterned().?); | | |
| 6477 | const low = if (min) |m| val.compareHetero(.lt, m, zcu) else true; | | |
| 6478 | const high = if (max) |m| val.compareHetero(.gt, m, zcu) else true; | | |
| 6479 | if (low) min = val; | | |
| 6480 | if (high) max = val; | | |
| 6481 | } | | |
| 6482 | for (case.ranges) |range| { | | |
| 6483 | const vals: [2]Value = .{ | | |
| 6484 | Value.fromInterned(range[0].toInterned().?), | | |
| 6485 | Value.fromInterned(range[1].toInterned().?), | | |
| 6486 | }; | | |
| 6487 | const low = if (min) |m| vals[0].compareHetero(.lt, m, zcu) else true; | | |
| 6488 | const high = if (max) |m| vals[1].compareHetero(.gt, m, zcu) else true; | | |
| 6489 | if (low) min = vals[0]; | | |
| 6490 | if (high) max = vals[1]; | | |
| 6491 | } | | |
| 6492 | } | | |
| 6493 | if (min == null) { | | |
| 6494 | assert(max == null); | | |
| 6495 | return null; | | |
| 6496 | } | | |
| 6497 | return .{ min.?, max.? }; | | |
| 6498 | } | | |
| 6499 | | | |
| 6500 | fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void { | | |
| 6501 | const block = self.air.unwrapBlock(inst); | | |
| 6502 | const body = block.body; | | |
| 6503 | const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time | | |
| 6504 | _ = try self.wip.br(loop_block); | | |
| 6505 | | | |
| 6506 | try self.loops.putNoClobber(self.gpa, inst, loop_block); | | |
| 6507 | defer assert(self.loops.remove(inst)); | | |
| 6508 | | | |
| 6509 | self.wip.cursor = .{ .block = loop_block }; | | |
| 6510 | try self.genBodyDebugScope(null, body, .none); | | |
| 6511 | } | | |
| 6512 | | | |
| 6513 | fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6514 | const o = self.ng.object; | | |
| 6515 | const pt = self.ng.pt; | | |
| 6516 | const zcu = pt.zcu; | | |
| 6517 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 6518 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 6519 | const array_ty = operand_ty.childType(zcu); | | |
| 6520 | const llvm_usize = try o.lowerType(pt, Type.usize); | | |
| 6521 | const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu)); | | |
| 6522 | const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst)); | | |
| 6523 | const operand = try self.resolveInst(ty_op.operand); | | |
| 6524 | if (!array_ty.hasRuntimeBits(zcu)) | | |
| 6525 | return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, ""); | | |
| 6526 | const ptr = try self.wip.gep(.inbounds, try o.lowerType(pt, array_ty), operand, &.{ | | |
| 6527 | try o.builder.intValue(llvm_usize, 0), try o.builder.intValue(llvm_usize, 0), | | |
| 6528 | }, ""); | | |
| 6529 | return self.wip.buildAggregate(slice_llvm_ty, &.{ ptr, len }, ""); | | |
| 6530 | } | | |
| 6531 | | | |
| 6532 | fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6533 | const o = self.ng.object; | | |
| 6534 | const pt = self.ng.pt; | | |
| 6535 | const zcu = pt.zcu; | | |
| 6536 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 6537 | | | |
| 6538 | const operand = try self.resolveInst(ty_op.operand); | | |
| 6539 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 6540 | const operand_scalar_ty = operand_ty.scalarType(zcu); | | |
| 6541 | const is_signed_int = operand_scalar_ty.isSignedInt(zcu); | | |
| 6542 | | | |
| 6543 | const dest_ty = self.typeOfIndex(inst); | | |
| 6544 | const dest_scalar_ty = dest_ty.scalarType(zcu); | | |
| 6545 | const dest_llvm_ty = try o.lowerType(pt, dest_ty); | | |
| 6546 | const target = zcu.getTarget(); | | |
| 6547 | | | |
| 6548 | if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv( | | |
| 6549 | if (is_signed_int) .signed else .unsigned, | | |
| 6550 | operand, | | |
| 6551 | dest_llvm_ty, | | |
| 6552 | "", | | |
| 6553 | ); | | |
| 6554 | | | |
| 6555 | const rt_int_bits = compilerRtIntBits(@intCast(operand_scalar_ty.bitSize(zcu))) orelse { | | |
| 6556 | return self.todo("float_from_int from '{f}' without intrinsics", .{operand_scalar_ty.fmt(pt)}); | | |
| 6557 | }; | | |
| 6558 | const rt_int_ty = try o.builder.intType(rt_int_bits); | | |
| 6559 | var extended = try self.wip.conv( | | |
| 6560 | if (is_signed_int) .signed else .unsigned, | | |
| 6561 | operand, | | |
| 6562 | rt_int_ty, | | |
| 6563 | "", | | |
| 6564 | ); | | |
| 6565 | const dest_bits = dest_scalar_ty.floatBits(target); | | |
| 6566 | const compiler_rt_operand_abbrev = compilerRtIntAbbrev(rt_int_bits); | | |
| 6567 | const compiler_rt_dest_abbrev = compilerRtFloatAbbrev(dest_bits); | | |
| 6568 | const sign_prefix = if (is_signed_int) "" else "un"; | | |
| 6569 | const fn_name = try o.builder.strtabStringFmt("__float{s}{s}i{s}f", .{ | | |
| 6570 | sign_prefix, | | |
| 6571 | compiler_rt_operand_abbrev, | | |
| 6572 | compiler_rt_dest_abbrev, | | |
| 6573 | }); | | |
| 6574 | | | |
| 6575 | var param_type = rt_int_ty; | | |
| 6576 | if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) { | | |
| 6577 | // On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard | | |
| 6578 | // i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have. | | |
| 6579 | param_type = try o.builder.vectorType(.normal, 2, .i64); | | |
| 6580 | extended = try self.wip.cast(.bitcast, extended, param_type, ""); | | |
| 6581 | } | | |
| 6582 | | | |
| 6583 | const libc_fn = try self.getLibcFunction(fn_name, &.{param_type}, dest_llvm_ty); | | |
| 6584 | return self.wip.call( | | |
| 6585 | .normal, | | |
| 6586 | .ccc, | | |
| 6587 | .none, | | |
| 6588 | libc_fn.typeOf(&o.builder), | | |
| 6589 | libc_fn.toValue(&o.builder), | | |
| 6590 | &.{extended}, | | |
| 6591 | "", | | |
| 6592 | ); | | |
| 6593 | } | | |
| 6594 | | | |
| 6595 | fn airIntFromFloat( | | |
| 6596 | self: *FuncGen, | | |
| 6597 | inst: Air.Inst.Index, | | |
| 6598 | fast: Builder.FastMathKind, | | |
| 6599 | ) !Builder.Value { | | |
| 6600 | _ = fast; | | |
| 6601 | | | |
| 6602 | const o = self.ng.object; | | |
| 6603 | const pt = self.ng.pt; | | |
| 6604 | const zcu = pt.zcu; | | |
| 6605 | const target = zcu.getTarget(); | | |
| 6606 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 6607 | | | |
| 6608 | const operand = try self.resolveInst(ty_op.operand); | | |
| 6609 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 6610 | const operand_scalar_ty = operand_ty.scalarType(zcu); | | |
| 6611 | | | |
| 6612 | const dest_ty = self.typeOfIndex(inst); | | |
| 6613 | const dest_scalar_ty = dest_ty.scalarType(zcu); | | |
| 6614 | const dest_llvm_ty = try o.lowerType(pt, dest_ty); | | |
| 6615 | | | |
| 6616 | if (intrinsicsAllowed(operand_scalar_ty, target)) { | | |
| 6617 | // TODO set fast math flag | | |
| 6618 | return self.wip.conv( | | |
| 6619 | if (dest_scalar_ty.isSignedInt(zcu)) .signed else .unsigned, | | |
| 6620 | operand, | | |
| 6621 | dest_llvm_ty, | | |
| 6622 | "", | | |
| 6623 | ); | | |
| 6624 | } | | |
| 6625 | | | |
| 6626 | const rt_int_bits = compilerRtIntBits(@intCast(dest_scalar_ty.bitSize(zcu))) orelse { | | |
| 6627 | return self.todo("int_from_float to '{f}' without intrinsics", .{dest_scalar_ty.fmt(pt)}); | | |
| 6628 | }; | | |
| 6629 | const ret_ty = try o.builder.intType(rt_int_bits); | | |
| 6630 | const libc_ret_ty = if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) b: { | | |
| 6631 | // On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard | | |
| 6632 | // i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have. | | |
| 6633 | break :b try o.builder.vectorType(.normal, 2, .i64); | | |
| 6634 | } else ret_ty; | | |
| 6635 | | | |
| 6636 | const operand_bits = operand_scalar_ty.floatBits(target); | | |
| 6637 | const compiler_rt_operand_abbrev = compilerRtFloatAbbrev(operand_bits); | | |
| 6638 | | | |
| 6639 | const compiler_rt_dest_abbrev = compilerRtIntAbbrev(rt_int_bits); | | |
| 6640 | const sign_prefix = if (dest_scalar_ty.isSignedInt(zcu)) "" else "uns"; | | |
| 6641 | | | |
| 6642 | const fn_name = try o.builder.strtabStringFmt("__fix{s}{s}f{s}i", .{ | | |
| 6643 | sign_prefix, | | |
| 6644 | compiler_rt_operand_abbrev, | | |
| 6645 | compiler_rt_dest_abbrev, | | |
| 6646 | }); | | |
| 6647 | | | |
| 6648 | const operand_llvm_ty = try o.lowerType(pt, operand_ty); | | |
| 6649 | const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty); | | |
| 6650 | var result = try self.wip.call( | | |
| 6651 | .normal, | | |
| 6652 | .ccc, | | |
| 6653 | .none, | | |
| 6654 | libc_fn.typeOf(&o.builder), | | |
| 6655 | libc_fn.toValue(&o.builder), | | |
| 6656 | &.{operand}, | | |
| 6657 | "", | | |
| 6658 | ); | | |
| 6659 | | | |
| 6660 | if (libc_ret_ty != ret_ty) result = try self.wip.cast(.bitcast, result, ret_ty, ""); | | |
| 6661 | if (ret_ty != dest_llvm_ty) result = try self.wip.cast(.trunc, result, dest_llvm_ty, ""); | | |
| 6662 | return result; | | |
| 6663 | } | | |
| 6664 | | | |
| 6665 | fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value { | | |
| 6666 | const zcu = fg.ng.pt.zcu; | | |
| 6667 | return if (ty.isSlice(zcu)) fg.wip.extractValue(ptr, &.{0}, "") else ptr; | | |
| 6668 | } | | |
| 6669 | | | |
| 6670 | fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value { | | |
| 6671 | const o = fg.ng.object; | | |
| 6672 | const pt = fg.ng.pt; | | |
| 6673 | const zcu = pt.zcu; | | |
| 6674 | const llvm_usize = try o.lowerType(pt, Type.usize); | | |
| 6675 | switch (ty.ptrSize(zcu)) { | | |
| 6676 | .slice => { | | |
| 6677 | const len = try fg.wip.extractValue(ptr, &.{1}, ""); | | |
| 6678 | const elem_ty = ty.childType(zcu); | | |
| 6679 | const abi_size = elem_ty.abiSize(zcu); | | |
| 6680 | if (abi_size == 1) return len; | | |
| 6681 | const abi_size_llvm_val = try o.builder.intValue(llvm_usize, abi_size); | | |
| 6682 | return fg.wip.bin(.@"mul nuw", len, abi_size_llvm_val, ""); | | |
| 6683 | }, | | |
| 6684 | .one => { | | |
| 6685 | const array_ty = ty.childType(zcu); | | |
| 6686 | const elem_ty = array_ty.childType(zcu); | | |
| 6687 | const abi_size = elem_ty.abiSize(zcu); | | |
| 6688 | return o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu) * abi_size); | | |
| 6689 | }, | | |
| 6690 | .many, .c => unreachable, | | |
| 6691 | } | | |
| 6692 | } | | |
| 6693 | | | |
| 6694 | fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Value { | | |
| 6695 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 6696 | const operand = try self.resolveInst(ty_op.operand); | | |
| 6697 | return self.wip.extractValue(operand, &.{index}, ""); | | |
| 6698 | } | | |
| 6699 | | | |
| 6700 | fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: c_uint) !Builder.Value { | | |
| 6701 | const o = self.ng.object; | | |
| 6702 | const pt = self.ng.pt; | | |
| 6703 | const zcu = pt.zcu; | | |
| 6704 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 6705 | const slice_ptr = try self.resolveInst(ty_op.operand); | | |
| 6706 | const slice_ptr_ty = self.typeOf(ty_op.operand); | | |
| 6707 | const slice_llvm_ty = try o.lowerType(pt, slice_ptr_ty.childType(zcu)); | | |
| 6708 | | | |
| 6709 | return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, ""); | | |
| 6710 | } | | |
| 6711 | | | |
| 6712 | fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6713 | const o = self.ng.object; | | |
| 6714 | const pt = self.ng.pt; | | |
| 6715 | const zcu = pt.zcu; | | |
| 6716 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 6717 | const slice_ty = self.typeOf(bin_op.lhs); | | |
| 6718 | const slice = try self.resolveInst(bin_op.lhs); | | |
| 6719 | const index = try self.resolveInst(bin_op.rhs); | | |
| 6720 | const slice_info = slice_ty.ptrInfo(zcu); | | |
| 6721 | assert(slice_info.flags.size == .slice); | | |
| 6722 | const elem_ty: Type = .fromInterned(slice_info.child); | | |
| 6723 | const llvm_elem_ty = try o.lowerType(pt, elem_ty); | | |
| 6724 | const base_ptr = try self.wip.extractValue(slice, &.{0}, ""); | | |
| 6725 | const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, ""); | | |
| 6726 | const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)); | | |
| 6727 | const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal; | | |
| 6728 | self.maybeMarkAllowZeroAccess(slice_info); | | |
| 6729 | if (isByRef(elem_ty, zcu)) { | | |
| 6730 | return self.loadByRef(ptr, elem_ty, elem_align.toLlvm(), access_kind); | | |
| 6731 | } else { | | |
| 6732 | return self.loadTruncate(access_kind, elem_ty, ptr, elem_align.toLlvm()); | | |
| 6733 | } | | |
| 6734 | } | | |
| 6735 | | | |
| 6736 | fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6737 | const o = self.ng.object; | | |
| 6738 | const pt = self.ng.pt; | | |
| 6739 | const zcu = pt.zcu; | | |
| 6740 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 6741 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; | | |
| 6742 | const slice_ty = self.typeOf(bin_op.lhs); | | |
| 6743 | | | |
| 6744 | const slice = try self.resolveInst(bin_op.lhs); | | |
| 6745 | const index = try self.resolveInst(bin_op.rhs); | | |
| 6746 | const llvm_elem_ty = try o.lowerType(pt, slice_ty.childType(zcu)); | | |
| 6747 | const base_ptr = try self.wip.extractValue(slice, &.{0}, ""); | | |
| 6748 | return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, ""); | | |
| 6749 | } | | |
| 6750 | | | |
| 6751 | fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6752 | const o = self.ng.object; | | |
| 6753 | const pt = self.ng.pt; | | |
| 6754 | const zcu = pt.zcu; | | |
| 6755 | | | |
| 6756 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 6757 | const array_ty = self.typeOf(bin_op.lhs); | | |
| 6758 | const array_llvm_val = try self.resolveInst(bin_op.lhs); | | |
| 6759 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 6760 | const array_llvm_ty = try o.lowerType(pt, array_ty); | | |
| 6761 | const elem_ty = array_ty.childType(zcu); | | |
| 6762 | if (isByRef(array_ty, zcu)) { | | |
| 6763 | const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{ | | |
| 6764 | try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), | | |
| 6765 | rhs, | | |
| 6766 | }, ""); | | |
| 6767 | if (isByRef(elem_ty, zcu)) { | | |
| 6768 | const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm(); | | |
| 6769 | return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal); | | |
| 6770 | } else { | | |
| 6771 | return self.loadTruncate(.normal, elem_ty, elem_ptr, .default); | | |
| 6772 | } | | |
| 6773 | } | | |
| 6774 | | | |
| 6775 | // This branch can be reached for vectors, which are always by-value. | | |
| 6776 | return self.wip.extractElement(array_llvm_val, rhs, ""); | | |
| 6777 | } | | |
| 6778 | | | |
| 6779 | fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6780 | const o = self.ng.object; | | |
| 6781 | const pt = self.ng.pt; | | |
| 6782 | const zcu = pt.zcu; | | |
| 6783 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 6784 | const ptr_ty = self.typeOf(bin_op.lhs); | | |
| 6785 | const elem_ty = ptr_ty.indexableElem(zcu); | | |
| 6786 | const llvm_elem_ty = try o.lowerType(pt, elem_ty); | | |
| 6787 | const base_ptr = try self.resolveInst(bin_op.lhs); | | |
| 6788 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 6789 | const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, ""); | | |
| 6790 | if (isByRef(elem_ty, zcu)) { | | |
| 6791 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); | | |
| 6792 | const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm(); | | |
| 6793 | return self.loadByRef(ptr, elem_ty, ptr_align, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal); | | |
| 6794 | } | | |
| 6795 | | | |
| 6796 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); | | |
| 6797 | | | |
| 6798 | return self.load(ptr, ptr_ty); | | |
| 6799 | } | | |
| 6800 | | | |
| 6801 | fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6802 | const o = self.ng.object; | | |
| 6803 | const pt = self.ng.pt; | | |
| 6804 | const zcu = pt.zcu; | | |
| 6805 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 6806 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; | | |
| 6807 | const ptr_ty = self.typeOf(bin_op.lhs); | | |
| 6808 | const elem_ty = ptr_ty.indexableElem(zcu); | | |
| 6809 | assert(elem_ty.hasRuntimeBits(zcu)); | | |
| 6810 | | | |
| 6811 | const base_ptr = try self.resolveInst(bin_op.lhs); | | |
| 6812 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 6813 | | | |
| 6814 | const elem_ptr = ty_pl.ty.toType(); | | |
| 6815 | if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr; | | |
| 6816 | | | |
| 6817 | const llvm_elem_ty = try o.lowerType(pt, elem_ty); | | |
| 6818 | return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, ""); | | |
| 6819 | } | | |
| 6820 | | | |
| 6821 | fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6822 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 6823 | const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data; | | |
| 6824 | const struct_ptr = try self.resolveInst(struct_field.struct_operand); | | |
| 6825 | const struct_ptr_ty = self.typeOf(struct_field.struct_operand); | | |
| 6826 | return self.fieldPtr(struct_ptr, struct_ptr_ty, struct_field.field_index); | | |
| 6827 | } | | |
| 6828 | | | |
| 6829 | fn airStructFieldPtrIndex( | | |
| 6830 | self: *FuncGen, | | |
| 6831 | inst: Air.Inst.Index, | | |
| 6832 | field_index: u32, | | |
| 6833 | ) !Builder.Value { | | |
| 6834 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 6835 | const struct_ptr = try self.resolveInst(ty_op.operand); | | |
| 6836 | const struct_ptr_ty = self.typeOf(ty_op.operand); | | |
| 6837 | return self.fieldPtr(struct_ptr, struct_ptr_ty, field_index); | | |
| 6838 | } | | |
| 6839 | | | |
| 6840 | fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6841 | const o = self.ng.object; | | |
| 6842 | const pt = self.ng.pt; | | |
| 6843 | const zcu = pt.zcu; | | |
| 6844 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 6845 | const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data; | | |
| 6846 | const struct_ty = self.typeOf(struct_field.struct_operand); | | |
| 6847 | const struct_llvm_val = try self.resolveInst(struct_field.struct_operand); | | |
| 6848 | const field_index = struct_field.field_index; | | |
| 6849 | const field_ty = struct_ty.fieldType(field_index, zcu); | | |
| 6850 | if (!field_ty.hasRuntimeBits(zcu)) return .none; | | |
| 6851 | | | |
| 6852 | if (!isByRef(struct_ty, zcu)) { | | |
| 6853 | assert(!isByRef(field_ty, zcu)); | | |
| 6854 | switch (struct_ty.zigTypeTag(zcu)) { | | |
| 6855 | .@"struct" => switch (struct_ty.containerLayout(zcu)) { | | |
| 6856 | .@"packed" => { | | |
| 6857 | const struct_type = zcu.typeToStruct(struct_ty).?; | | |
| 6858 | const bit_offset = zcu.structPackedFieldBitOffset(struct_type, field_index); | | |
| 6859 | const containing_int = struct_llvm_val; | | |
| 6860 | const shift_amt = | | |
| 6861 | try o.builder.intValue(containing_int.typeOfWip(&self.wip), bit_offset); | | |
| 6862 | const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, ""); | | |
| 6863 | const elem_llvm_ty = try o.lowerType(pt, field_ty); | | |
| 6864 | if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) { | | |
| 6865 | const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu))); | | |
| 6866 | const truncated_int = | | |
| 6867 | try self.wip.cast(.trunc, shifted_value, same_size_int, ""); | | |
| 6868 | return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); | | |
| 6869 | } | | |
| 6870 | return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, ""); | | |
| 6871 | }, | | |
| 6872 | else => { | | |
| 6873 | const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; | | |
| 6874 | return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, ""); | | |
| 6875 | }, | | |
| 6876 | }, | | |
| 6877 | .@"union" => { | | |
| 6878 | assert(struct_ty.containerLayout(zcu) == .@"packed"); | | |
| 6879 | const containing_int = struct_llvm_val; | | |
| 6880 | const elem_llvm_ty = try o.lowerType(pt, field_ty); | | |
| 6881 | if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) { | | |
| 6882 | const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu))); | | |
| 6883 | const truncated_int = | | |
| 6884 | try self.wip.cast(.trunc, containing_int, same_size_int, ""); | | |
| 6885 | return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); | | |
| 6886 | } | | |
| 6887 | return self.wip.cast(.trunc, containing_int, elem_llvm_ty, ""); | | |
| 6888 | }, | | |
| 6889 | else => unreachable, | | |
| 6890 | } | | |
| 6891 | } | | |
| 6892 | | | |
| 6893 | switch (struct_ty.zigTypeTag(zcu)) { | | |
| 6894 | .@"struct" => { | | |
| 6895 | const layout = struct_ty.containerLayout(zcu); | | |
| 6896 | assert(layout != .@"packed"); | | |
| 6897 | const struct_llvm_ty = try o.lowerType(pt, struct_ty); | | |
| 6898 | const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; | | |
| 6899 | const field_ptr = | | |
| 6900 | try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, ""); | | |
| 6901 | const explicit_alignment = struct_ty.explicitFieldAlignment(field_index, zcu); | | |
| 6902 | const field_ptr_ty = try pt.ptrType(.{ | | |
| 6903 | .child = field_ty.toIntern(), | | |
| 6904 | .flags = .{ .alignment = explicit_alignment }, | | |
| 6905 | }); | | |
| 6906 | if (isByRef(field_ty, zcu)) { | | |
| 6907 | const alignment = switch (explicit_alignment) { | | |
| 6908 | .none => field_ty.abiAlignment(zcu), | | |
| 6909 | else => |a| a, | | |
| 6910 | }; | | |
| 6911 | return self.loadByRef(field_ptr, field_ty, alignment.toLlvm(), .normal); | | |
| 6912 | } else { | | |
| 6913 | return self.load(field_ptr, field_ptr_ty); | | |
| 6914 | } | | |
| 6915 | }, | | |
| 6916 | .@"union" => { | | |
| 6917 | const union_llvm_ty = try o.lowerType(pt, struct_ty); | | |
| 6918 | const layout = struct_ty.unionGetLayout(zcu); | | |
| 6919 | const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align)); | | |
| 6920 | const field_ptr = | | |
| 6921 | try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, ""); | | |
| 6922 | const payload_alignment = layout.payload_align.toLlvm(); | | |
| 6923 | if (isByRef(field_ty, zcu)) { | | |
| 6924 | return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal); | | |
| 6925 | } else { | | |
| 6926 | return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment); | | |
| 6927 | } | | |
| 6928 | }, | | |
| 6929 | else => unreachable, | | |
| 6930 | } | | |
| 6931 | } | | |
| 6932 | | | |
| 6933 | fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6934 | const o = self.ng.object; | | |
| 6935 | const pt = self.ng.pt; | | |
| 6936 | const zcu = pt.zcu; | | |
| 6937 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 6938 | const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; | | |
| 6939 | | | |
| 6940 | const field_ptr = try self.resolveInst(extra.field_ptr); | | |
| 6941 | | | |
| 6942 | const parent_ty = ty_pl.ty.toType().childType(zcu); | | |
| 6943 | const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu); | | |
| 6944 | if (field_offset == 0) return field_ptr; | | |
| 6945 | | | |
| 6946 | const res_ty = try o.lowerType(pt, ty_pl.ty.toType()); | | |
| 6947 | const llvm_usize = try o.lowerType(pt, Type.usize); | | |
| 6948 | | | |
| 6949 | const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, ""); | | |
| 6950 | const base_ptr_int = try self.wip.bin( | | |
| 6951 | .@"sub nuw", | | |
| 6952 | field_ptr_int, | | |
| 6953 | try o.builder.intValue(llvm_usize, field_offset), | | |
| 6954 | "", | | |
| 6955 | ); | | |
| 6956 | return self.wip.cast(.inttoptr, base_ptr_int, res_ty, ""); | | |
| 6957 | } | | |
| 6958 | | | |
| 6959 | fn airNot(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6960 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 6961 | const operand = try self.resolveInst(ty_op.operand); | | |
| 6962 | | | |
| 6963 | return self.wip.not(operand, ""); | | |
| 6964 | } | | |
| 6965 | | | |
| 6966 | fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) !void { | | |
| 6967 | _ = inst; | | |
| 6968 | _ = try self.wip.@"unreachable"(); | | |
| 6969 | } | | |
| 6970 | | | |
| 6971 | fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6972 | const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt; | | |
| 6973 | self.prev_dbg_line = @intCast(self.base_line + dbg_stmt.line + 1); | | |
| 6974 | self.prev_dbg_column = @intCast(dbg_stmt.column + 1); | | |
| 6975 | | | |
| 6976 | self.wip.debug_location = .{ .location = .{ | | |
| 6977 | .line = self.prev_dbg_line, | | |
| 6978 | .column = self.prev_dbg_column, | | |
| 6979 | .scope = self.scope.toOptional(), | | |
| 6980 | .inlined_at = self.inlined_at, | | |
| 6981 | } }; | | |
| 6982 | | | |
| 6983 | return .none; | | |
| 6984 | } | | |
| 6985 | | | |
| 6986 | fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6987 | _ = self; | | |
| 6988 | _ = inst; | | |
| 6989 | return .none; | | |
| 6990 | } | | |
| 6991 | | | |
| 6992 | fn airDbgInlineBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6993 | const block = self.air.unwrapDbgBlock(inst); | | |
| 6994 | self.arg_inline_index = 0; | | |
| 6995 | return self.lowerBlock(inst, block.func, block.body); | | |
| 6996 | } | | |
| 6997 | | | |
| 6998 | fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 6999 | const o = self.ng.object; | | |
| 7000 | const pt = self.ng.pt; | | |
| 7001 | const zcu = pt.zcu; | | |
| 7002 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | | |
| 7003 | const operand = try self.resolveInst(pl_op.operand); | | |
| 7004 | const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); | | |
| 7005 | const ptr_ty = self.typeOf(pl_op.operand); | | |
| 7006 | | | |
| 7007 | const debug_local_var = try o.builder.debugLocalVar( | | |
| 7008 | try o.builder.metadataString(name.toSlice(self.air)), | | |
| 7009 | self.file, | | |
| 7010 | self.scope, | | |
| 7011 | self.prev_dbg_line, | | |
| 7012 | try o.getDebugType(pt, ptr_ty.childType(zcu)), | | |
| 7013 | ); | | |
| 7014 | | | |
| 7015 | _ = try self.wip.callIntrinsic( | | |
| 7016 | .normal, | | |
| 7017 | .none, | | |
| 7018 | .@"dbg.declare", | | |
| 7019 | &.{}, | | |
| 7020 | &.{ | | |
| 7021 | (try self.wip.debugValue(operand)).toValue(), | | |
| 7022 | debug_local_var.toValue(), | | |
| 7023 | (try o.builder.debugExpression(&.{})).toValue(), | | |
| 7024 | }, | | |
| 7025 | "", | | |
| 7026 | ); | | |
| 7027 | | | |
| 7028 | return .none; | | |
| 7029 | } | | |
| 7030 | | | |
| 7031 | fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) !Builder.Value { | | |
| 7032 | const o = self.ng.object; | | |
| 7033 | const pt = self.ng.pt; | | |
| 7034 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | | |
| 7035 | const operand = try self.resolveInst(pl_op.operand); | | |
| 7036 | const operand_ty = self.typeOf(pl_op.operand); | | |
| 7037 | const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); | | |
| 7038 | const name_slice = name.toSlice(self.air); | | |
| 7039 | const metadata_name = if (name_slice.len > 0) try o.builder.metadataString(name_slice) else null; | | |
| 7040 | const debug_local_var = if (is_arg) try o.builder.debugParameter( | | |
| 7041 | metadata_name, | | |
| 7042 | self.file, | | |
| 7043 | self.scope, | | |
| 7044 | self.prev_dbg_line, | | |
| 7045 | try o.getDebugType(pt, operand_ty), | | |
| 7046 | arg_no: { | | |
| 7047 | self.arg_inline_index += 1; | | |
| 7048 | break :arg_no self.arg_inline_index; | | |
| 7049 | }, | | |
| 7050 | ) else try o.builder.debugLocalVar( | | |
| 7051 | metadata_name, | | |
| 7052 | self.file, | | |
| 7053 | self.scope, | | |
| 7054 | self.prev_dbg_line, | | |
| 7055 | try o.getDebugType(pt, operand_ty), | | |
| 7056 | ); | | |
| 7057 | | | |
| 7058 | const zcu = pt.zcu; | | |
| 7059 | const owner_mod = self.ng.ownerModule(); | | |
| 7060 | if (isByRef(operand_ty, zcu)) { | | |
| 7061 | _ = try self.wip.callIntrinsic( | | |
| 7062 | .normal, | | |
| 7063 | .none, | | |
| 7064 | .@"dbg.declare", | | |
| 7065 | &.{}, | | |
| 7066 | &.{ | | |
| 7067 | (try self.wip.debugValue(operand)).toValue(), | | |
| 7068 | debug_local_var.toValue(), | | |
| 7069 | (try o.builder.debugExpression(&.{})).toValue(), | | |
| 7070 | }, | | |
| 7071 | "", | | |
| 7072 | ); | | |
| 7073 | } else if (owner_mod.optimize_mode == .Debug and !self.is_naked) { | | |
| 7074 | // We avoid taking this path for naked functions because there's no guarantee that such | | |
| 7075 | // functions even have a valid stack pointer, making the `alloca` + `store` unsafe. | | |
| 7076 | | | |
| 7077 | const alignment = operand_ty.abiAlignment(zcu).toLlvm(); | | |
| 7078 | const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment); | | |
| 7079 | _ = try self.wip.store(.normal, operand, alloca, alignment); | | |
| 7080 | _ = try self.wip.callIntrinsic( | | |
| 7081 | .normal, | | |
| 7082 | .none, | | |
| 7083 | .@"dbg.declare", | | |
| 7084 | &.{}, | | |
| 7085 | &.{ | | |
| 7086 | (try self.wip.debugValue(alloca)).toValue(), | | |
| 7087 | debug_local_var.toValue(), | | |
| 7088 | (try o.builder.debugExpression(&.{})).toValue(), | | |
| 7089 | }, | | |
| 7090 | "", | | |
| 7091 | ); | | |
| 7092 | } else { | | |
| 7093 | _ = try self.wip.callIntrinsic( | | |
| 7094 | .normal, | | |
| 7095 | .none, | | |
| 7096 | .@"dbg.value", | | |
| 7097 | &.{}, | | |
| 7098 | &.{ | | |
| 7099 | (try self.wip.debugValue(operand)).toValue(), | | |
| 7100 | debug_local_var.toValue(), | | |
| 7101 | (try o.builder.debugExpression(&.{})).toValue(), | | |
| 7102 | }, | | |
| 7103 | "", | | |
| 7104 | ); | | |
| 7105 | } | | |
| 7106 | return .none; | | |
| 7107 | } | | |
| 7108 | | | |
| 7109 | fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 7110 | // Eventually, the Zig compiler needs to be reworked to have inline | | |
| 7111 | // assembly go through the same parsing code regardless of backend, and | | |
| 7112 | // have LLVM-flavored inline assembly be *output* from that assembler. | | |
| 7113 | // We don't have such an assembler implemented yet though. For now, | | |
| 7114 | // this implementation feeds the inline assembly code directly to LLVM. | | |
| 7115 | | | |
| 7116 | const o = self.ng.object; | | |
| 7117 | const unwrapped_asm = self.air.unwrapAsm(inst); | | |
| 7118 | const is_volatile = unwrapped_asm.is_volatile; | | |
| 7119 | const gpa = self.gpa; | | |
| 7120 | | | |
| 7121 | const outputs = unwrapped_asm.outputs; | | |
| 7122 | const inputs = unwrapped_asm.inputs; | | |
| 7123 | | | |
| 7124 | var llvm_constraints: std.ArrayList(u8) = .empty; | | |
| 7125 | defer llvm_constraints.deinit(gpa); | | |
| 7126 | | | |
| 7127 | var arena_allocator = std.heap.ArenaAllocator.init(gpa); | | |
| 7128 | defer arena_allocator.deinit(); | | |
| 7129 | const arena = arena_allocator.allocator(); | | |
| 7130 | | | |
| 7131 | // The exact number of return / parameter values depends on which output values | | |
| 7132 | // are passed by reference as indirect outputs (determined below). | | |
| 7133 | const max_return_count = outputs.len; | | |
| 7134 | const llvm_ret_types = try arena.alloc(Builder.Type, max_return_count); | | |
| 7135 | const llvm_ret_indirect = try arena.alloc(bool, max_return_count); | | |
| 7136 | const llvm_rw_vals = try arena.alloc(Builder.Value, max_return_count); | | |
| 7137 | | | |
| 7138 | const max_param_count = max_return_count + inputs.len + outputs.len; | | |
| 7139 | const llvm_param_types = try arena.alloc(Builder.Type, max_param_count); | | |
| 7140 | const llvm_param_values = try arena.alloc(Builder.Value, max_param_count); | | |
| 7141 | // This stores whether we need to add an elementtype attribute and | | |
| 7142 | // if so, the element type itself. | | |
| 7143 | const llvm_param_attrs = try arena.alloc(Builder.Type, max_param_count); | | |
| 7144 | const pt = self.ng.pt; | | |
| 7145 | const zcu = pt.zcu; | | |
| 7146 | const ip = &zcu.intern_pool; | | |
| 7147 | const target = zcu.getTarget(); | | |
| 7148 | | | |
| 7149 | var llvm_ret_i: usize = 0; | | |
| 7150 | var llvm_param_i: usize = 0; | | |
| 7151 | var total_i: usize = 0; | | |
| 7152 | | | |
| 7153 | var name_map: std.StringArrayHashMapUnmanaged(u16) = .empty; | | |
| 7154 | try name_map.ensureUnusedCapacity(arena, max_param_count); | | |
| 7155 | | | |
| 7156 | var it = unwrapped_asm.iterateOutputs(); | | |
| 7157 | while (it.next()) |output| { | | |
| 7158 | const constraint = output.constraint; | | |
| 7159 | const name = output.name; | | |
| 7160 | | | |
| 7161 | try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 3); | | |
| 7162 | if (total_i != 0) { | | |
| 7163 | llvm_constraints.appendAssumeCapacity(','); | | |
| 7164 | } | | |
| 7165 | llvm_constraints.appendAssumeCapacity('='); | | |
| 7166 | | | |
| 7167 | if (output.operand != .none) { | | |
| 7168 | const output_inst = try self.resolveInst(output.operand); | | |
| 7169 | const output_ty = self.typeOf(output.operand); | | |
| 7170 | assert(output_ty.zigTypeTag(zcu) == .pointer); | | |
| 7171 | const elem_llvm_ty = try o.lowerType(pt, output_ty.childType(zcu)); | | |
| 7172 | | | |
| 7173 | switch (constraint[0]) { | | |
| 7174 | '=' => {}, | | |
| 7175 | '+' => llvm_rw_vals[output.index] = output_inst, | | |
| 7176 | else => return self.todo("unsupported output constraint on output type '{c}'", .{ | | |
| 7177 | constraint[0], | | |
| 7178 | }), | | |
| 7179 | } | | |
| 7180 | | | |
| 7181 | self.maybeMarkAllowZeroAccess(output_ty.ptrInfo(zcu)); | | |
| 7182 | | | |
| 7183 | // Pass any non-return outputs indirectly, if the constraint accepts a memory location | | |
| 7184 | llvm_ret_indirect[output.index] = constraintAllowsMemory(constraint); | | |
| 7185 | if (llvm_ret_indirect[output.index]) { | | |
| 7186 | // Pass the result by reference as an indirect output (e.g. "=*m") | | |
| 7187 | llvm_constraints.appendAssumeCapacity('*'); | | |
| 7188 | | | |
| 7189 | llvm_param_values[llvm_param_i] = output_inst; | | |
| 7190 | llvm_param_types[llvm_param_i] = output_inst.typeOfWip(&self.wip); | | |
| 7191 | llvm_param_attrs[llvm_param_i] = elem_llvm_ty; | | |
| 7192 | llvm_param_i += 1; | | |
| 7193 | } else { | | |
| 7194 | // Pass the result directly (e.g. "=r") | | |
| 7195 | llvm_ret_types[llvm_ret_i] = elem_llvm_ty; | | |
| 7196 | llvm_ret_i += 1; | | |
| 7197 | } | | |
| 7198 | } else { | | |
| 7199 | switch (constraint[0]) { | | |
| 7200 | '=' => {}, | | |
| 7201 | else => return self.todo("unsupported output constraint on result type '{s}'", .{ | | |
| 7202 | constraint, | | |
| 7203 | }), | | |
| 7204 | } | | |
| 7205 | | | |
| 7206 | llvm_ret_indirect[output.index] = false; | | |
| 7207 | | | |
| 7208 | const ret_ty = self.typeOfIndex(inst); | | |
| 7209 | llvm_ret_types[llvm_ret_i] = try o.lowerType(pt, ret_ty); | | |
| 7210 | llvm_ret_i += 1; | | |
| 7211 | } | | |
| 7212 | | | |
| 7213 | // LLVM uses commas internally to separate different constraints, | | |
| 7214 | // alternative constraints are achieved with pipes. | | |
| 7215 | // We still allow the user to use commas in a way that is similar | | |
| 7216 | // to GCC's inline assembly. | | |
| 7217 | // http://llvm.org/docs/LangRef.html#constraint-codes | | |
| 7218 | for (constraint[1..]) |byte| { | | |
| 7219 | switch (byte) { | | |
| 7220 | ',' => llvm_constraints.appendAssumeCapacity('|'), | | |
| 7221 | '*' => {}, // Indirect outputs are handled above | | |
| 7222 | else => llvm_constraints.appendAssumeCapacity(byte), | | |
| 7223 | } | | |
| 7224 | } | | |
| 7225 | | | |
| 7226 | if (!std.mem.eql(u8, name, "_")) { | | |
| 7227 | const gop = name_map.getOrPutAssumeCapacity(name); | | |
| 7228 | if (gop.found_existing) return self.todo("duplicate asm output name '{s}'", .{name}); | | |
| 7229 | gop.value_ptr.* = @intCast(total_i); | | |
| 7230 | } | | |
| 7231 | total_i += 1; | | |
| 7232 | } | | |
| 7233 | | | |
| 7234 | it = unwrapped_asm.iterateInputs(); | | |
| 7235 | while (it.next()) |input| { | | |
| 7236 | const constraint = input.constraint; | | |
| 7237 | const name = input.name; | | |
| 7238 | | | |
| 7239 | const arg_llvm_value = try self.resolveInst(input.operand); | | |
| 7240 | const arg_ty = self.typeOf(input.operand); | | |
| 7241 | const is_by_ref = isByRef(arg_ty, zcu); | | |
| 7242 | if (is_by_ref) { | | |
| 7243 | if (constraintAllowsMemory(constraint)) { | | |
| 7244 | llvm_param_values[llvm_param_i] = arg_llvm_value; | | |
| 7245 | llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip); | | |
| 7246 | } else { | | |
| 7247 | const alignment = arg_ty.abiAlignment(zcu).toLlvm(); | | |
| 7248 | const arg_llvm_ty = try o.lowerType(pt, arg_ty); | | |
| 7249 | const load_inst = | | |
| 7250 | try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, ""); | | |
| 7251 | llvm_param_values[llvm_param_i] = load_inst; | | |
| 7252 | llvm_param_types[llvm_param_i] = arg_llvm_ty; | | |
| 7253 | } | | |
| 7254 | } else { | | |
| 7255 | if (constraintAllowsRegister(constraint)) { | | |
| 7256 | llvm_param_values[llvm_param_i] = arg_llvm_value; | | |
| 7257 | llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip); | | |
| 7258 | } else { | | |
| 7259 | const alignment = arg_ty.abiAlignment(zcu).toLlvm(); | | |
| 7260 | const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment); | | |
| 7261 | _ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment); | | |
| 7262 | llvm_param_values[llvm_param_i] = arg_ptr; | | |
| 7263 | llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip); | | |
| 7264 | } | | |
| 7265 | } | | |
| 7266 | | | |
| 7267 | try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 1); | | |
| 7268 | if (total_i != 0) { | | |
| 7269 | llvm_constraints.appendAssumeCapacity(','); | | |
| 7270 | } | | |
| 7271 | for (constraint) |byte| { | | |
| 7272 | llvm_constraints.appendAssumeCapacity(switch (byte) { | | |
| 7273 | ',' => '|', | | |
| 7274 | else => byte, | | |
| 7275 | }); | | |
| 7276 | } | | |
| 7277 | | | |
| 7278 | if (!std.mem.eql(u8, name, "_")) { | | |
| 7279 | const gop = name_map.getOrPutAssumeCapacity(name); | | |
| 7280 | if (gop.found_existing) return self.todo("duplicate asm input name '{s}'", .{name}); | | |
| 7281 | gop.value_ptr.* = @intCast(total_i); | | |
| 7282 | } | | |
| 7283 | | | |
| 7284 | // In the case of indirect inputs, LLVM requires the callsite to have | | |
| 7285 | // an elementtype(<ty>) attribute. | | |
| 7286 | llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: { | | |
| 7287 | if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu)); | | |
| 7288 | | | |
| 7289 | break :blk try o.lowerType(pt, if (is_by_ref) arg_ty else arg_ty.childType(zcu)); | | |
| 7290 | } else .none; | | |
| 7291 | | | |
| 7292 | llvm_param_i += 1; | | |
| 7293 | total_i += 1; | | |
| 7294 | } | | |
| 7295 | | | |
| 7296 | it = unwrapped_asm.iterateOutputs(); | | |
| 7297 | while (it.next()) |output| { | | |
| 7298 | const constraint = output.constraint; | | |
| 7299 | | | |
| 7300 | if (constraint[0] != '+') continue; | | |
| 7301 | | | |
| 7302 | const rw_ty = self.typeOf(output.operand); | | |
| 7303 | const llvm_elem_ty = try o.lowerType(pt, rw_ty.childType(zcu)); | | |
| 7304 | if (llvm_ret_indirect[output.index]) { | | |
| 7305 | llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index]; | | |
| 7306 | llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip); | | |
| 7307 | } else { | | |
| 7308 | const alignment = rw_ty.abiAlignment(zcu).toLlvm(); | | |
| 7309 | const loaded = try self.wip.load( | | |
| 7310 | if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, | | |
| 7311 | llvm_elem_ty, | | |
| 7312 | llvm_rw_vals[output.index], | | |
| 7313 | alignment, | | |
| 7314 | "", | | |
| 7315 | ); | | |
| 7316 | llvm_param_values[llvm_param_i] = loaded; | | |
| 7317 | llvm_param_types[llvm_param_i] = llvm_elem_ty; | | |
| 7318 | } | | |
| 7319 | | | |
| 7320 | try llvm_constraints.print(gpa, ",{d}", .{output.index}); | | |
| 7321 | | | |
| 7322 | // In the case of indirect inputs, LLVM requires the callsite to have | | |
| 7323 | // an elementtype(<ty>) attribute. | | |
| 7324 | llvm_param_attrs[llvm_param_i] = if (llvm_ret_indirect[output.index]) llvm_elem_ty else .none; | | |
| 7325 | | | |
| 7326 | llvm_param_i += 1; | | |
| 7327 | total_i += 1; | | |
| 7328 | } | | |
| 7329 | | | |
| 7330 | if (total_i != 0) try llvm_constraints.append(gpa, ','); | | |
| 7331 | const clobbers_val: Value = .fromInterned(unwrapped_asm.clobbers); | | |
| 7332 | const clobbers_ty = clobbers_val.typeOf(zcu); | | |
| 7333 | var clobbers_bigint_buf: Value.BigIntSpace = undefined; | | |
| 7334 | const clobbers_bigint = clobbers_val.toBigInt(&clobbers_bigint_buf, zcu); | | |
| 7335 | for (0..clobbers_ty.structFieldCount(zcu)) |field_index| { | | |
| 7336 | assert(clobbers_ty.fieldType(field_index, zcu).toIntern() == .bool_type); | | |
| 7337 | const limb_bits = @bitSizeOf(std.math.big.Limb); | | |
| 7338 | if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false | | |
| 7339 | switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) { | | |
| 7340 | 0 => continue, // field is false | | |
| 7341 | 1 => {}, // field is true | | |
| 7342 | } | | |
| 7343 | const name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?; | | |
| 7344 | total_i += try appendConstraints(gpa, &llvm_constraints, name, target); | | |
| 7345 | } | | |
| 7346 | | | |
| 7347 | // We have finished scanning through all inputs/outputs, so the number of | | |
| 7348 | // parameters and return values is known. | | |
| 7349 | const param_count = llvm_param_i; | | |
| 7350 | const return_count = llvm_ret_i; | | |
| 7351 | | | |
| 7352 | // For some targets, Clang unconditionally adds some clobbers to all inline assembly. | | |
| 7353 | // While this is probably not strictly necessary, if we don't follow Clang's lead | | |
| 7354 | // here then we may risk tripping LLVM bugs since anything not used by Clang tends | | |
| 7355 | // to be buggy and regress often. | | |
| 7356 | switch (target.cpu.arch) { | | |
| 7357 | .x86_64, .x86 => { | | |
| 7358 | try llvm_constraints.appendSlice(gpa, "~{dirflag},~{fpsr},~{flags},"); | | |
| 7359 | total_i += 3; | | |
| 7360 | }, | | |
| 7361 | .mips, .mipsel, .mips64, .mips64el => { | | |
| 7362 | try llvm_constraints.appendSlice(gpa, "~{$1},"); | | |
| 7363 | total_i += 1; | | |
| 7364 | }, | | |
| 7365 | else => {}, | | |
| 7366 | } | | |
| 7367 | | | |
| 7368 | if (std.mem.endsWith(u8, llvm_constraints.items, ",")) llvm_constraints.items.len -= 1; | | |
| 7369 | | | |
| 7370 | const asm_source = unwrapped_asm.source; | | |
| 7371 | | | |
| 7372 | // hackety hacks until stage2 has proper inline asm in the frontend. | | |
| 7373 | var rendered_template = std.array_list.Managed(u8).init(gpa); | | |
| 7374 | defer rendered_template.deinit(); | | |
| 7375 | | | |
| 7376 | const State = enum { start, percent, input, modifier }; | | |
| 7377 | | | |
| 7378 | var state: State = .start; | | |
| 7379 | | | |
| 7380 | var name_start: usize = undefined; | | |
| 7381 | var modifier_start: usize = undefined; | | |
| 7382 | for (asm_source, 0..) |byte, i| { | | |
| 7383 | switch (state) { | | |
| 7384 | .start => switch (byte) { | | |
| 7385 | '%' => state = .percent, | | |
| 7386 | '$' => try rendered_template.appendSlice("$$"), | | |
| 7387 | else => try rendered_template.append(byte), | | |
| 7388 | }, | | |
| 7389 | .percent => switch (byte) { | | |
| 7390 | '%' => { | | |
| 7391 | try rendered_template.append('%'); | | |
| 7392 | state = .start; | | |
| 7393 | }, | | |
| 7394 | '[' => { | | |
| 7395 | try rendered_template.append('$'); | | |
| 7396 | try rendered_template.append('{'); | | |
| 7397 | name_start = i + 1; | | |
| 7398 | state = .input; | | |
| 7399 | }, | | |
| 7400 | '=' => { | | |
| 7401 | try rendered_template.appendSlice("${:uid}"); | | |
| 7402 | state = .start; | | |
| 7403 | }, | | |
| 7404 | else => { | | |
| 7405 | try rendered_template.append('%'); | | |
| 7406 | try rendered_template.append(byte); | | |
| 7407 | state = .start; | | |
| 7408 | }, | | |
| 7409 | }, | | |
| 7410 | .input => switch (byte) { | | |
| 7411 | ']', ':' => { | | |
| 7412 | const name = asm_source[name_start..i]; | | |
| 7413 | | | |
| 7414 | const index = name_map.get(name) orelse { | | |
| 7415 | // we should validate the assembly in Sema; by now it is too late | | |
| 7416 | return self.todo("unknown input or output name: '{s}'", .{name}); | | |
| 7417 | }; | | |
| 7418 | try rendered_template.print("{d}", .{index}); | | |
| 7419 | if (byte == ':') { | | |
| 7420 | try rendered_template.append(':'); | | |
| 7421 | modifier_start = i + 1; | | |
| 7422 | state = .modifier; | | |
| 7423 | } else { | | |
| 7424 | try rendered_template.append('}'); | | |
| 7425 | state = .start; | | |
| 7426 | } | | |
| 7427 | }, | | |
| 7428 | else => {}, | | |
| 7429 | }, | | |
| 7430 | .modifier => switch (byte) { | | |
| 7431 | ']' => { | | |
| 7432 | try rendered_template.appendSlice(asm_source[modifier_start..i]); | | |
| 7433 | try rendered_template.append('}'); | | |
| 7434 | state = .start; | | |
| 7435 | }, | | |
| 7436 | else => {}, | | |
| 7437 | }, | | |
| 7438 | } | | |
| 7439 | } | | |
| 7440 | | | |
| 7441 | var attributes: Builder.FunctionAttributes.Wip = .{}; | | |
| 7442 | defer attributes.deinit(&o.builder); | | |
| 7443 | for (llvm_param_attrs[0..param_count], 0..) |llvm_elem_ty, i| if (llvm_elem_ty != .none) | | |
| 7444 | try attributes.addParamAttr(i, .{ .elementtype = llvm_elem_ty }, &o.builder); | | |
| 7445 | | | |
| 7446 | const ret_llvm_ty = switch (return_count) { | | |
| 7447 | 0 => .void, | | |
| 7448 | 1 => llvm_ret_types[0], | | |
| 7449 | else => try o.builder.structType(.normal, llvm_ret_types), | | |
| 7450 | }; | | |
| 7451 | const llvm_fn_ty = try o.builder.fnType(ret_llvm_ty, llvm_param_types[0..param_count], .normal); | | |
| 7452 | const call = try self.wip.callAsm( | | |
| 7453 | try attributes.finish(&o.builder), | | |
| 7454 | llvm_fn_ty, | | |
| 7455 | .{ .sideeffect = is_volatile }, | | |
| 7456 | try o.builder.string(rendered_template.items), | | |
| 7457 | try o.builder.string(llvm_constraints.items), | | |
| 7458 | llvm_param_values[0..param_count], | | |
| 7459 | "", | | |
| 7460 | ); | | |
| 7461 | | | |
| 7462 | var ret_val = call; | | |
| 7463 | llvm_ret_i = 0; | | |
| 7464 | for (outputs, 0..) |output, i| { | | |
| 7465 | if (llvm_ret_indirect[i]) continue; | | |
| 7466 | | | |
| 7467 | const output_value = if (return_count > 1) | | |
| 7468 | try self.wip.extractValue(call, &[_]u32{@intCast(llvm_ret_i)}, "") | | |
| 7469 | else | | |
| 7470 | call; | | |
| 7471 | | | |
| 7472 | if (output != .none) { | | |
| 7473 | const output_ptr = try self.resolveInst(output); | | |
| 7474 | const output_ptr_ty = self.typeOf(output); | | |
| 7475 | const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm(); | | |
| 7476 | _ = try self.wip.store( | | |
| 7477 | if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, | | |
| 7478 | output_value, | | |
| 7479 | output_ptr, | | |
| 7480 | alignment, | | |
| 7481 | ); | | |
| 7482 | } else { | | |
| 7483 | ret_val = output_value; | | |
| 7484 | } | | |
| 7485 | llvm_ret_i += 1; | | |
| 7486 | } | | |
| 7487 | | | |
| 7488 | return ret_val; | | |
| 7489 | } | | |
| 7490 | | | |
| 7491 | fn airIsNonNull( | | |
| 7492 | self: *FuncGen, | | |
| 7493 | inst: Air.Inst.Index, | | |
| 7494 | operand_is_ptr: bool, | | |
| 7495 | cond: Builder.IntegerCondition, | | |
| 7496 | ) !Builder.Value { | | |
| 7497 | const o = self.ng.object; | | |
| 7498 | const pt = self.ng.pt; | | |
| 7499 | const zcu = pt.zcu; | | |
| 7500 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 7501 | const operand = try self.resolveInst(un_op); | | |
| 7502 | const operand_ty = self.typeOf(un_op); | | |
| 7503 | const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; | | |
| 7504 | const optional_llvm_ty = try o.lowerType(pt, optional_ty); | | |
| 7505 | const payload_ty = optional_ty.optionalChild(zcu); | | |
| 7506 | | | |
| 7507 | const access_kind: Builder.MemoryAccessKind = | | |
| 7508 | if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | | |
| 7509 | | | |
| 7510 | if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); | | |
| 7511 | | | |
| 7512 | if (optional_ty.optionalReprIsPayload(zcu)) { | | |
| 7513 | const loaded = if (operand_is_ptr) | | |
| 7514 | try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") | | |
| 7515 | else | | |
| 7516 | operand; | | |
| 7517 | if (payload_ty.isSlice(zcu)) { | | |
| 7518 | const slice_ptr = try self.wip.extractValue(loaded, &.{0}, ""); | | |
| 7519 | const ptr_ty = try o.builder.ptrType(toLlvmAddressSpace( | | |
| 7520 | payload_ty.ptrAddressSpace(zcu), | | |
| 7521 | zcu.getTarget(), | | |
| 7522 | )); | | |
| 7523 | return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), ""); | | |
| 7524 | } | | |
| 7525 | return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), ""); | | |
| 7526 | } | | |
| 7527 | | | |
| 7528 | comptime assert(optional_layout_version == 3); | | |
| 7529 | | | |
| 7530 | if (!payload_ty.hasRuntimeBits(zcu)) { | | |
| 7531 | const loaded = if (operand_is_ptr) | | |
| 7532 | try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") | | |
| 7533 | else | | |
| 7534 | operand; | | |
| 7535 | return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), ""); | | |
| 7536 | } | | |
| 7537 | | | |
| 7538 | const is_by_ref = operand_is_ptr or isByRef(optional_ty, zcu); | | |
| 7539 | return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref, access_kind); | | |
| 7540 | } | | |
| 7541 | | | |
| 7542 | fn airIsErr( | | |
| 7543 | self: *FuncGen, | | |
| 7544 | inst: Air.Inst.Index, | | |
| 7545 | cond: Builder.IntegerCondition, | | |
| 7546 | operand_is_ptr: bool, | | |
| 7547 | ) !Builder.Value { | | |
| 7548 | const o = self.ng.object; | | |
| 7549 | const pt = self.ng.pt; | | |
| 7550 | const zcu = pt.zcu; | | |
| 7551 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 7552 | const operand = try self.resolveInst(un_op); | | |
| 7553 | const operand_ty = self.typeOf(un_op); | | |
| 7554 | const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; | | |
| 7555 | const payload_ty = err_union_ty.errorUnionPayload(zcu); | | |
| 7556 | const error_type = try o.errorIntType(pt); | | |
| 7557 | const zero = try o.builder.intValue(error_type, 0); | | |
| 7558 | | | |
| 7559 | const access_kind: Builder.MemoryAccessKind = | | |
| 7560 | if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | | |
| 7561 | | | |
| 7562 | if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { | | |
| 7563 | const val: Builder.Constant = switch (cond) { | | |
| 7564 | .eq => .true, // 0 == 0 | | |
| 7565 | .ne => .false, // 0 != 0 | | |
| 7566 | else => unreachable, | | |
| 7567 | }; | | |
| 7568 | return val.toValue(); | | |
| 7569 | } | | |
| 7570 | | | |
| 7571 | if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); | | |
| 7572 | | | |
| 7573 | if (!payload_ty.hasRuntimeBits(zcu)) { | | |
| 7574 | const loaded = if (operand_is_ptr) | | |
| 7575 | try self.wip.load(access_kind, try o.lowerType(pt, err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") | | |
| 7576 | else | | |
| 7577 | operand; | | |
| 7578 | return self.wip.icmp(cond, loaded, zero, ""); | | |
| 7579 | } | | |
| 7580 | | | |
| 7581 | const err_field_index = try errUnionErrorOffset(payload_ty, pt); | | |
| 7582 | | | |
| 7583 | const loaded = if (operand_is_ptr or isByRef(err_union_ty, zcu)) loaded: { | | |
| 7584 | const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); | | |
| 7585 | const err_alignment = if (operand_is_ptr) | | |
| 7586 | operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu)) | | |
| 7587 | else | | |
| 7588 | .none; | | |
| 7589 | const err_field_ptr = | | |
| 7590 | try self.wip.gepStruct(err_union_llvm_ty, operand, err_field_index, ""); | | |
| 7591 | break :loaded try self.wip.load(access_kind, error_type, err_field_ptr, err_alignment.toLlvm(), ""); | | |
| 7592 | } else try self.wip.extractValue(operand, &.{err_field_index}, ""); | | |
| 7593 | return self.wip.icmp(cond, loaded, zero, ""); | | |
| 7594 | } | | |
| 7595 | | | |
| 7596 | fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 7597 | const o = self.ng.object; | | |
| 7598 | const pt = self.ng.pt; | | |
| 7599 | const zcu = pt.zcu; | | |
| 7600 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 7601 | const operand = try self.resolveInst(ty_op.operand); | | |
| 7602 | const optional_ty = self.typeOf(ty_op.operand).childType(zcu); | | |
| 7603 | const payload_ty = optional_ty.optionalChild(zcu); | | |
| 7604 | if (!payload_ty.hasRuntimeBits(zcu)) { | | |
| 7605 | // We have a pointer to a zero-bit value and we need to return | | |
| 7606 | // a pointer to a zero-bit value. | | |
| 7607 | return operand; | | |
| 7608 | } | | |
| 7609 | if (optional_ty.optionalReprIsPayload(zcu)) { | | |
| 7610 | // The payload and the optional are the same value. | | |
| 7611 | return operand; | | |
| 7612 | } | | |
| 7613 | return self.wip.gepStruct(try o.lowerType(pt, optional_ty), operand, 0, ""); | | |
| 7614 | } | | |
| 7615 | | | |
| 7616 | fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 7617 | comptime assert(optional_layout_version == 3); | | |
| 7618 | | | |
| 7619 | const o = self.ng.object; | | |
| 7620 | const pt = self.ng.pt; | | |
| 7621 | const zcu = pt.zcu; | | |
| 7622 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 7623 | const operand = try self.resolveInst(ty_op.operand); | | |
| 7624 | const optional_ptr_ty = self.typeOf(ty_op.operand); | | |
| 7625 | const optional_ty = optional_ptr_ty.childType(zcu); | | |
| 7626 | const payload_ty = optional_ty.optionalChild(zcu); | | |
| 7627 | const non_null_bit = try o.builder.intValue(.i8, 1); | | |
| 7628 | | | |
| 7629 | const access_kind: Builder.MemoryAccessKind = | | |
| 7630 | if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | | |
| 7631 | | | |
| 7632 | if (!payload_ty.hasRuntimeBits(zcu)) { | | |
| 7633 | self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu)); | | |
| 7634 | | | |
| 7635 | // We have a pointer to a i8. We need to set it to 1 and then return the same pointer. | | |
| 7636 | // Default alignment store because align of the non null bit is 1 anyway. | | |
| 7637 | _ = try self.wip.store(access_kind, non_null_bit, operand, .default); | | |
| 7638 | return operand; | | |
| 7639 | } | | |
| 7640 | if (optional_ty.optionalReprIsPayload(zcu)) { | | |
| 7641 | // The payload and the optional are the same value. | | |
| 7642 | // Setting to non-null will be done when the payload is set. | | |
| 7643 | return operand; | | |
| 7644 | } | | |
| 7645 | | | |
| 7646 | // First set the non-null bit. | | |
| 7647 | const optional_llvm_ty = try o.lowerType(pt, optional_ty); | | |
| 7648 | const non_null_ptr = try self.wip.gepStruct(optional_llvm_ty, operand, 1, ""); | | |
| 7649 | | | |
| 7650 | self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu)); | | |
| 7651 | | | |
| 7652 | // Default alignment store because align of the non null bit is 1 anyway. | | |
| 7653 | _ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default); | | |
| 7654 | | | |
| 7655 | // Then return the payload pointer (only if it's used). | | |
| 7656 | if (self.liveness.isUnused(inst)) return .none; | | |
| 7657 | | | |
| 7658 | return self.wip.gepStruct(optional_llvm_ty, operand, 0, ""); | | |
| 7659 | } | | |
| 7660 | | | |
| 7661 | fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 7662 | const o = self.ng.object; | | |
| 7663 | const pt = self.ng.pt; | | |
| 7664 | const zcu = pt.zcu; | | |
| 7665 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 7666 | const operand = try self.resolveInst(ty_op.operand); | | |
| 7667 | const optional_ty = self.typeOf(ty_op.operand); | | |
| 7668 | const payload_ty = self.typeOfIndex(inst); | | |
| 7669 | if (!payload_ty.hasRuntimeBits(zcu)) return .none; | | |
| 7670 | | | |
| 7671 | if (optional_ty.optionalReprIsPayload(zcu)) { | | |
| 7672 | // Payload value is the same as the optional value. | | |
| 7673 | return operand; | | |
| 7674 | } | | |
| 7675 | | | |
| 7676 | const opt_llvm_ty = try o.lowerType(pt, optional_ty); | | |
| 7677 | return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false); | | |
| 7678 | } | | |
| 7679 | | | |
| 7680 | fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value { | | |
| 7681 | const o = self.ng.object; | | |
| 7682 | const pt = self.ng.pt; | | |
| 7683 | const zcu = pt.zcu; | | |
| 7684 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 7685 | const operand = try self.resolveInst(ty_op.operand); | | |
| 7686 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 7687 | const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; | | |
| 7688 | const result_ty = self.typeOfIndex(inst); | | |
| 7689 | const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty; | | |
| 7690 | | | |
| 7691 | if (!payload_ty.hasRuntimeBits(zcu)) { | | |
| 7692 | return if (operand_is_ptr) operand else .none; | | |
| 7693 | } | | |
| 7694 | const offset = try errUnionPayloadOffset(payload_ty, pt); | | |
| 7695 | const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); | | |
| 7696 | if (operand_is_ptr) { | | |
| 7697 | return self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); | | |
| 7698 | } else if (isByRef(err_union_ty, zcu)) { | | |
| 7699 | const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm(); | | |
| 7700 | const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); | | |
| 7701 | if (isByRef(payload_ty, zcu)) { | | |
| 7702 | return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal); | | |
| 7703 | } | | |
| 7704 | const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset]; | | |
| 7705 | return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, ""); | | |
| 7706 | } | | |
| 7707 | return self.wip.extractValue(operand, &.{offset}, ""); | | |
| 7708 | } | | |
| 7709 | | | |
| 7710 | fn airErrUnionErr( | | |
| 7711 | self: *FuncGen, | | |
| 7712 | inst: Air.Inst.Index, | | |
| 7713 | operand_is_ptr: bool, | | |
| 7714 | ) !Builder.Value { | | |
| 7715 | const o = self.ng.object; | | |
| 7716 | const pt = self.ng.pt; | | |
| 7717 | const zcu = pt.zcu; | | |
| 7718 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 7719 | const operand = try self.resolveInst(ty_op.operand); | | |
| 7720 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 7721 | const error_type = try o.errorIntType(pt); | | |
| 7722 | const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; | | |
| 7723 | if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { | | |
| 7724 | if (operand_is_ptr) { | | |
| 7725 | return operand; | | |
| 7726 | } else { | | |
| 7727 | return o.builder.intValue(error_type, 0); | | |
| 7728 | } | | |
| 7729 | } | | |
| 7730 | | | |
| 7731 | const access_kind: Builder.MemoryAccessKind = | | |
| 7732 | if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | | |
| 7733 | | | |
| 7734 | const payload_ty = err_union_ty.errorUnionPayload(zcu); | | |
| 7735 | if (!payload_ty.hasRuntimeBits(zcu)) { | | |
| 7736 | if (!operand_is_ptr) return operand; | | |
| 7737 | | | |
| 7738 | self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); | | |
| 7739 | | | |
| 7740 | return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), ""); | | |
| 7741 | } | | |
| 7742 | | | |
| 7743 | const offset = try errUnionErrorOffset(payload_ty, pt); | | |
| 7744 | | | |
| 7745 | if (operand_is_ptr or isByRef(err_union_ty, zcu)) { | | |
| 7746 | if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); | | |
| 7747 | | | |
| 7748 | const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); | | |
| 7749 | const err_field_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); | | |
| 7750 | return self.wip.load(access_kind, error_type, err_field_ptr, .default, ""); | | |
| 7751 | } | | |
| 7752 | | | |
| 7753 | return self.wip.extractValue(operand, &.{offset}, ""); | | |
| 7754 | } | | |
| 7755 | | | |
| 7756 | fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 7757 | const o = self.ng.object; | | |
| 7758 | const pt = self.ng.pt; | | |
| 7759 | const zcu = pt.zcu; | | |
| 7760 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 7761 | const operand = try self.resolveInst(ty_op.operand); | | |
| 7762 | const err_union_ptr_ty = self.typeOf(ty_op.operand); | | |
| 7763 | const err_union_ty = err_union_ptr_ty.childType(zcu); | | |
| 7764 | const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu); | | |
| 7765 | | | |
| 7766 | const payload_ty = err_union_ty.errorUnionPayload(zcu); | | |
| 7767 | const non_error_val = try o.builder.intValue(try o.errorIntType(pt), 0); | | |
| 7768 | | | |
| 7769 | const access_kind: Builder.MemoryAccessKind = | | |
| 7770 | if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | | |
| 7771 | | | |
| 7772 | if (!payload_ty.hasRuntimeBits(zcu)) { | | |
| 7773 | self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu)); | | |
| 7774 | _ = try self.wip.store(access_kind, non_error_val, operand, err_union_ptr_align.toLlvm()); | | |
| 7775 | return operand; | | |
| 7776 | } | | |
| 7777 | const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); | | |
| 7778 | { | | |
| 7779 | self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu)); | | |
| 7780 | | | |
| 7781 | const err_int_ty = try pt.errorIntType(); | | |
| 7782 | const error_alignment = err_int_ty.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm(); | | |
| 7783 | const error_offset = try errUnionErrorOffset(payload_ty, pt); | | |
| 7784 | // First set the non-error value. | | |
| 7785 | const non_null_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, error_offset, ""); | | |
| 7786 | _ = try self.wip.store(access_kind, non_error_val, non_null_ptr, error_alignment); | | |
| 7787 | } | | |
| 7788 | // Then return the payload pointer (only if it is used). | | |
| 7789 | if (self.liveness.isUnused(inst)) return .none; | | |
| 7790 | | | |
| 7791 | const payload_offset = try errUnionPayloadOffset(payload_ty, pt); | | |
| 7792 | return self.wip.gepStruct(err_union_llvm_ty, operand, payload_offset, ""); | | |
| 7793 | } | | |
| 7794 | | | |
| 7795 | fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value { | | |
| 7796 | assert(self.err_ret_trace != .none); | | |
| 7797 | return self.err_ret_trace; | | |
| 7798 | } | | |
| 7799 | | | |
| 7800 | fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 7801 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 7802 | self.err_ret_trace = try self.resolveInst(un_op); | | |
| 7803 | return .none; | | |
| 7804 | } | | |
| 7805 | | | |
| 7806 | fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 7807 | const o = self.ng.object; | | |
| 7808 | const pt = self.ng.pt; | | |
| 7809 | const zcu = pt.zcu; | | |
| 7810 | | | |
| 7811 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 7812 | const struct_ty = ty_pl.ty.toType(); | | |
| 7813 | const field_index = ty_pl.payload; | | |
| 7814 | | | |
| 7815 | const struct_llvm_ty = try o.lowerType(pt, struct_ty); | | |
| 7816 | const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; | | |
| 7817 | assert(self.err_ret_trace != .none); | | |
| 7818 | const field_ptr = try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, ""); | | |
| 7819 | const field_alignment = struct_ty.explicitFieldAlignment(field_index, zcu); | | |
| 7820 | const field_ty = struct_ty.fieldType(field_index, zcu); | | |
| 7821 | const field_ptr_ty = try pt.ptrType(.{ | | |
| 7822 | .child = field_ty.toIntern(), | | |
| 7823 | .flags = .{ .alignment = field_alignment }, | | |
| 7824 | }); | | |
| 7825 | return self.load(field_ptr, field_ptr_ty); | | |
| 7826 | } | | |
| 7827 | | | |
| 7828 | /// As an optimization, we want to avoid unnecessary copies of | | |
| 7829 | /// error union/optional types when returning from a function. | | |
| 7830 | /// Here, we scan forward in the current block, looking to see | | |
| 7831 | /// if the next instruction is a return (ignoring debug instructions). | | |
| 7832 | /// | | |
| 7833 | /// The first instruction of `body_tail` is a wrap instruction. | | |
| 7834 | fn isNextRet( | | |
| 7835 | self: *FuncGen, | | |
| 7836 | body_tail: []const Air.Inst.Index, | | |
| 7837 | ) bool { | | |
| 7838 | const air_tags = self.air.instructions.items(.tag); | | |
| 7839 | for (body_tail[1..]) |body_inst| { | | |
| 7840 | switch (air_tags[@intFromEnum(body_inst)]) { | | |
| 7841 | .ret => return true, | | |
| 7842 | .dbg_stmt => continue, | | |
| 7843 | else => return false, | | |
| 7844 | } | | |
| 7845 | } | | |
| 7846 | // The only way to get here is to hit the end of a loop instruction | | |
| 7847 | // (implicit repeat). | | |
| 7848 | return false; | | |
| 7849 | } | | |
| 7850 | | | |
| 7851 | fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { | | |
| 7852 | const o = self.ng.object; | | |
| 7853 | const pt = self.ng.pt; | | |
| 7854 | const zcu = pt.zcu; | | |
| 7855 | const inst = body_tail[0]; | | |
| 7856 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 7857 | const payload_ty = self.typeOf(ty_op.operand); | | |
| 7858 | const non_null_bit = try o.builder.intValue(.i8, 1); | | |
| 7859 | comptime assert(optional_layout_version == 3); | | |
| 7860 | assert(payload_ty.hasRuntimeBits(zcu)); | | |
| 7861 | const operand = try self.resolveInst(ty_op.operand); | | |
| 7862 | const optional_ty = self.typeOfIndex(inst); | | |
| 7863 | if (optional_ty.optionalReprIsPayload(zcu)) return operand; | | |
| 7864 | const llvm_optional_ty = try o.lowerType(pt, optional_ty); | | |
| 7865 | if (isByRef(optional_ty, zcu)) { | | |
| 7866 | const directReturn = self.isNextRet(body_tail); | | |
| 7867 | const optional_ptr = if (directReturn) | | |
| 7868 | self.ret_ptr | | |
| 7869 | else brk: { | | |
| 7870 | const alignment = optional_ty.abiAlignment(zcu).toLlvm(); | | |
| 7871 | const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment); | | |
| 7872 | break :brk optional_ptr; | | |
| 7873 | }; | | |
| 7874 | | | |
| 7875 | const payload_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 0, ""); | | |
| 7876 | const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); | | |
| 7877 | try self.store(payload_ptr, payload_ptr_ty, operand, .none); | | |
| 7878 | const non_null_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 1, ""); | | |
| 7879 | _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default); | | |
| 7880 | return optional_ptr; | | |
| 7881 | } | | |
| 7882 | return self.wip.buildAggregate(llvm_optional_ty, &.{ operand, non_null_bit }, ""); | | |
| 7883 | } | | |
| 7884 | | | |
| 7885 | fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { | | |
| 7886 | const o = self.ng.object; | | |
| 7887 | const pt = self.ng.pt; | | |
| 7888 | const zcu = pt.zcu; | | |
| 7889 | const inst = body_tail[0]; | | |
| 7890 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 7891 | const err_un_ty = self.typeOfIndex(inst); | | |
| 7892 | const operand = try self.resolveInst(ty_op.operand); | | |
| 7893 | const payload_ty = self.typeOf(ty_op.operand); | | |
| 7894 | assert(payload_ty.hasRuntimeBits(zcu)); | | |
| 7895 | const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0); | | |
| 7896 | const err_un_llvm_ty = try o.lowerType(pt, err_un_ty); | | |
| 7897 | | | |
| 7898 | const payload_offset = try errUnionPayloadOffset(payload_ty, pt); | | |
| 7899 | const error_offset = try errUnionErrorOffset(payload_ty, pt); | | |
| 7900 | if (isByRef(err_un_ty, zcu)) { | | |
| 7901 | const directReturn = self.isNextRet(body_tail); | | |
| 7902 | const result_ptr = if (directReturn) | | |
| 7903 | self.ret_ptr | | |
| 7904 | else brk: { | | |
| 7905 | const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm(); | | |
| 7906 | const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment); | | |
| 7907 | break :brk result_ptr; | | |
| 7908 | }; | | |
| 7909 | | | |
| 7910 | const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, ""); | | |
| 7911 | const err_int_ty = try pt.errorIntType(); | | |
| 7912 | const error_alignment = err_int_ty.abiAlignment(pt.zcu).toLlvm(); | | |
| 7913 | _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment); | | |
| 7914 | const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, ""); | | |
| 7915 | const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); | | |
| 7916 | try self.store(payload_ptr, payload_ptr_ty, operand, .none); | | |
| 7917 | return result_ptr; | | |
| 7918 | } | | |
| 7919 | var fields: [2]Builder.Value = undefined; | | |
| 7920 | fields[payload_offset] = operand; | | |
| 7921 | fields[error_offset] = ok_err_code; | | |
| 7922 | return self.wip.buildAggregate(err_un_llvm_ty, &fields, ""); | | |
| 7923 | } | | |
| 7924 | | | |
| 7925 | fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { | | |
| 7926 | const o = self.ng.object; | | |
| 7927 | const pt = self.ng.pt; | | |
| 7928 | const zcu = pt.zcu; | | |
| 7929 | const inst = body_tail[0]; | | |
| 7930 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 7931 | const err_un_ty = self.typeOfIndex(inst); | | |
| 7932 | const payload_ty = err_un_ty.errorUnionPayload(zcu); | | |
| 7933 | const operand = try self.resolveInst(ty_op.operand); | | |
| 7934 | if (!payload_ty.hasRuntimeBits(zcu)) return operand; | | |
| 7935 | const err_un_llvm_ty = try o.lowerType(pt, err_un_ty); | | |
| 7936 | | | |
| 7937 | const payload_offset = try errUnionPayloadOffset(payload_ty, pt); | | |
| 7938 | const error_offset = try errUnionErrorOffset(payload_ty, pt); | | |
| 7939 | if (isByRef(err_un_ty, zcu)) { | | |
| 7940 | const directReturn = self.isNextRet(body_tail); | | |
| 7941 | const result_ptr = if (directReturn) | | |
| 7942 | self.ret_ptr | | |
| 7943 | else brk: { | | |
| 7944 | const alignment = err_un_ty.abiAlignment(zcu).toLlvm(); | | |
| 7945 | const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment); | | |
| 7946 | break :brk result_ptr; | | |
| 7947 | }; | | |
| 7948 | | | |
| 7949 | const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, ""); | | |
| 7950 | const err_int_ty = try pt.errorIntType(); | | |
| 7951 | const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm(); | | |
| 7952 | _ = try self.wip.store(.normal, operand, err_ptr, error_alignment); | | |
| 7953 | const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, ""); | | |
| 7954 | const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); | | |
| 7955 | // TODO store undef to payload_ptr | | |
| 7956 | _ = payload_ptr; | | |
| 7957 | _ = payload_ptr_ty; | | |
| 7958 | return result_ptr; | | |
| 7959 | } | | |
| 7960 | | | |
| 7961 | // TODO set payload bytes to undef | | |
| 7962 | const undef = try o.builder.undefValue(err_un_llvm_ty); | | |
| 7963 | return self.wip.insertValue(undef, operand, &.{error_offset}, ""); | | |
| 7964 | } | | |
| 7965 | | | |
| 7966 | fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 7967 | const o = self.ng.object; | | |
| 7968 | const pt = self.ng.pt; | | |
| 7969 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | | |
| 7970 | const index = pl_op.payload; | | |
| 7971 | const llvm_usize = try o.lowerType(pt, Type.usize); | | |
| 7972 | return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{ | | |
| 7973 | try o.builder.intValue(.i32, index), | | |
| 7974 | }, ""); | | |
| 7975 | } | | |
| 7976 | | | |
| 7977 | fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 7978 | const o = self.ng.object; | | |
| 7979 | const pt = self.ng.pt; | | |
| 7980 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | | |
| 7981 | const index = pl_op.payload; | | |
| 7982 | const llvm_isize = try o.lowerType(pt, Type.isize); | | |
| 7983 | return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{ | | |
| 7984 | try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand), | | |
| 7985 | }, ""); | | |
| 7986 | } | | |
| 7987 | | | |
| 7988 | fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 7989 | const o = fg.ng.object; | | |
| 7990 | const pt = fg.ng.pt; | | |
| 7991 | const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; | | |
| 7992 | const llvm_ptr_const = try o.lowerNavRefValue(pt, ty_nav.nav); | | |
| 7993 | return llvm_ptr_const.toValue(); | | |
| 7994 | } | | |
| 7995 | | | |
| 7996 | fn airMin(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 7997 | const o = self.ng.object; | | |
| 7998 | const pt = self.ng.pt; | | |
| 7999 | const zcu = pt.zcu; | | |
| 8000 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8001 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8002 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8003 | const inst_ty = self.typeOfIndex(inst); | | |
| 8004 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8005 | | | |
| 8006 | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmin, .normal, inst_ty, 2, .{ lhs, rhs }); | | |
| 8007 | return self.wip.callIntrinsic( | | |
| 8008 | .normal, | | |
| 8009 | .none, | | |
| 8010 | if (scalar_ty.isSignedInt(zcu)) .smin else .umin, | | |
| 8011 | &.{try o.lowerType(pt, inst_ty)}, | | |
| 8012 | &.{ lhs, rhs }, | | |
| 8013 | "", | | |
| 8014 | ); | | |
| 8015 | } | | |
| 8016 | | | |
| 8017 | fn airMax(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8018 | const o = self.ng.object; | | |
| 8019 | const pt = self.ng.pt; | | |
| 8020 | const zcu = pt.zcu; | | |
| 8021 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8022 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8023 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8024 | const inst_ty = self.typeOfIndex(inst); | | |
| 8025 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8026 | | | |
| 8027 | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmax, .normal, inst_ty, 2, .{ lhs, rhs }); | | |
| 8028 | return self.wip.callIntrinsic( | | |
| 8029 | .normal, | | |
| 8030 | .none, | | |
| 8031 | if (scalar_ty.isSignedInt(zcu)) .smax else .umax, | | |
| 8032 | &.{try o.lowerType(pt, inst_ty)}, | | |
| 8033 | &.{ lhs, rhs }, | | |
| 8034 | "", | | |
| 8035 | ); | | |
| 8036 | } | | |
| 8037 | | | |
| 8038 | fn airSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8039 | const o = self.ng.object; | | |
| 8040 | const pt = self.ng.pt; | | |
| 8041 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 8042 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; | | |
| 8043 | const ptr = try self.resolveInst(bin_op.lhs); | | |
| 8044 | const len = try self.resolveInst(bin_op.rhs); | | |
| 8045 | const inst_ty = self.typeOfIndex(inst); | | |
| 8046 | return self.wip.buildAggregate(try o.lowerType(pt, inst_ty), &.{ ptr, len }, ""); | | |
| 8047 | } | | |
| 8048 | | | |
| 8049 | fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 8050 | const zcu = self.ng.pt.zcu; | | |
| 8051 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8052 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8053 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8054 | const inst_ty = self.typeOfIndex(inst); | | |
| 8055 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8056 | | | |
| 8057 | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, fast, inst_ty, 2, .{ lhs, rhs }); | | |
| 8058 | return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"add nsw" else .@"add nuw", lhs, rhs, ""); | | |
| 8059 | } | | |
| 8060 | | | |
| 8061 | fn airSafeArithmetic( | | |
| 8062 | fg: *FuncGen, | | |
| 8063 | inst: Air.Inst.Index, | | |
| 8064 | signed_intrinsic: Builder.Intrinsic, | | |
| 8065 | unsigned_intrinsic: Builder.Intrinsic, | | |
| 8066 | ) !Builder.Value { | | |
| 8067 | const o = fg.ng.object; | | |
| 8068 | const pt = fg.ng.pt; | | |
| 8069 | const zcu = pt.zcu; | | |
| 8070 | | | |
| 8071 | const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8072 | const lhs = try fg.resolveInst(bin_op.lhs); | | |
| 8073 | const rhs = try fg.resolveInst(bin_op.rhs); | | |
| 8074 | const inst_ty = fg.typeOfIndex(inst); | | |
| 8075 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8076 | | | |
| 8077 | const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic; | | |
| 8078 | const llvm_inst_ty = try o.lowerType(pt, inst_ty); | | |
| 8079 | const results = | | |
| 8080 | try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, ""); | | |
| 8081 | | | |
| 8082 | const overflow_bits = try fg.wip.extractValue(results, &.{1}, ""); | | |
| 8083 | const overflow_bits_ty = overflow_bits.typeOfWip(&fg.wip); | | |
| 8084 | const overflow_bit = if (overflow_bits_ty.isVector(&o.builder)) | | |
| 8085 | try fg.wip.callIntrinsic( | | |
| 8086 | .normal, | | |
| 8087 | .none, | | |
| 8088 | .@"vector.reduce.or", | | |
| 8089 | &.{overflow_bits_ty}, | | |
| 8090 | &.{overflow_bits}, | | |
| 8091 | "", | | |
| 8092 | ) | | |
| 8093 | else | | |
| 8094 | overflow_bits; | | |
| 8095 | | | |
| 8096 | const fail_block = try fg.wip.block(1, "OverflowFail"); | | |
| 8097 | const ok_block = try fg.wip.block(1, "OverflowOk"); | | |
| 8098 | _ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none); | | |
| 8099 | | | |
| 8100 | fg.wip.cursor = .{ .block = fail_block }; | | |
| 8101 | try fg.buildSimplePanic(.integer_overflow); | | |
| 8102 | | | |
| 8103 | fg.wip.cursor = .{ .block = ok_block }; | | |
| 8104 | return fg.wip.extractValue(results, &.{0}, ""); | | |
| 8105 | } | | |
| 8106 | | | |
| 8107 | fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8108 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8109 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8110 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8111 | | | |
| 8112 | return self.wip.bin(.add, lhs, rhs, ""); | | |
| 8113 | } | | |
| 8114 | | | |
| 8115 | fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8116 | const o = self.ng.object; | | |
| 8117 | const pt = self.ng.pt; | | |
| 8118 | const zcu = pt.zcu; | | |
| 8119 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8120 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8121 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8122 | const inst_ty = self.typeOfIndex(inst); | | |
| 8123 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8124 | assert(scalar_ty.zigTypeTag(zcu) == .int); | | |
| 8125 | return self.wip.callIntrinsic( | | |
| 8126 | .normal, | | |
| 8127 | .none, | | |
| 8128 | if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat", | | |
| 8129 | &.{try o.lowerType(pt, inst_ty)}, | | |
| 8130 | &.{ lhs, rhs }, | | |
| 8131 | "", | | |
| 8132 | ); | | |
| 8133 | } | | |
| 8134 | | | |
| 8135 | fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 8136 | const zcu = self.ng.pt.zcu; | | |
| 8137 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8138 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8139 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8140 | const inst_ty = self.typeOfIndex(inst); | | |
| 8141 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8142 | | | |
| 8143 | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, fast, inst_ty, 2, .{ lhs, rhs }); | | |
| 8144 | return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"sub nsw" else .@"sub nuw", lhs, rhs, ""); | | |
| 8145 | } | | |
| 8146 | | | |
| 8147 | fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8148 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8149 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8150 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8151 | | | |
| 8152 | return self.wip.bin(.sub, lhs, rhs, ""); | | |
| 8153 | } | | |
| 8154 | | | |
| 8155 | fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8156 | const o = self.ng.object; | | |
| 8157 | const pt = self.ng.pt; | | |
| 8158 | const zcu = pt.zcu; | | |
| 8159 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8160 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8161 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8162 | const inst_ty = self.typeOfIndex(inst); | | |
| 8163 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8164 | assert(scalar_ty.zigTypeTag(zcu) == .int); | | |
| 8165 | return self.wip.callIntrinsic( | | |
| 8166 | .normal, | | |
| 8167 | .none, | | |
| 8168 | if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat", | | |
| 8169 | &.{try o.lowerType(pt, inst_ty)}, | | |
| 8170 | &.{ lhs, rhs }, | | |
| 8171 | "", | | |
| 8172 | ); | | |
| 8173 | } | | |
| 8174 | | | |
| 8175 | fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 8176 | const zcu = self.ng.pt.zcu; | | |
| 8177 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8178 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8179 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8180 | const inst_ty = self.typeOfIndex(inst); | | |
| 8181 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8182 | | | |
| 8183 | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, fast, inst_ty, 2, .{ lhs, rhs }); | | |
| 8184 | return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"mul nsw" else .@"mul nuw", lhs, rhs, ""); | | |
| 8185 | } | | |
| 8186 | | | |
| 8187 | fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8188 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8189 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8190 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8191 | | | |
| 8192 | return self.wip.bin(.mul, lhs, rhs, ""); | | |
| 8193 | } | | |
| 8194 | | | |
| 8195 | fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8196 | const o = self.ng.object; | | |
| 8197 | const pt = self.ng.pt; | | |
| 8198 | const zcu = pt.zcu; | | |
| 8199 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8200 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8201 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8202 | const inst_ty = self.typeOfIndex(inst); | | |
| 8203 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8204 | assert(scalar_ty.zigTypeTag(zcu) == .int); | | |
| 8205 | return self.wip.callIntrinsic( | | |
| 8206 | .normal, | | |
| 8207 | .none, | | |
| 8208 | if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat", | | |
| 8209 | &.{try o.lowerType(pt, inst_ty)}, | | |
| 8210 | &.{ lhs, rhs, .@"0" }, | | |
| 8211 | "", | | |
| 8212 | ); | | |
| 8213 | } | | |
| 8214 | | | |
| 8215 | fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 8216 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8217 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8218 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8219 | const inst_ty = self.typeOfIndex(inst); | | |
| 8220 | | | |
| 8221 | return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); | | |
| 8222 | } | | |
| 8223 | | | |
| 8224 | fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 8225 | const zcu = self.ng.pt.zcu; | | |
| 8226 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8227 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8228 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8229 | const inst_ty = self.typeOfIndex(inst); | | |
| 8230 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8231 | | | |
| 8232 | if (scalar_ty.isRuntimeFloat()) { | | |
| 8233 | const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); | | |
| 8234 | return self.buildFloatOp(.trunc, fast, inst_ty, 1, .{result}); | | |
| 8235 | } | | |
| 8236 | return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .sdiv else .udiv, lhs, rhs, ""); | | |
| 8237 | } | | |
| 8238 | | | |
| 8239 | fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 8240 | const o = self.ng.object; | | |
| 8241 | const pt = self.ng.pt; | | |
| 8242 | const zcu = pt.zcu; | | |
| 8243 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8244 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8245 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8246 | const inst_ty = self.typeOfIndex(inst); | | |
| 8247 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8248 | | | |
| 8249 | if (scalar_ty.isRuntimeFloat()) { | | |
| 8250 | const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); | | |
| 8251 | return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result}); | | |
| 8252 | } | | |
| 8253 | if (scalar_ty.isSignedInt(zcu)) { | | |
| 8254 | const inst_llvm_ty = try o.lowerType(pt, inst_ty); | | |
| 8255 | | | |
| 8256 | const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb; | | |
| 8257 | var stack align(@max( | | |
| 8258 | @alignOf(std.heap.StackFallbackAllocator(0)), | | |
| 8259 | @alignOf(ExpectedContents), | | |
| 8260 | )) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); | | |
| 8261 | const allocator = stack.get(); | | |
| 8262 | | | |
| 8263 | const scalar_bits = inst_llvm_ty.scalarBits(&o.builder); | | |
| 8264 | var smin_big_int: std.math.big.int.Mutable = .{ | | |
| 8265 | .limbs = try allocator.alloc( | | |
| 8266 | std.math.big.Limb, | | |
| 8267 | std.math.big.int.calcTwosCompLimbCount(scalar_bits), | | |
| 8268 | ), | | |
| 8269 | .len = undefined, | | |
| 8270 | .positive = undefined, | | |
| 8271 | }; | | |
| 8272 | defer allocator.free(smin_big_int.limbs); | | |
| 8273 | smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits); | | |
| 8274 | const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst( | | |
| 8275 | inst_llvm_ty.scalarType(&o.builder), | | |
| 8276 | smin_big_int.toConst(), | | |
| 8277 | )); | | |
| 8278 | | | |
| 8279 | const div = try self.wip.bin(.sdiv, lhs, rhs, "divFloor.div"); | | |
| 8280 | const rem = try self.wip.bin(.srem, lhs, rhs, "divFloor.rem"); | | |
| 8281 | const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "divFloor.rhs_sign"); | | |
| 8282 | const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "divFloor.rem_xor_rhs_sign"); | | |
| 8283 | const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "divFloor.need_correction"); | | |
| 8284 | const correction = try self.wip.cast(.sext, need_correction, inst_llvm_ty, "divFloor.correction"); | | |
| 8285 | return self.wip.bin(.@"add nsw", div, correction, "divFloor"); | | |
| 8286 | } | | |
| 8287 | return self.wip.bin(.udiv, lhs, rhs, ""); | | |
| 8288 | } | | |
| 8289 | | | |
| 8290 | fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 8291 | const zcu = self.ng.pt.zcu; | | |
| 8292 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8293 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8294 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8295 | const inst_ty = self.typeOfIndex(inst); | | |
| 8296 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8297 | | | |
| 8298 | if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); | | |
| 8299 | return self.wip.bin( | | |
| 8300 | if (scalar_ty.isSignedInt(zcu)) .@"sdiv exact" else .@"udiv exact", | | |
| 8301 | lhs, | | |
| 8302 | rhs, | | |
| 8303 | "", | | |
| 8304 | ); | | |
| 8305 | } | | |
| 8306 | | | |
| 8307 | fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 8308 | const zcu = self.ng.pt.zcu; | | |
| 8309 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8310 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8311 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8312 | const inst_ty = self.typeOfIndex(inst); | | |
| 8313 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8314 | | | |
| 8315 | if (scalar_ty.isRuntimeFloat()) | | |
| 8316 | return self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs }); | | |
| 8317 | return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) | | |
| 8318 | .srem | | |
| 8319 | else | | |
| 8320 | .urem, lhs, rhs, ""); | | |
| 8321 | } | | |
| 8322 | | | |
| 8323 | fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 8324 | const o = self.ng.object; | | |
| 8325 | const pt = self.ng.pt; | | |
| 8326 | const zcu = pt.zcu; | | |
| 8327 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8328 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8329 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8330 | const inst_ty = self.typeOfIndex(inst); | | |
| 8331 | const inst_llvm_ty = try o.lowerType(pt, inst_ty); | | |
| 8332 | const scalar_ty = inst_ty.scalarType(zcu); | | |
| 8333 | | | |
| 8334 | if (scalar_ty.isRuntimeFloat()) { | | |
| 8335 | const a = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs }); | | |
| 8336 | const b = try self.buildFloatOp(.add, fast, inst_ty, 2, .{ a, rhs }); | | |
| 8337 | const c = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ b, rhs }); | | |
| 8338 | const zero = try o.builder.zeroInitValue(inst_llvm_ty); | | |
| 8339 | const ltz = try self.buildFloatCmp(fast, .lt, inst_ty, .{ lhs, zero }); | | |
| 8340 | return self.wip.select(fast, ltz, c, a, ""); | | |
| 8341 | } | | |
| 8342 | if (scalar_ty.isSignedInt(zcu)) { | | |
| 8343 | const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb; | | |
| 8344 | var stack align(@max( | | |
| 8345 | @alignOf(std.heap.StackFallbackAllocator(0)), | | |
| 8346 | @alignOf(ExpectedContents), | | |
| 8347 | )) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); | | |
| 8348 | const allocator = stack.get(); | | |
| 8349 | | | |
| 8350 | const scalar_bits = inst_llvm_ty.scalarBits(&o.builder); | | |
| 8351 | var smin_big_int: std.math.big.int.Mutable = .{ | | |
| 8352 | .limbs = try allocator.alloc( | | |
| 8353 | std.math.big.Limb, | | |
| 8354 | std.math.big.int.calcTwosCompLimbCount(scalar_bits), | | |
| 8355 | ), | | |
| 8356 | .len = undefined, | | |
| 8357 | .positive = undefined, | | |
| 8358 | }; | | |
| 8359 | defer allocator.free(smin_big_int.limbs); | | |
| 8360 | smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits); | | |
| 8361 | const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst( | | |
| 8362 | inst_llvm_ty.scalarType(&o.builder), | | |
| 8363 | smin_big_int.toConst(), | | |
| 8364 | )); | | |
| 8365 | | | |
| 8366 | const rem = try self.wip.bin(.srem, lhs, rhs, "mod.rem"); | | |
| 8367 | const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "mod.rhs_sign"); | | |
| 8368 | const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "mod.rem_xor_rhs_sign"); | | |
| 8369 | const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "mod.need_correction"); | | |
| 8370 | const zero = try o.builder.zeroInitValue(inst_llvm_ty); | | |
| 8371 | const correction = try self.wip.select(.normal, need_correction, rhs, zero, "mod.correction"); | | |
| 8372 | return self.wip.bin(.@"add nsw", correction, rem, "mod"); | | |
| 8373 | } | | |
| 8374 | return self.wip.bin(.urem, lhs, rhs, ""); | | |
| 8375 | } | | |
| 8376 | | | |
| 8377 | fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8378 | const o = self.ng.object; | | |
| 8379 | const pt = self.ng.pt; | | |
| 8380 | const zcu = pt.zcu; | | |
| 8381 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 8382 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; | | |
| 8383 | const ptr = try self.resolveInst(bin_op.lhs); | | |
| 8384 | const offset = try self.resolveInst(bin_op.rhs); | | |
| 8385 | const ptr_ty = self.typeOf(bin_op.lhs); | | |
| 8386 | const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu)); | | |
| 8387 | switch (ptr_ty.ptrSize(zcu)) { | | |
| 8388 | // It's a pointer to an array, so according to LLVM we need an extra GEP index. | | |
| 8389 | .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{ | | |
| 8390 | try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), offset, | | |
| 8391 | }, ""), | | |
| 8392 | .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{offset}, ""), | | |
| 8393 | .slice => { | | |
| 8394 | const base = try self.wip.extractValue(ptr, &.{0}, ""); | | |
| 8395 | return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{offset}, ""); | | |
| 8396 | }, | | |
| 8397 | } | | |
| 8398 | } | | |
| 8399 | | | |
| 8400 | fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8401 | const o = self.ng.object; | | |
| 8402 | const pt = self.ng.pt; | | |
| 8403 | const zcu = pt.zcu; | | |
| 8404 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 8405 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; | | |
| 8406 | const ptr = try self.resolveInst(bin_op.lhs); | | |
| 8407 | const offset = try self.resolveInst(bin_op.rhs); | | |
| 8408 | const negative_offset = try self.wip.neg(offset, ""); | | |
| 8409 | const ptr_ty = self.typeOf(bin_op.lhs); | | |
| 8410 | const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu)); | | |
| 8411 | switch (ptr_ty.ptrSize(zcu)) { | | |
| 8412 | // It's a pointer to an array, so according to LLVM we need an extra GEP index. | | |
| 8413 | .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{ | | |
| 8414 | try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), negative_offset, | | |
| 8415 | }, ""), | | |
| 8416 | .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{negative_offset}, ""), | | |
| 8417 | .slice => { | | |
| 8418 | const base = try self.wip.extractValue(ptr, &.{0}, ""); | | |
| 8419 | return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{negative_offset}, ""); | | |
| 8420 | }, | | |
| 8421 | } | | |
| 8422 | } | | |
| 8423 | | | |
| 8424 | fn airOverflow( | | |
| 8425 | self: *FuncGen, | | |
| 8426 | inst: Air.Inst.Index, | | |
| 8427 | signed_intrinsic: Builder.Intrinsic, | | |
| 8428 | unsigned_intrinsic: Builder.Intrinsic, | | |
| 8429 | ) !Builder.Value { | | |
| 8430 | const o = self.ng.object; | | |
| 8431 | const pt = self.ng.pt; | | |
| 8432 | const zcu = pt.zcu; | | |
| 8433 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 8434 | const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; | | |
| 8435 | | | |
| 8436 | const lhs = try self.resolveInst(extra.lhs); | | |
| 8437 | const rhs = try self.resolveInst(extra.rhs); | | |
| 8438 | | | |
| 8439 | const lhs_ty = self.typeOf(extra.lhs); | | |
| 8440 | const scalar_ty = lhs_ty.scalarType(zcu); | | |
| 8441 | const inst_ty = self.typeOfIndex(inst); | | |
| 8442 | | | |
| 8443 | const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic; | | |
| 8444 | const llvm_inst_ty = try o.lowerType(pt, inst_ty); | | |
| 8445 | const llvm_lhs_ty = try o.lowerType(pt, lhs_ty); | | |
| 8446 | const results = | | |
| 8447 | try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, ""); | | |
| 8448 | | | |
| 8449 | const result_val = try self.wip.extractValue(results, &.{0}, ""); | | |
| 8450 | const overflow_bit = try self.wip.extractValue(results, &.{1}, ""); | | |
| 8451 | | | |
| 8452 | const result_index = o.llvmFieldIndex(inst_ty, 0).?; | | |
| 8453 | const overflow_index = o.llvmFieldIndex(inst_ty, 1).?; | | |
| 8454 | | | |
| 8455 | if (isByRef(inst_ty, zcu)) { | | |
| 8456 | const result_alignment = inst_ty.abiAlignment(zcu).toLlvm(); | | |
| 8457 | const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment); | | |
| 8458 | { | | |
| 8459 | const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, ""); | | |
| 8460 | _ = try self.wip.store(.normal, result_val, field_ptr, result_alignment); | | |
| 8461 | } | | |
| 8462 | { | | |
| 8463 | const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, overflow_index, ""); | | |
| 8464 | _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1)); | | |
| 8465 | } | | |
| 8466 | | | |
| 8467 | return alloca_inst; | | |
| 8468 | } | | |
| 8469 | | | |
| 8470 | var fields: [2]Builder.Value = undefined; | | |
| 8471 | fields[result_index] = result_val; | | |
| 8472 | fields[overflow_index] = overflow_bit; | | |
| 8473 | return self.wip.buildAggregate(llvm_inst_ty, &fields, ""); | | |
| 8474 | } | | |
| 8475 | | | |
| 8476 | fn buildElementwiseCall( | | |
| 8477 | self: *FuncGen, | | |
| 8478 | llvm_fn: Builder.Function.Index, | | |
| 8479 | args_vectors: []const Builder.Value, | | |
| 8480 | result_vector: Builder.Value, | | |
| 8481 | vector_len: usize, | | |
| 8482 | ) !Builder.Value { | | |
| 8483 | const o = self.ng.object; | | |
| 8484 | assert(args_vectors.len <= 3); | | |
| 8485 | | | |
| 8486 | var i: usize = 0; | | |
| 8487 | var result = result_vector; | | |
| 8488 | while (i < vector_len) : (i += 1) { | | |
| 8489 | const index_i32 = try o.builder.intValue(.i32, i); | | |
| 8490 | | | |
| 8491 | var args: [3]Builder.Value = undefined; | | |
| 8492 | for (args[0..args_vectors.len], args_vectors) |*arg_elem, arg_vector| { | | |
| 8493 | arg_elem.* = try self.wip.extractElement(arg_vector, index_i32, ""); | | |
| 8494 | } | | |
| 8495 | const result_elem = try self.wip.call( | | |
| 8496 | .normal, | | |
| 8497 | .ccc, | | |
| 8498 | .none, | | |
| 8499 | llvm_fn.typeOf(&o.builder), | | |
| 8500 | llvm_fn.toValue(&o.builder), | | |
| 8501 | args[0..args_vectors.len], | | |
| 8502 | "", | | |
| 8503 | ); | | |
| 8504 | result = try self.wip.insertElement(result, result_elem, index_i32, ""); | | |
| 8505 | } | | |
| 8506 | return result; | | |
| 8507 | } | | |
| 8508 | | | |
| 8509 | fn getLibcFunction( | | |
| 8510 | self: *FuncGen, | | |
| 8511 | fn_name: Builder.StrtabString, | | |
| 8512 | param_types: []const Builder.Type, | | |
| 8513 | return_type: Builder.Type, | | |
| 8514 | ) Allocator.Error!Builder.Function.Index { | | |
| 8515 | const o = self.ng.object; | | |
| 8516 | if (o.builder.getGlobal(fn_name)) |global| return switch (global.ptrConst(&o.builder).kind) { | | |
| 8517 | .alias => |alias| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function, | | |
| 8518 | .function => |function| function, | | |
| 8519 | .variable, .replaced => unreachable, | | |
| 8520 | }; | | |
| 8521 | return o.builder.addFunction( | | |
| 8522 | try o.builder.fnType(return_type, param_types, .normal), | | |
| 8523 | fn_name, | | |
| 8524 | toLlvmAddressSpace(.generic, self.ng.pt.zcu.getTarget()), | | |
| 8525 | ); | | |
| 8526 | } | | |
| 8527 | | | |
| 8528 | /// Creates a floating point comparison by lowering to the appropriate | | |
| 8529 | /// hardware instruction or softfloat routine for the target | | |
| 8530 | fn buildFloatCmp( | | |
| 8531 | self: *FuncGen, | | |
| 8532 | fast: Builder.FastMathKind, | | |
| 8533 | pred: math.CompareOperator, | | |
| 8534 | ty: Type, | | |
| 8535 | params: [2]Builder.Value, | | |
| 8536 | ) !Builder.Value { | | |
| 8537 | const o = self.ng.object; | | |
| 8538 | const pt = self.ng.pt; | | |
| 8539 | const zcu = pt.zcu; | | |
| 8540 | const target = zcu.getTarget(); | | |
| 8541 | const scalar_ty = ty.scalarType(zcu); | | |
| 8542 | const scalar_llvm_ty = try o.lowerType(pt, scalar_ty); | | |
| 8543 | | | |
| 8544 | if (intrinsicsAllowed(scalar_ty, target)) { | | |
| 8545 | const cond: Builder.FloatCondition = switch (pred) { | | |
| 8546 | .eq => .oeq, | | |
| 8547 | .neq => .une, | | |
| 8548 | .lt => .olt, | | |
| 8549 | .lte => .ole, | | |
| 8550 | .gt => .ogt, | | |
| 8551 | .gte => .oge, | | |
| 8552 | }; | | |
| 8553 | return self.wip.fcmp(fast, cond, params[0], params[1], ""); | | |
| 8554 | } | | |
| 8555 | | | |
| 8556 | const float_bits = scalar_ty.floatBits(target); | | |
| 8557 | const compiler_rt_float_abbrev = compilerRtFloatAbbrev(float_bits); | | |
| 8558 | const fn_base_name = switch (pred) { | | |
| 8559 | .neq => "ne", | | |
| 8560 | .eq => "eq", | | |
| 8561 | .lt => "lt", | | |
| 8562 | .lte => "le", | | |
| 8563 | .gt => "gt", | | |
| 8564 | .gte => "ge", | | |
| 8565 | }; | | |
| 8566 | const fn_name = try o.builder.strtabStringFmt("__{s}{s}f2", .{ fn_base_name, compiler_rt_float_abbrev }); | | |
| 8567 | | | |
| 8568 | const libc_fn = try self.getLibcFunction(fn_name, &.{ scalar_llvm_ty, scalar_llvm_ty }, .i32); | | |
| 8569 | | | |
| 8570 | const int_cond: Builder.IntegerCondition = switch (pred) { | | |
| 8571 | .eq => .eq, | | |
| 8572 | .neq => .ne, | | |
| 8573 | .lt => .slt, | | |
| 8574 | .lte => .sle, | | |
| 8575 | .gt => .sgt, | | |
| 8576 | .gte => .sge, | | |
| 8577 | }; | | |
| 8578 | | | |
| 8579 | if (ty.zigTypeTag(zcu) == .vector) { | | |
| 8580 | const vec_len = ty.vectorLen(zcu); | | |
| 8581 | const vector_result_ty = try o.builder.vectorType(.normal, vec_len, .i32); | | |
| 8582 | | | |
| 8583 | const init = try o.builder.poisonValue(vector_result_ty); | | |
| 8584 | const result = try self.buildElementwiseCall(libc_fn, ¶ms, init, vec_len); | | |
| 8585 | | | |
| 8586 | const zero_vector = try o.builder.splatValue(vector_result_ty, .@"0"); | | |
| 8587 | return self.wip.icmp(int_cond, result, zero_vector, ""); | | |
| 8588 | } | | |
| 8589 | | | |
| 8590 | const result = try self.wip.call( | | |
| 8591 | .normal, | | |
| 8592 | .ccc, | | |
| 8593 | .none, | | |
| 8594 | libc_fn.typeOf(&o.builder), | | |
| 8595 | libc_fn.toValue(&o.builder), | | |
| 8596 | ¶ms, | | |
| 8597 | "", | | |
| 8598 | ); | | |
| 8599 | return self.wip.icmp(int_cond, result, .@"0", ""); | | |
| 8600 | } | | |
| 8601 | | | |
| 8602 | const FloatOp = enum { | | |
| 8603 | add, | | |
| 8604 | ceil, | | |
| 8605 | cos, | | |
| 8606 | div, | | |
| 8607 | exp, | | |
| 8608 | exp2, | | |
| 8609 | fabs, | | |
| 8610 | floor, | | |
| 8611 | fma, | | |
| 8612 | fmax, | | |
| 8613 | fmin, | | |
| 8614 | fmod, | | |
| 8615 | log, | | |
| 8616 | log10, | | |
| 8617 | log2, | | |
| 8618 | mul, | | |
| 8619 | neg, | | |
| 8620 | round, | | |
| 8621 | sin, | | |
| 8622 | sqrt, | | |
| 8623 | sub, | | |
| 8624 | tan, | | |
| 8625 | trunc, | | |
| 8626 | }; | | |
| 8627 | | | |
| 8628 | const FloatOpStrat = union(enum) { | | |
| 8629 | intrinsic: []const u8, | | |
| 8630 | libc: Builder.String, | | |
| 8631 | }; | | |
| 8632 | | | |
| 8633 | /// Creates a floating point operation (add, sub, fma, sqrt, exp, etc.) | | |
| 8634 | /// by lowering to the appropriate hardware instruction or softfloat | | |
| 8635 | /// routine for the target | | |
| 8636 | fn buildFloatOp( | | |
| 8637 | self: *FuncGen, | | |
| 8638 | comptime op: FloatOp, | | |
| 8639 | fast: Builder.FastMathKind, | | |
| 8640 | ty: Type, | | |
| 8641 | comptime params_len: usize, | | |
| 8642 | params: [params_len]Builder.Value, | | |
| 8643 | ) !Builder.Value { | | |
| 8644 | const o = self.ng.object; | | |
| 8645 | const pt = self.ng.pt; | | |
| 8646 | const zcu = pt.zcu; | | |
| 8647 | const target = zcu.getTarget(); | | |
| 8648 | const scalar_ty = ty.scalarType(zcu); | | |
| 8649 | const llvm_ty = try o.lowerType(pt, ty); | | |
| 8650 | | | |
| 8651 | if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) { | | |
| 8652 | // Some operations are dedicated LLVM instructions, not available as intrinsics | | |
| 8653 | .neg => return self.wip.un(.fneg, params[0], ""), | | |
| 8654 | .add, .sub, .mul, .div, .fmod => return self.wip.bin(switch (fast) { | | |
| 8655 | .normal => switch (op) { | | |
| 8656 | .add => .fadd, | | |
| 8657 | .sub => .fsub, | | |
| 8658 | .mul => .fmul, | | |
| 8659 | .div => .fdiv, | | |
| 8660 | .fmod => .frem, | | |
| 8661 | else => unreachable, | | |
| 8662 | }, | | |
| 8663 | .fast => switch (op) { | | |
| 8664 | .add => .@"fadd fast", | | |
| 8665 | .sub => .@"fsub fast", | | |
| 8666 | .mul => .@"fmul fast", | | |
| 8667 | .div => .@"fdiv fast", | | |
| 8668 | .fmod => .@"frem fast", | | |
| 8669 | else => unreachable, | | |
| 8670 | }, | | |
| 8671 | }, params[0], params[1], ""), | | |
| 8672 | .fmax, | | |
| 8673 | .fmin, | | |
| 8674 | .ceil, | | |
| 8675 | .cos, | | |
| 8676 | .exp, | | |
| 8677 | .exp2, | | |
| 8678 | .fabs, | | |
| 8679 | .floor, | | |
| 8680 | .log, | | |
| 8681 | .log10, | | |
| 8682 | .log2, | | |
| 8683 | .round, | | |
| 8684 | .sin, | | |
| 8685 | .sqrt, | | |
| 8686 | .trunc, | | |
| 8687 | .fma, | | |
| 8688 | => return self.wip.callIntrinsic(fast, .none, switch (op) { | | |
| 8689 | .fmax => .maxnum, | | |
| 8690 | .fmin => .minnum, | | |
| 8691 | .ceil => .ceil, | | |
| 8692 | .cos => .cos, | | |
| 8693 | .exp => .exp, | | |
| 8694 | .exp2 => .exp2, | | |
| 8695 | .fabs => .fabs, | | |
| 8696 | .floor => .floor, | | |
| 8697 | .log => .log, | | |
| 8698 | .log10 => .log10, | | |
| 8699 | .log2 => .log2, | | |
| 8700 | .round => .round, | | |
| 8701 | .sin => .sin, | | |
| 8702 | .sqrt => .sqrt, | | |
| 8703 | .trunc => .trunc, | | |
| 8704 | .fma => .fma, | | |
| 8705 | else => unreachable, | | |
| 8706 | }, &.{llvm_ty}, ¶ms, ""), | | |
| 8707 | .tan => unreachable, | | |
| 8708 | }; | | |
| 8709 | | | |
| 8710 | const float_bits = scalar_ty.floatBits(target); | | |
| 8711 | const fn_name = switch (op) { | | |
| 8712 | .neg => { | | |
| 8713 | // In this case we can generate a softfloat negation by XORing the | | |
| 8714 | // bits with a constant. | | |
| 8715 | const int_ty = try o.builder.intType(@intCast(float_bits)); | | |
| 8716 | const cast_ty = try llvm_ty.changeScalar(int_ty, &o.builder); | | |
| 8717 | const sign_mask = try o.builder.splatValue( | | |
| 8718 | cast_ty, | | |
| 8719 | try o.builder.intConst(int_ty, @as(u128, 1) << @intCast(float_bits - 1)), | | |
| 8720 | ); | | |
| 8721 | const bitcasted_operand = try self.wip.cast(.bitcast, params[0], cast_ty, ""); | | |
| 8722 | const result = try self.wip.bin(.xor, bitcasted_operand, sign_mask, ""); | | |
| 8723 | return self.wip.cast(.bitcast, result, llvm_ty, ""); | | |
| 8724 | }, | | |
| 8725 | .add, .sub, .div, .mul => try o.builder.strtabStringFmt("__{s}{s}f3", .{ | | |
| 8726 | @tagName(op), compilerRtFloatAbbrev(float_bits), | | |
| 8727 | }), | | |
| 8728 | .ceil, | | |
| 8729 | .cos, | | |
| 8730 | .exp, | | |
| 8731 | .exp2, | | |
| 8732 | .fabs, | | |
| 8733 | .floor, | | |
| 8734 | .fma, | | |
| 8735 | .fmax, | | |
| 8736 | .fmin, | | |
| 8737 | .fmod, | | |
| 8738 | .log, | | |
| 8739 | .log10, | | |
| 8740 | .log2, | | |
| 8741 | .round, | | |
| 8742 | .sin, | | |
| 8743 | .sqrt, | | |
| 8744 | .tan, | | |
| 8745 | .trunc, | | |
| 8746 | => try o.builder.strtabStringFmt("{s}{s}{s}", .{ | | |
| 8747 | libcFloatPrefix(float_bits), @tagName(op), libcFloatSuffix(float_bits), | | |
| 8748 | }), | | |
| 8749 | }; | | |
| 8750 | | | |
| 8751 | const scalar_llvm_ty = llvm_ty.scalarType(&o.builder); | | |
| 8752 | const libc_fn = try self.getLibcFunction( | | |
| 8753 | fn_name, | | |
| 8754 | ([1]Builder.Type{scalar_llvm_ty} ** 3)[0..params.len], | | |
| 8755 | scalar_llvm_ty, | | |
| 8756 | ); | | |
| 8757 | if (ty.zigTypeTag(zcu) == .vector) { | | |
| 8758 | const result = try o.builder.poisonValue(llvm_ty); | | |
| 8759 | return self.buildElementwiseCall(libc_fn, ¶ms, result, ty.vectorLen(zcu)); | | |
| 8760 | } | | |
| 8761 | | | |
| 8762 | return self.wip.call( | | |
| 8763 | fast.toCallKind(), | | |
| 8764 | .ccc, | | |
| 8765 | .none, | | |
| 8766 | libc_fn.typeOf(&o.builder), | | |
| 8767 | libc_fn.toValue(&o.builder), | | |
| 8768 | ¶ms, | | |
| 8769 | "", | | |
| 8770 | ); | | |
| 8771 | } | | |
| 8772 | | | |
| 8773 | fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8774 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | | |
| 8775 | const extra = self.air.extraData(Air.Bin, pl_op.payload).data; | | |
| 8776 | | | |
| 8777 | const mulend1 = try self.resolveInst(extra.lhs); | | |
| 8778 | const mulend2 = try self.resolveInst(extra.rhs); | | |
| 8779 | const addend = try self.resolveInst(pl_op.operand); | | |
| 8780 | | | |
| 8781 | const ty = self.typeOfIndex(inst); | | |
| 8782 | return self.buildFloatOp(.fma, .normal, ty, 3, .{ mulend1, mulend2, addend }); | | |
| 8783 | } | | |
| 8784 | | | |
| 8785 | fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8786 | const o = self.ng.object; | | |
| 8787 | const pt = self.ng.pt; | | |
| 8788 | const zcu = pt.zcu; | | |
| 8789 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 8790 | const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; | | |
| 8791 | | | |
| 8792 | const lhs = try self.resolveInst(extra.lhs); | | |
| 8793 | const rhs = try self.resolveInst(extra.rhs); | | |
| 8794 | | | |
| 8795 | const lhs_ty = self.typeOf(extra.lhs); | | |
| 8796 | if (lhs_ty.isVector(zcu) and !self.typeOf(extra.rhs).isVector(zcu)) | | |
| 8797 | return self.ng.todo("implement vector shifts with scalar rhs", .{}); | | |
| 8798 | const lhs_scalar_ty = lhs_ty.scalarType(zcu); | | |
| 8799 | | | |
| 8800 | const dest_ty = self.typeOfIndex(inst); | | |
| 8801 | const llvm_dest_ty = try o.lowerType(pt, dest_ty); | | |
| 8802 | | | |
| 8803 | const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); | | |
| 8804 | | | |
| 8805 | const result = try self.wip.bin(.shl, lhs, casted_rhs, ""); | | |
| 8806 | const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu)) | | |
| 8807 | .ashr | | |
| 8808 | else | | |
| 8809 | .lshr, result, casted_rhs, ""); | | |
| 8810 | | | |
| 8811 | const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, ""); | | |
| 8812 | | | |
| 8813 | const result_index = o.llvmFieldIndex(dest_ty, 0).?; | | |
| 8814 | const overflow_index = o.llvmFieldIndex(dest_ty, 1).?; | | |
| 8815 | | | |
| 8816 | if (isByRef(dest_ty, zcu)) { | | |
| 8817 | const result_alignment = dest_ty.abiAlignment(zcu).toLlvm(); | | |
| 8818 | const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment); | | |
| 8819 | { | | |
| 8820 | const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, ""); | | |
| 8821 | _ = try self.wip.store(.normal, result, field_ptr, result_alignment); | | |
| 8822 | } | | |
| 8823 | { | | |
| 8824 | const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, overflow_index, ""); | | |
| 8825 | _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1)); | | |
| 8826 | } | | |
| 8827 | return alloca_inst; | | |
| 8828 | } | | |
| 8829 | | | |
| 8830 | var fields: [2]Builder.Value = undefined; | | |
| 8831 | fields[result_index] = result; | | |
| 8832 | fields[overflow_index] = overflow_bit; | | |
| 8833 | return self.wip.buildAggregate(llvm_dest_ty, &fields, ""); | | |
| 8834 | } | | |
| 8835 | | | |
| 8836 | fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8837 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8838 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8839 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8840 | return self.wip.bin(.@"and", lhs, rhs, ""); | | |
| 8841 | } | | |
| 8842 | | | |
| 8843 | fn airOr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8844 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8845 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8846 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8847 | return self.wip.bin(.@"or", lhs, rhs, ""); | | |
| 8848 | } | | |
| 8849 | | | |
| 8850 | fn airXor(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8851 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8852 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8853 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8854 | return self.wip.bin(.xor, lhs, rhs, ""); | | |
| 8855 | } | | |
| 8856 | | | |
| 8857 | fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8858 | const o = self.ng.object; | | |
| 8859 | const pt = self.ng.pt; | | |
| 8860 | const zcu = pt.zcu; | | |
| 8861 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8862 | | | |
| 8863 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8864 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8865 | | | |
| 8866 | const lhs_ty = self.typeOf(bin_op.lhs); | | |
| 8867 | if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) | | |
| 8868 | return self.ng.todo("implement vector shifts with scalar rhs", .{}); | | |
| 8869 | const lhs_scalar_ty = lhs_ty.scalarType(zcu); | | |
| 8870 | | | |
| 8871 | const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); | | |
| 8872 | return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu)) | | |
| 8873 | .@"shl nsw" | | |
| 8874 | else | | |
| 8875 | .@"shl nuw", lhs, casted_rhs, ""); | | |
| 8876 | } | | |
| 8877 | | | |
| 8878 | fn airShl(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8879 | const o = self.ng.object; | | |
| 8880 | const pt = self.ng.pt; | | |
| 8881 | const zcu = pt.zcu; | | |
| 8882 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8883 | | | |
| 8884 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8885 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8886 | | | |
| 8887 | const lhs_ty = self.typeOf(bin_op.lhs); | | |
| 8888 | if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) | | |
| 8889 | return self.ng.todo("implement vector shifts with scalar rhs", .{}); | | |
| 8890 | | | |
| 8891 | const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); | | |
| 8892 | return self.wip.bin(.shl, lhs, casted_rhs, ""); | | |
| 8893 | } | | |
| 8894 | | | |
| 8895 | fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8896 | const o = self.ng.object; | | |
| 8897 | const pt = self.ng.pt; | | |
| 8898 | const zcu = pt.zcu; | | |
| 8899 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8900 | | | |
| 8901 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8902 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8903 | | | |
| 8904 | const lhs_ty = self.typeOf(bin_op.lhs); | | |
| 8905 | const lhs_info = lhs_ty.intInfo(zcu); | | |
| 8906 | const llvm_lhs_ty = try o.lowerType(pt, lhs_ty); | | |
| 8907 | const llvm_lhs_scalar_ty = llvm_lhs_ty.scalarType(&o.builder); | | |
| 8908 | | | |
| 8909 | const rhs_ty = self.typeOf(bin_op.rhs); | | |
| 8910 | if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) | | |
| 8911 | return self.ng.todo("implement vector shifts with scalar rhs", .{}); | | |
| 8912 | const rhs_info = rhs_ty.intInfo(zcu); | | |
| 8913 | assert(rhs_info.signedness == .unsigned); | | |
| 8914 | const llvm_rhs_ty = try o.lowerType(pt, rhs_ty); | | |
| 8915 | const llvm_rhs_scalar_ty = llvm_rhs_ty.scalarType(&o.builder); | | |
| 8916 | | | |
| 8917 | const result = try self.wip.callIntrinsic( | | |
| 8918 | .normal, | | |
| 8919 | .none, | | |
| 8920 | switch (lhs_info.signedness) { | | |
| 8921 | .signed => .@"sshl.sat", | | |
| 8922 | .unsigned => .@"ushl.sat", | | |
| 8923 | }, | | |
| 8924 | &.{llvm_lhs_ty}, | | |
| 8925 | &.{ lhs, try self.wip.conv(.unsigned, rhs, llvm_lhs_ty, "") }, | | |
| 8926 | "", | | |
| 8927 | ); | | |
| 8928 | | | |
| 8929 | // LLVM langref says "If b is (statically or dynamically) equal to or | | |
| 8930 | // larger than the integer bit width of the arguments, the result is a | | |
| 8931 | // poison value." | | |
| 8932 | // However Zig semantics says that saturating shift left can never produce | | |
| 8933 | // undefined; instead it saturates. | | |
| 8934 | if (rhs_info.bits <= math.log2_int(u16, lhs_info.bits)) return result; | | |
| 8935 | const bits = try o.builder.splatValue( | | |
| 8936 | llvm_rhs_ty, | | |
| 8937 | try o.builder.intConst(llvm_rhs_scalar_ty, lhs_info.bits), | | |
| 8938 | ); | | |
| 8939 | const in_range = try self.wip.icmp(.ult, rhs, bits, ""); | | |
| 8940 | const lhs_sat = lhs_sat: switch (lhs_info.signedness) { | | |
| 8941 | .signed => { | | |
| 8942 | const zero = try o.builder.splatValue( | | |
| 8943 | llvm_lhs_ty, | | |
| 8944 | try o.builder.intConst(llvm_lhs_scalar_ty, 0), | | |
| 8945 | ); | | |
| 8946 | const smin = try o.builder.splatValue( | | |
| 8947 | llvm_lhs_ty, | | |
| 8948 | try minIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu), | | |
| 8949 | ); | | |
| 8950 | const smax = try o.builder.splatValue( | | |
| 8951 | llvm_lhs_ty, | | |
| 8952 | try maxIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu), | | |
| 8953 | ); | | |
| 8954 | const lhs_lt_zero = try self.wip.icmp(.slt, lhs, zero, ""); | | |
| 8955 | const slimit = try self.wip.select(.normal, lhs_lt_zero, smin, smax, ""); | | |
| 8956 | const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, ""); | | |
| 8957 | break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, slimit, ""); | | |
| 8958 | }, | | |
| 8959 | .unsigned => { | | |
| 8960 | const zero = try o.builder.splatValue( | | |
| 8961 | llvm_lhs_ty, | | |
| 8962 | try o.builder.intConst(llvm_lhs_scalar_ty, 0), | | |
| 8963 | ); | | |
| 8964 | const umax = try o.builder.splatValue( | | |
| 8965 | llvm_lhs_ty, | | |
| 8966 | try o.builder.intConst(llvm_lhs_scalar_ty, -1), | | |
| 8967 | ); | | |
| 8968 | const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, ""); | | |
| 8969 | break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, umax, ""); | | |
| 8970 | }, | | |
| 8971 | }; | | |
| 8972 | return self.wip.select(.normal, in_range, result, lhs_sat, ""); | | |
| 8973 | } | | |
| 8974 | | | |
| 8975 | fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) !Builder.Value { | | |
| 8976 | const o = self.ng.object; | | |
| 8977 | const pt = self.ng.pt; | | |
| 8978 | const zcu = pt.zcu; | | |
| 8979 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 8980 | | | |
| 8981 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 8982 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 8983 | | | |
| 8984 | const lhs_ty = self.typeOf(bin_op.lhs); | | |
| 8985 | if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) | | |
| 8986 | return self.ng.todo("implement vector shifts with scalar rhs", .{}); | | |
| 8987 | const lhs_scalar_ty = lhs_ty.scalarType(zcu); | | |
| 8988 | | | |
| 8989 | const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); | | |
| 8990 | const is_signed_int = lhs_scalar_ty.isSignedInt(zcu); | | |
| 8991 | | | |
| 8992 | return self.wip.bin(if (is_exact) | | |
| 8993 | if (is_signed_int) .@"ashr exact" else .@"lshr exact" | | |
| 8994 | else if (is_signed_int) .ashr else .lshr, lhs, casted_rhs, ""); | | |
| 8995 | } | | |
| 8996 | | | |
| 8997 | fn airAbs(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 8998 | const o = self.ng.object; | | |
| 8999 | const pt = self.ng.pt; | | |
| 9000 | const zcu = pt.zcu; | | |
| 9001 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 9002 | const operand = try self.resolveInst(ty_op.operand); | | |
| 9003 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 9004 | const scalar_ty = operand_ty.scalarType(zcu); | | |
| 9005 | | | |
| 9006 | switch (scalar_ty.zigTypeTag(zcu)) { | | |
| 9007 | .int => return self.wip.callIntrinsic( | | |
| 9008 | .normal, | | |
| 9009 | .none, | | |
| 9010 | .abs, | | |
| 9011 | &.{try o.lowerType(pt, operand_ty)}, | | |
| 9012 | &.{ operand, try o.builder.intValue(.i1, 0) }, | | |
| 9013 | "", | | |
| 9014 | ), | | |
| 9015 | .float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}), | | |
| 9016 | else => unreachable, | | |
| 9017 | } | | |
| 9018 | } | | |
| 9019 | | | |
| 9020 | fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { | | |
| 9021 | const o = fg.ng.object; | | |
| 9022 | const pt = fg.ng.pt; | | |
| 9023 | const zcu = pt.zcu; | | |
| 9024 | const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 9025 | const dest_ty = fg.typeOfIndex(inst); | | |
| 9026 | const dest_llvm_ty = try o.lowerType(pt, dest_ty); | | |
| 9027 | const operand = try fg.resolveInst(ty_op.operand); | | |
| 9028 | const operand_ty = fg.typeOf(ty_op.operand); | | |
| 9029 | const operand_info = operand_ty.intInfo(zcu); | | |
| 9030 | | | |
| 9031 | const dest_is_enum = dest_ty.zigTypeTag(zcu) == .@"enum"; | | |
| 9032 | | | |
| 9033 | bounds_check: { | | |
| 9034 | const dest_scalar = dest_ty.scalarType(zcu); | | |
| 9035 | const operand_scalar = operand_ty.scalarType(zcu); | | |
| 9036 | | | |
| 9037 | const dest_info = dest_ty.intInfo(zcu); | | |
| 9038 | | | |
| 9039 | const have_min_check, const have_max_check = c: { | | |
| 9040 | const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed); | | |
| 9041 | const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed); | | |
| 9042 | | | |
| 9043 | const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0; | | |
| 9044 | const operand_maybe_neg = operand_info.signedness == .signed and operand_info.bits > 0; | | |
| 9045 | | | |
| 9046 | break :c .{ | | |
| 9047 | operand_maybe_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits), | | |
| 9048 | dest_pos_bits < operand_pos_bits, | | |
| 9049 | }; | | |
| 9050 | }; | | |
| 9051 | | | |
| 9052 | if (!have_min_check and !have_max_check) break :bounds_check; | | |
| 9053 | | | |
| 9054 | const operand_llvm_ty = try o.lowerType(pt, operand_ty); | | |
| 9055 | const operand_scalar_llvm_ty = try o.lowerType(pt, operand_scalar); | | |
| 9056 | | | |
| 9057 | const is_vector = operand_ty.zigTypeTag(zcu) == .vector; | | |
| 9058 | assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector)); | | |
| 9059 | | | |
| 9060 | const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .integer_out_of_bounds; | | |
| 9061 | | | |
| 9062 | if (have_min_check) { | | |
| 9063 | const min_const_scalar = try minIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu); | | |
| 9064 | const min_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, min_const_scalar) else min_const_scalar.toValue(); | | |
| 9065 | const ok_maybe_vec = try fg.cmp(.normal, .gte, operand_ty, operand, min_val); | | |
| 9066 | const ok = if (is_vector) ok: { | | |
| 9067 | const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip); | | |
| 9068 | break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, ""); | | |
| 9069 | } else ok_maybe_vec; | | |
| 9070 | if (safety) { | | |
| 9071 | const fail_block = try fg.wip.block(1, "IntMinFail"); | | |
| 9072 | const ok_block = try fg.wip.block(1, "IntMinOk"); | | |
| 9073 | _ = try fg.wip.brCond(ok, ok_block, fail_block, .none); | | |
| 9074 | fg.wip.cursor = .{ .block = fail_block }; | | |
| 9075 | try fg.buildSimplePanic(panic_id); | | |
| 9076 | fg.wip.cursor = .{ .block = ok_block }; | | |
| 9077 | } else { | | |
| 9078 | _ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, ""); | | |
| 9079 | } | | |
| 9080 | } | | |
| 9081 | | | |
| 9082 | if (have_max_check) { | | |
| 9083 | const max_const_scalar = try maxIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu); | | |
| 9084 | const max_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, max_const_scalar) else max_const_scalar.toValue(); | | |
| 9085 | const ok_maybe_vec = try fg.cmp(.normal, .lte, operand_ty, operand, max_val); | | |
| 9086 | const ok = if (is_vector) ok: { | | |
| 9087 | const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip); | | |
| 9088 | break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, ""); | | |
| 9089 | } else ok_maybe_vec; | | |
| 9090 | if (safety) { | | |
| 9091 | const fail_block = try fg.wip.block(1, "IntMaxFail"); | | |
| 9092 | const ok_block = try fg.wip.block(1, "IntMaxOk"); | | |
| 9093 | _ = try fg.wip.brCond(ok, ok_block, fail_block, .none); | | |
| 9094 | fg.wip.cursor = .{ .block = fail_block }; | | |
| 9095 | try fg.buildSimplePanic(panic_id); | | |
| 9096 | fg.wip.cursor = .{ .block = ok_block }; | | |
| 9097 | } else { | | |
| 9098 | _ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, ""); | | |
| 9099 | } | | |
| 9100 | } | | |
| 9101 | } | | |
| 9102 | | | |
| 9103 | const result = try fg.wip.conv(switch (operand_info.signedness) { | | |
| 9104 | .signed => .signed, | | |
| 9105 | .unsigned => .unsigned, | | |
| 9106 | }, operand, dest_llvm_ty, ""); | | |
| 9107 | | | |
| 9108 | if (safety and dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu)) { | | |
| 9109 | const llvm_fn = try fg.getIsNamedEnumValueFunction(dest_ty); | | |
| 9110 | const is_valid_enum_val = try fg.wip.call( | | |
| 9111 | .normal, | | |
| 9112 | .fastcc, | | |
| 9113 | .none, | | |
| 9114 | llvm_fn.typeOf(&o.builder), | | |
| 9115 | llvm_fn.toValue(&o.builder), | | |
| 9116 | &.{result}, | | |
| 9117 | "", | | |
| 9118 | ); | | |
| 9119 | const fail_block = try fg.wip.block(1, "ValidEnumFail"); | | |
| 9120 | const ok_block = try fg.wip.block(1, "ValidEnumOk"); | | |
| 9121 | _ = try fg.wip.brCond(is_valid_enum_val, ok_block, fail_block, .none); | | |
| 9122 | fg.wip.cursor = .{ .block = fail_block }; | | |
| 9123 | try fg.buildSimplePanic(.invalid_enum_value); | | |
| 9124 | fg.wip.cursor = .{ .block = ok_block }; | | |
| 9125 | } | | |
| 9126 | | | |
| 9127 | return result; | | |
| 9128 | } | | |
| 9129 | | | |
| 9130 | fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9131 | const o = self.ng.object; | | |
| 9132 | const pt = self.ng.pt; | | |
| 9133 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 9134 | const operand = try self.resolveInst(ty_op.operand); | | |
| 9135 | const dest_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst)); | | |
| 9136 | return self.wip.cast(.trunc, operand, dest_llvm_ty, ""); | | |
| 9137 | } | | |
| 9138 | | | |
| 9139 | fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9140 | const o = self.ng.object; | | |
| 9141 | const pt = self.ng.pt; | | |
| 9142 | const zcu = pt.zcu; | | |
| 9143 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 9144 | const operand = try self.resolveInst(ty_op.operand); | | |
| 9145 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 9146 | const dest_ty = self.typeOfIndex(inst); | | |
| 9147 | const target = zcu.getTarget(); | | |
| 9148 | | | |
| 9149 | if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) { | | |
| 9150 | return self.wip.cast(.fptrunc, operand, try o.lowerType(pt, dest_ty), ""); | | |
| 9151 | } else { | | |
| 9152 | const operand_llvm_ty = try o.lowerType(pt, operand_ty); | | |
| 9153 | const dest_llvm_ty = try o.lowerType(pt, dest_ty); | | |
| 9154 | | | |
| 9155 | const dest_bits = dest_ty.floatBits(target); | | |
| 9156 | const src_bits = operand_ty.floatBits(target); | | |
| 9157 | const fn_name = try o.builder.strtabStringFmt("__trunc{s}f{s}f2", .{ | | |
| 9158 | compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits), | | |
| 9159 | }); | | |
| 9160 | | | |
| 9161 | const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty); | | |
| 9162 | return self.wip.call( | | |
| 9163 | .normal, | | |
| 9164 | .ccc, | | |
| 9165 | .none, | | |
| 9166 | libc_fn.typeOf(&o.builder), | | |
| 9167 | libc_fn.toValue(&o.builder), | | |
| 9168 | &.{operand}, | | |
| 9169 | "", | | |
| 9170 | ); | | |
| 9171 | } | | |
| 9172 | } | | |
| 9173 | | | |
| 9174 | fn airFpext(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9175 | const o = self.ng.object; | | |
| 9176 | const pt = self.ng.pt; | | |
| 9177 | const zcu = pt.zcu; | | |
| 9178 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 9179 | const operand = try self.resolveInst(ty_op.operand); | | |
| 9180 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 9181 | const dest_ty = self.typeOfIndex(inst); | | |
| 9182 | const target = zcu.getTarget(); | | |
| 9183 | | | |
| 9184 | if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) { | | |
| 9185 | return self.wip.cast(.fpext, operand, try o.lowerType(pt, dest_ty), ""); | | |
| 9186 | } else { | | |
| 9187 | const operand_llvm_ty = try o.lowerType(pt, operand_ty); | | |
| 9188 | const dest_llvm_ty = try o.lowerType(pt, dest_ty); | | |
| 9189 | | | |
| 9190 | const dest_bits = dest_ty.scalarType(zcu).floatBits(target); | | |
| 9191 | const src_bits = operand_ty.scalarType(zcu).floatBits(target); | | |
| 9192 | const fn_name = try o.builder.strtabStringFmt("__extend{s}f{s}f2", .{ | | |
| 9193 | compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits), | | |
| 9194 | }); | | |
| 9195 | | | |
| 9196 | const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty); | | |
| 9197 | if (dest_ty.isVector(zcu)) return self.buildElementwiseCall( | | |
| 9198 | libc_fn, | | |
| 9199 | &.{operand}, | | |
| 9200 | try o.builder.poisonValue(dest_llvm_ty), | | |
| 9201 | dest_ty.vectorLen(zcu), | | |
| 9202 | ); | | |
| 9203 | return self.wip.call( | | |
| 9204 | .normal, | | |
| 9205 | .ccc, | | |
| 9206 | .none, | | |
| 9207 | libc_fn.typeOf(&o.builder), | | |
| 9208 | libc_fn.toValue(&o.builder), | | |
| 9209 | &.{operand}, | | |
| 9210 | "", | | |
| 9211 | ); | | |
| 9212 | } | | |
| 9213 | } | | |
| 9214 | | | |
| 9215 | fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9216 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 9217 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 9218 | const inst_ty = self.typeOfIndex(inst); | | |
| 9219 | const operand = try self.resolveInst(ty_op.operand); | | |
| 9220 | return self.bitCast(operand, operand_ty, inst_ty); | | |
| 9221 | } | | |
| 9222 | | | |
| 9223 | fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) !Builder.Value { | | |
| 9224 | const o = self.ng.object; | | |
| 9225 | const pt = self.ng.pt; | | |
| 9226 | const zcu = pt.zcu; | | |
| 9227 | const operand_is_ref = isByRef(operand_ty, zcu); | | |
| 9228 | const result_is_ref = isByRef(inst_ty, zcu); | | |
| 9229 | const llvm_dest_ty = try o.lowerType(pt, inst_ty); | | |
| 9230 | | | |
| 9231 | if (operand_is_ref and result_is_ref) { | | |
| 9232 | // They are both pointers, so just return the same opaque pointer :) | | |
| 9233 | return operand; | | |
| 9234 | } | | |
| 9235 | | | |
| 9236 | if (llvm_dest_ty.isInteger(&o.builder) and | | |
| 9237 | operand.typeOfWip(&self.wip).isInteger(&o.builder)) | | |
| 9238 | { | | |
| 9239 | return self.wip.conv(.unsigned, operand, llvm_dest_ty, ""); | | |
| 9240 | } | | |
| 9241 | | | |
| 9242 | const operand_scalar_ty = operand_ty.scalarType(zcu); | | |
| 9243 | const inst_scalar_ty = inst_ty.scalarType(zcu); | | |
| 9244 | if (operand_scalar_ty.zigTypeTag(zcu) == .int and inst_scalar_ty.isPtrAtRuntime(zcu)) { | | |
| 9245 | return self.wip.cast(.inttoptr, operand, llvm_dest_ty, ""); | | |
| 9246 | } | | |
| 9247 | if (operand_scalar_ty.isPtrAtRuntime(zcu) and inst_scalar_ty.zigTypeTag(zcu) == .int) { | | |
| 9248 | return self.wip.cast(.ptrtoint, operand, llvm_dest_ty, ""); | | |
| 9249 | } | | |
| 9250 | | | |
| 9251 | if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) { | | |
| 9252 | const elem_ty = operand_ty.childType(zcu); | | |
| 9253 | if (!result_is_ref) { | | |
| 9254 | return self.ng.todo("implement bitcast vector to non-ref array", .{}); | | |
| 9255 | } | | |
| 9256 | const alignment = inst_ty.abiAlignment(zcu).toLlvm(); | | |
| 9257 | const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment); | | |
| 9258 | const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8; | | |
| 9259 | if (bitcast_ok) { | | |
| 9260 | _ = try self.wip.store(.normal, operand, array_ptr, alignment); | | |
| 9261 | } else { | | |
| 9262 | // If the ABI size of the element type is not evenly divisible by size in bits; | | |
| 9263 | // a simple bitcast will not work, and we fall back to extractelement. | | |
| 9264 | const llvm_usize = try o.lowerType(pt, Type.usize); | | |
| 9265 | const usize_zero = try o.builder.intValue(llvm_usize, 0); | | |
| 9266 | const vector_len = operand_ty.arrayLen(zcu); | | |
| 9267 | var i: u64 = 0; | | |
| 9268 | while (i < vector_len) : (i += 1) { | | |
| 9269 | const elem_ptr = try self.wip.gep(.inbounds, llvm_dest_ty, array_ptr, &.{ | | |
| 9270 | usize_zero, try o.builder.intValue(llvm_usize, i), | | |
| 9271 | }, ""); | | |
| 9272 | const elem = | | |
| 9273 | try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), ""); | | |
| 9274 | _ = try self.wip.store(.normal, elem, elem_ptr, .default); | | |
| 9275 | } | | |
| 9276 | } | | |
| 9277 | return array_ptr; | | |
| 9278 | } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) { | | |
| 9279 | const elem_ty = operand_ty.childType(zcu); | | |
| 9280 | const llvm_vector_ty = try o.lowerType(pt, inst_ty); | | |
| 9281 | if (!operand_is_ref) return self.ng.todo("implement bitcast non-ref array to vector", .{}); | | |
| 9282 | | | |
| 9283 | const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8; | | |
| 9284 | if (bitcast_ok) { | | |
| 9285 | // The array is aligned to the element's alignment, while the vector might have a completely | | |
| 9286 | // different alignment. This means we need to enforce the alignment of this load. | | |
| 9287 | const alignment = elem_ty.abiAlignment(zcu).toLlvm(); | | |
| 9288 | return self.wip.load(.normal, llvm_vector_ty, operand, alignment, ""); | | |
| 9289 | } else { | | |
| 9290 | // If the ABI size of the element type is not evenly divisible by size in bits; | | |
| 9291 | // a simple bitcast will not work, and we fall back to extractelement. | | |
| 9292 | const array_llvm_ty = try o.lowerType(pt, operand_ty); | | |
| 9293 | const elem_llvm_ty = try o.lowerType(pt, elem_ty); | | |
| 9294 | const llvm_usize = try o.lowerType(pt, Type.usize); | | |
| 9295 | const usize_zero = try o.builder.intValue(llvm_usize, 0); | | |
| 9296 | const vector_len = operand_ty.arrayLen(zcu); | | |
| 9297 | var vector = try o.builder.poisonValue(llvm_vector_ty); | | |
| 9298 | var i: u64 = 0; | | |
| 9299 | while (i < vector_len) : (i += 1) { | | |
| 9300 | const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, operand, &.{ | | |
| 9301 | usize_zero, try o.builder.intValue(llvm_usize, i), | | |
| 9302 | }, ""); | | |
| 9303 | const elem = try self.wip.load(.normal, elem_llvm_ty, elem_ptr, .default, ""); | | |
| 9304 | vector = | | |
| 9305 | try self.wip.insertElement(vector, elem, try o.builder.intValue(.i32, i), ""); | | |
| 9306 | } | | |
| 9307 | return vector; | | |
| 9308 | } | | |
| 9309 | } | | |
| 9310 | | | |
| 9311 | if (operand_is_ref) { | | |
| 9312 | const alignment = operand_ty.abiAlignment(zcu).toLlvm(); | | |
| 9313 | return self.wip.load(.normal, llvm_dest_ty, operand, alignment, ""); | | |
| 9314 | } | | |
| 9315 | | | |
| 9316 | if (result_is_ref) { | | |
| 9317 | const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm(); | | |
| 9318 | const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment); | | |
| 9319 | _ = try self.wip.store(.normal, operand, result_ptr, alignment); | | |
| 9320 | return result_ptr; | | |
| 9321 | } | | |
| 9322 | | | |
| 9323 | if (llvm_dest_ty.isStruct(&o.builder) or | | |
| 9324 | ((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and | | |
| 9325 | operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu))) | | |
| 9326 | { | | |
| 9327 | // Both our operand and our result are values, not pointers, | | |
| 9328 | // but LLVM won't let us bitcast struct values or vectors with padding bits. | | |
| 9329 | // Therefore, we store operand to alloca, then load for result. | | |
| 9330 | const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm(); | | |
| 9331 | const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment); | | |
| 9332 | _ = try self.wip.store(.normal, operand, result_ptr, alignment); | | |
| 9333 | return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, ""); | | |
| 9334 | } | | |
| 9335 | | | |
| 9336 | return self.wip.cast(.bitcast, operand, llvm_dest_ty, ""); | | |
| 9337 | } | | |
| 9338 | | | |
| 9339 | fn airArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9340 | const o = self.ng.object; | | |
| 9341 | const pt = self.ng.pt; | | |
| 9342 | const zcu = pt.zcu; | | |
| 9343 | const arg_val = self.args[self.arg_index]; | | |
| 9344 | self.arg_index += 1; | | |
| 9345 | | | |
| 9346 | // llvm does not support debug info for naked function arguments | | |
| 9347 | if (self.is_naked) return arg_val; | | |
| 9348 | | | |
| 9349 | const inst_ty = self.typeOfIndex(inst); | | |
| 9350 | | | |
| 9351 | const func = zcu.funcInfo(zcu.navValue(self.ng.nav_index).toIntern()); | | |
| 9352 | const func_zir = func.zir_body_inst.resolveFull(&zcu.intern_pool).?; | | |
| 9353 | const file = zcu.fileByIndex(func_zir.file); | | |
| 9354 | | | |
| 9355 | const mod = file.mod.?; | | |
| 9356 | if (mod.strip) return arg_val; | | |
| 9357 | const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg; | | |
| 9358 | const zir = &file.zir.?; | | |
| 9359 | const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?); | | |
| 9360 | | | |
| 9361 | const lbrace_line = zcu.navSrcLine(func.owner_nav) + func.lbrace_line + 1; | | |
| 9362 | const lbrace_col = func.lbrace_column + 1; | | |
| 9363 | | | |
| 9364 | const debug_parameter = try o.builder.debugParameter( | | |
| 9365 | if (name.len > 0) try o.builder.metadataString(name) else null, | | |
| 9366 | self.file, | | |
| 9367 | self.scope, | | |
| 9368 | lbrace_line, | | |
| 9369 | try o.getDebugType(pt, inst_ty), | | |
| 9370 | self.arg_index, | | |
| 9371 | ); | | |
| 9372 | | | |
| 9373 | const old_location = self.wip.debug_location; | | |
| 9374 | self.wip.debug_location = .{ .location = .{ | | |
| 9375 | .line = lbrace_line, | | |
| 9376 | .column = lbrace_col, | | |
| 9377 | .scope = self.scope.toOptional(), | | |
| 9378 | .inlined_at = .none, | | |
| 9379 | } }; | | |
| 9380 | | | |
| 9381 | if (isByRef(inst_ty, zcu)) { | | |
| 9382 | _ = try self.wip.callIntrinsic( | | |
| 9383 | .normal, | | |
| 9384 | .none, | | |
| 9385 | .@"dbg.declare", | | |
| 9386 | &.{}, | | |
| 9387 | &.{ | | |
| 9388 | (try self.wip.debugValue(arg_val)).toValue(), | | |
| 9389 | debug_parameter.toValue(), | | |
| 9390 | (try o.builder.debugExpression(&.{})).toValue(), | | |
| 9391 | }, | | |
| 9392 | "", | | |
| 9393 | ); | | |
| 9394 | } else if (mod.optimize_mode == .Debug) { | | |
| 9395 | const alignment = inst_ty.abiAlignment(zcu).toLlvm(); | | |
| 9396 | const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment); | | |
| 9397 | _ = try self.wip.store(.normal, arg_val, alloca, alignment); | | |
| 9398 | _ = try self.wip.callIntrinsic( | | |
| 9399 | .normal, | | |
| 9400 | .none, | | |
| 9401 | .@"dbg.declare", | | |
| 9402 | &.{}, | | |
| 9403 | &.{ | | |
| 9404 | (try self.wip.debugValue(alloca)).toValue(), | | |
| 9405 | debug_parameter.toValue(), | | |
| 9406 | (try o.builder.debugExpression(&.{})).toValue(), | | |
| 9407 | }, | | |
| 9408 | "", | | |
| 9409 | ); | | |
| 9410 | } else { | | |
| 9411 | _ = try self.wip.callIntrinsic( | | |
| 9412 | .normal, | | |
| 9413 | .none, | | |
| 9414 | .@"dbg.value", | | |
| 9415 | &.{}, | | |
| 9416 | &.{ | | |
| 9417 | (try self.wip.debugValue(arg_val)).toValue(), | | |
| 9418 | debug_parameter.toValue(), | | |
| 9419 | (try o.builder.debugExpression(&.{})).toValue(), | | |
| 9420 | }, | | |
| 9421 | "", | | |
| 9422 | ); | | |
| 9423 | } | | |
| 9424 | | | |
| 9425 | self.wip.debug_location = old_location; | | |
| 9426 | return arg_val; | | |
| 9427 | } | | |
| 9428 | | | |
| 9429 | fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9430 | const o = self.ng.object; | | |
| 9431 | const pt = self.ng.pt; | | |
| 9432 | const zcu = pt.zcu; | | |
| 9433 | const ptr_ty = self.typeOfIndex(inst); | | |
| 9434 | const pointee_type = ptr_ty.childType(zcu); | | |
| 9435 | if (!pointee_type.hasRuntimeBits(zcu)) | | |
| 9436 | return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue(); | | |
| 9437 | | | |
| 9438 | const pointee_llvm_ty = try o.lowerType(pt, pointee_type); | | |
| 9439 | const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); | | |
| 9440 | return self.buildAlloca(pointee_llvm_ty, alignment); | | |
| 9441 | } | | |
| 9442 | | | |
| 9443 | fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9444 | const o = self.ng.object; | | |
| 9445 | const pt = self.ng.pt; | | |
| 9446 | const zcu = pt.zcu; | | |
| 9447 | const ptr_ty = self.typeOfIndex(inst); | | |
| 9448 | const ret_ty = ptr_ty.childType(zcu); | | |
| 9449 | if (!ret_ty.hasRuntimeBits(zcu)) | | |
| 9450 | return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue(); | | |
| 9451 | if (self.ret_ptr != .none) return self.ret_ptr; | | |
| 9452 | const ret_llvm_ty = try o.lowerType(pt, ret_ty); | | |
| 9453 | const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); | | |
| 9454 | return self.buildAlloca(ret_llvm_ty, alignment); | | |
| 9455 | } | | |
| 9456 | | | |
| 9457 | /// Use this instead of builder.buildAlloca, because this function makes sure to | | |
| 9458 | /// put the alloca instruction at the top of the function! | | |
| 9459 | fn buildAlloca( | | |
| 9460 | self: *FuncGen, | | |
| 9461 | llvm_ty: Builder.Type, | | |
| 9462 | alignment: Builder.Alignment, | | |
| 9463 | ) Allocator.Error!Builder.Value { | | |
| 9464 | const target = self.ng.pt.zcu.getTarget(); | | |
| 9465 | return buildAllocaInner(&self.wip, llvm_ty, alignment, target); | | |
| 9466 | } | | |
| 9467 | | | |
| 9468 | fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { | | |
| 9469 | const o = self.ng.object; | | |
| 9470 | const pt = self.ng.pt; | | |
| 9471 | const zcu = pt.zcu; | | |
| 9472 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 9473 | const dest_ptr = try self.resolveInst(bin_op.lhs); | | |
| 9474 | const ptr_ty = self.typeOf(bin_op.lhs); | | |
| 9475 | const operand_ty = ptr_ty.childType(zcu); | | |
| 9476 | | | |
| 9477 | const val_is_undef = if (try self.air.value(bin_op.rhs, pt)) |val| val.isUndef(zcu) else false; | | |
| 9478 | if (val_is_undef) { | | |
| 9479 | const owner_mod = self.ng.ownerModule(); | | |
| 9480 | | | |
| 9481 | // Even if safety is disabled, we still emit a memset to undefined since it conveys | | |
| 9482 | // extra information to LLVM, and LLVM will optimize it out. Safety makes the difference | | |
| 9483 | // between using 0xaa or actual undefined for the fill byte. | | |
| 9484 | // | | |
| 9485 | // However, for Debug builds specifically, we avoid emitting the memset because LLVM | | |
| 9486 | // will neither use the information nor get rid of the memset, thus leaving an | | |
| 9487 | // unexpected call in the user's code. This is problematic if the code in question is | | |
| 9488 | // not ready to correctly make calls yet, such as in our early PIE startup code, or in | | |
| 9489 | // the early stages of a dynamic linker, etc. | | |
| 9490 | if (!safety and owner_mod.optimize_mode == .Debug) { | | |
| 9491 | return .none; | | |
| 9492 | } | | |
| 9493 | | | |
| 9494 | const ptr_info = ptr_ty.ptrInfo(zcu); | | |
| 9495 | const needs_bitmask = (ptr_info.packed_offset.host_size != 0); | | |
| 9496 | if (needs_bitmask) { | | |
| 9497 | // TODO: only some bits are to be undef, we cannot write with a simple memset. | | |
| 9498 | // meanwhile, ignore the write rather than stomping over valid bits. | | |
| 9499 | // https://github.com/ziglang/zig/issues/15337 | | |
| 9500 | return .none; | | |
| 9501 | } | | |
| 9502 | | | |
| 9503 | self.maybeMarkAllowZeroAccess(ptr_info); | | |
| 9504 | | | |
| 9505 | const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), operand_ty.abiSize(zcu)); | | |
| 9506 | _ = try self.wip.callMemSet( | | |
| 9507 | dest_ptr, | | |
| 9508 | ptr_ty.ptrAlignment(zcu).toLlvm(), | | |
| 9509 | if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8), | | |
| 9510 | len, | | |
| 9511 | if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, | | |
| 9512 | self.disable_intrinsics, | | |
| 9513 | ); | | |
| 9514 | if (safety and owner_mod.valgrind) { | | |
| 9515 | try self.valgrindMarkUndef(dest_ptr, len); | | |
| 9516 | } | | |
| 9517 | return .none; | | |
| 9518 | } | | |
| 9519 | | | |
| 9520 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); | | |
| 9521 | | | |
| 9522 | const src_operand = try self.resolveInst(bin_op.rhs); | | |
| 9523 | try self.store(dest_ptr, ptr_ty, src_operand, .none); | | |
| 9524 | return .none; | | |
| 9525 | } | | |
| 9526 | | | |
| 9527 | fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9528 | const pt = fg.ng.pt; | | |
| 9529 | const zcu = pt.zcu; | | |
| 9530 | const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 9531 | const ptr_ty = fg.typeOf(ty_op.operand); | | |
| 9532 | const ptr_info = ptr_ty.ptrInfo(zcu); | | |
| 9533 | const ptr = try fg.resolveInst(ty_op.operand); | | |
| 9534 | fg.maybeMarkAllowZeroAccess(ptr_info); | | |
| 9535 | return fg.load(ptr, ptr_ty); | | |
| 9536 | } | | |
| 9537 | | | |
| 9538 | fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !void { | | |
| 9539 | _ = inst; | | |
| 9540 | const target = self.ng.object.target; | | |
| 9541 | if ((target.cpu.arch == .mips or target.cpu.arch == .mipsel) and | | |
| 9542 | target.cpu.has(.mips, .notraps)) | | |
| 9543 | { | | |
| 9544 | // Emit a MIPS `break` instruction followed by an infinite loop (to fulfill the noreturn) | | |
| 9545 | // since this CPU does not support trap instructions. | | |
| 9546 | const o = self.ng.object; | | |
| 9547 | _ = try self.wip.callAsm( | | |
| 9548 | .none, | | |
| 9549 | try o.builder.fnType(.void, &.{}, .normal), | | |
| 9550 | .{ .sideeffect = true }, | | |
| 9551 | try o.builder.string("break\n0:\nj 0b\nnop"), | | |
| 9552 | try o.builder.string("~{memory}"), | | |
| 9553 | &.{}, | | |
| 9554 | "", | | |
| 9555 | ); | | |
| 9556 | } else { | | |
| 9557 | _ = try self.wip.callIntrinsic(.normal, .none, .trap, &.{}, &.{}, ""); | | |
| 9558 | } | | |
| 9559 | _ = try self.wip.@"unreachable"(); | | |
| 9560 | } | | |
| 9561 | | | |
| 9562 | fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9563 | _ = inst; | | |
| 9564 | _ = try self.wip.callIntrinsic(.normal, .none, .debugtrap, &.{}, &.{}, ""); | | |
| 9565 | return .none; | | |
| 9566 | } | | |
| 9567 | | | |
| 9568 | fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9569 | _ = inst; | | |
| 9570 | const o = self.ng.object; | | |
| 9571 | const pt = self.ng.pt; | | |
| 9572 | const llvm_usize = try o.lowerType(pt, Type.usize); | | |
| 9573 | if (!target_util.supportsReturnAddress(self.ng.pt.zcu.getTarget(), self.ng.ownerModule().optimize_mode)) { | | |
| 9574 | // https://github.com/ziglang/zig/issues/11946 | | |
| 9575 | return o.builder.intValue(llvm_usize, 0); | | |
| 9576 | } | | |
| 9577 | const result = try self.wip.callIntrinsic(.normal, .none, .returnaddress, &.{}, &.{.@"0"}, ""); | | |
| 9578 | return self.wip.cast(.ptrtoint, result, llvm_usize, ""); | | |
| 9579 | } | | |
| 9580 | | | |
| 9581 | fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9582 | _ = inst; | | |
| 9583 | const o = self.ng.object; | | |
| 9584 | const pt = self.ng.pt; | | |
| 9585 | const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, ""); | | |
| 9586 | return self.wip.cast(.ptrtoint, result, try o.lowerType(pt, Type.usize), ""); | | |
| 9587 | } | | |
| 9588 | | | |
| 9589 | fn airCmpxchg( | | |
| 9590 | self: *FuncGen, | | |
| 9591 | inst: Air.Inst.Index, | | |
| 9592 | kind: Builder.Function.Instruction.CmpXchg.Kind, | | |
| 9593 | ) !Builder.Value { | | |
| 9594 | const o = self.ng.object; | | |
| 9595 | const pt = self.ng.pt; | | |
| 9596 | const zcu = pt.zcu; | | |
| 9597 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 9598 | const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data; | | |
| 9599 | const ptr = try self.resolveInst(extra.ptr); | | |
| 9600 | const ptr_ty = self.typeOf(extra.ptr); | | |
| 9601 | var expected_value = try self.resolveInst(extra.expected_value); | | |
| 9602 | var new_value = try self.resolveInst(extra.new_value); | | |
| 9603 | const operand_ty = ptr_ty.childType(zcu); | | |
| 9604 | const llvm_operand_ty = try o.lowerType(pt, operand_ty); | | |
| 9605 | const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, false); | | |
| 9606 | if (llvm_abi_ty != .none) { | | |
| 9607 | // operand needs widening and truncating | | |
| 9608 | const signedness: Builder.Function.Instruction.Cast.Signedness = | | |
| 9609 | if (operand_ty.isSignedInt(zcu)) .signed else .unsigned; | | |
| 9610 | expected_value = try self.wip.conv(signedness, expected_value, llvm_abi_ty, ""); | | |
| 9611 | new_value = try self.wip.conv(signedness, new_value, llvm_abi_ty, ""); | | |
| 9612 | } | | |
| 9613 | | | |
| 9614 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); | | |
| 9615 | | | |
| 9616 | const result = try self.wip.cmpxchg( | | |
| 9617 | kind, | | |
| 9618 | if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, | | |
| 9619 | ptr, | | |
| 9620 | expected_value, | | |
| 9621 | new_value, | | |
| 9622 | self.sync_scope, | | |
| 9623 | toLlvmAtomicOrdering(extra.successOrder()), | | |
| 9624 | toLlvmAtomicOrdering(extra.failureOrder()), | | |
| 9625 | ptr_ty.ptrAlignment(zcu).toLlvm(), | | |
| 9626 | "", | | |
| 9627 | ); | | |
| 9628 | | | |
| 9629 | const optional_ty = self.typeOfIndex(inst); | | |
| 9630 | | | |
| 9631 | var payload = try self.wip.extractValue(result, &.{0}, ""); | | |
| 9632 | if (llvm_abi_ty != .none) payload = try self.wip.cast(.trunc, payload, llvm_operand_ty, ""); | | |
| 9633 | const success_bit = try self.wip.extractValue(result, &.{1}, ""); | | |
| 9634 | | | |
| 9635 | if (optional_ty.optionalReprIsPayload(zcu)) { | | |
| 9636 | const zero = try o.builder.zeroInitValue(payload.typeOfWip(&self.wip)); | | |
| 9637 | return self.wip.select(.normal, success_bit, zero, payload, ""); | | |
| 9638 | } | | |
| 9639 | | | |
| 9640 | comptime assert(optional_layout_version == 3); | | |
| 9641 | | | |
| 9642 | const non_null_bit = try self.wip.not(success_bit, ""); | | |
| 9643 | return buildOptional(self, optional_ty, payload, non_null_bit); | | |
| 9644 | } | | |
| 9645 | | | |
| 9646 | fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9647 | const o = self.ng.object; | | |
| 9648 | const pt = self.ng.pt; | | |
| 9649 | const zcu = pt.zcu; | | |
| 9650 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | | |
| 9651 | const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; | | |
| 9652 | const ptr = try self.resolveInst(pl_op.operand); | | |
| 9653 | const ptr_ty = self.typeOf(pl_op.operand); | | |
| 9654 | const operand_ty = ptr_ty.childType(zcu); | | |
| 9655 | const operand = try self.resolveInst(extra.operand); | | |
| 9656 | const is_signed_int = operand_ty.isSignedInt(zcu); | | |
| 9657 | const is_float = operand_ty.isRuntimeFloat(); | | |
| 9658 | const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float); | | |
| 9659 | const ordering = toLlvmAtomicOrdering(extra.ordering()); | | |
| 9660 | const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, op == .xchg); | | |
| 9661 | const llvm_operand_ty = try o.lowerType(pt, operand_ty); | | |
| 9662 | | | |
| 9663 | const access_kind: Builder.MemoryAccessKind = | | |
| 9664 | if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | | |
| 9665 | const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); | | |
| 9666 | | | |
| 9667 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); | | |
| 9668 | | | |
| 9669 | if (llvm_abi_ty != .none) { | | |
| 9670 | // operand needs widening and truncating or bitcasting. | | |
| 9671 | return self.wip.cast(if (is_float) .bitcast else .trunc, try self.wip.atomicrmw( | | |
| 9672 | access_kind, | | |
| 9673 | op, | | |
| 9674 | ptr, | | |
| 9675 | try self.wip.cast( | | |
| 9676 | if (is_float) .bitcast else if (is_signed_int) .sext else .zext, | | |
| 9677 | operand, | | |
| 9678 | llvm_abi_ty, | | |
| 9679 | "", | | |
| 9680 | ), | | |
| 9681 | self.sync_scope, | | |
| 9682 | ordering, | | |
| 9683 | ptr_alignment, | | |
| 9684 | "", | | |
| 9685 | ), llvm_operand_ty, ""); | | |
| 9686 | } | | |
| 9687 | | | |
| 9688 | if (!llvm_operand_ty.isPointer(&o.builder)) return self.wip.atomicrmw( | | |
| 9689 | access_kind, | | |
| 9690 | op, | | |
| 9691 | ptr, | | |
| 9692 | operand, | | |
| 9693 | self.sync_scope, | | |
| 9694 | ordering, | | |
| 9695 | ptr_alignment, | | |
| 9696 | "", | | |
| 9697 | ); | | |
| 9698 | | | |
| 9699 | // It's a pointer but we need to treat it as an int. | | |
| 9700 | return self.wip.cast(.inttoptr, try self.wip.atomicrmw( | | |
| 9701 | access_kind, | | |
| 9702 | op, | | |
| 9703 | ptr, | | |
| 9704 | try self.wip.cast(.ptrtoint, operand, try o.lowerType(pt, Type.usize), ""), | | |
| 9705 | self.sync_scope, | | |
| 9706 | ordering, | | |
| 9707 | ptr_alignment, | | |
| 9708 | "", | | |
| 9709 | ), llvm_operand_ty, ""); | | |
| 9710 | } | | |
| 9711 | | | |
| 9712 | fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9713 | const o = self.ng.object; | | |
| 9714 | const pt = self.ng.pt; | | |
| 9715 | const zcu = pt.zcu; | | |
| 9716 | const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; | | |
| 9717 | const ptr = try self.resolveInst(atomic_load.ptr); | | |
| 9718 | const ptr_ty = self.typeOf(atomic_load.ptr); | | |
| 9719 | const info = ptr_ty.ptrInfo(zcu); | | |
| 9720 | const elem_ty = Type.fromInterned(info.child); | | |
| 9721 | if (!elem_ty.hasRuntimeBits(zcu)) return .none; | | |
| 9722 | const ordering = toLlvmAtomicOrdering(atomic_load.order); | | |
| 9723 | const llvm_abi_ty = try o.getAtomicAbiType(pt, elem_ty, false); | | |
| 9724 | const ptr_alignment = (if (info.flags.alignment != .none) | | |
| 9725 | @as(InternPool.Alignment, info.flags.alignment) | | |
| 9726 | else | | |
| 9727 | Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm(); | | |
| 9728 | const access_kind: Builder.MemoryAccessKind = | | |
| 9729 | if (info.flags.is_volatile) .@"volatile" else .normal; | | |
| 9730 | const elem_llvm_ty = try o.lowerType(pt, elem_ty); | | |
| 9731 | | | |
| 9732 | self.maybeMarkAllowZeroAccess(info); | | |
| 9733 | | | |
| 9734 | if (llvm_abi_ty != .none) { | | |
| 9735 | // operand needs widening and truncating | | |
| 9736 | const loaded = try self.wip.loadAtomic( | | |
| 9737 | access_kind, | | |
| 9738 | llvm_abi_ty, | | |
| 9739 | ptr, | | |
| 9740 | self.sync_scope, | | |
| 9741 | ordering, | | |
| 9742 | ptr_alignment, | | |
| 9743 | "", | | |
| 9744 | ); | | |
| 9745 | return self.wip.cast(.trunc, loaded, elem_llvm_ty, ""); | | |
| 9746 | } | | |
| 9747 | return self.wip.loadAtomic( | | |
| 9748 | access_kind, | | |
| 9749 | elem_llvm_ty, | | |
| 9750 | ptr, | | |
| 9751 | self.sync_scope, | | |
| 9752 | ordering, | | |
| 9753 | ptr_alignment, | | |
| 9754 | "", | | |
| 9755 | ); | | |
| 9756 | } | | |
| 9757 | | | |
| 9758 | fn airAtomicStore( | | |
| 9759 | self: *FuncGen, | | |
| 9760 | inst: Air.Inst.Index, | | |
| 9761 | ordering: Builder.AtomicOrdering, | | |
| 9762 | ) !Builder.Value { | | |
| 9763 | const o = self.ng.object; | | |
| 9764 | const pt = self.ng.pt; | | |
| 9765 | const zcu = pt.zcu; | | |
| 9766 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 9767 | const ptr_ty = self.typeOf(bin_op.lhs); | | |
| 9768 | const operand_ty = ptr_ty.childType(zcu); | | |
| 9769 | if (!operand_ty.hasRuntimeBits(zcu)) return .none; | | |
| 9770 | const ptr = try self.resolveInst(bin_op.lhs); | | |
| 9771 | var element = try self.resolveInst(bin_op.rhs); | | |
| 9772 | const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, false); | | |
| 9773 | | | |
| 9774 | if (llvm_abi_ty != .none) { | | |
| 9775 | // operand needs widening | | |
| 9776 | element = try self.wip.conv( | | |
| 9777 | if (operand_ty.isSignedInt(zcu)) .signed else .unsigned, | | |
| 9778 | element, | | |
| 9779 | llvm_abi_ty, | | |
| 9780 | "", | | |
| 9781 | ); | | |
| 9782 | } | | |
| 9783 | | | |
| 9784 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); | | |
| 9785 | | | |
| 9786 | try self.store(ptr, ptr_ty, element, ordering); | | |
| 9787 | return .none; | | |
| 9788 | } | | |
| 9789 | | | |
| 9790 | fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { | | |
| 9791 | const o = self.ng.object; | | |
| 9792 | const pt = self.ng.pt; | | |
| 9793 | const zcu = pt.zcu; | | |
| 9794 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 9795 | const dest_slice = try self.resolveInst(bin_op.lhs); | | |
| 9796 | const ptr_ty = self.typeOf(bin_op.lhs); | | |
| 9797 | const elem_ty = self.typeOf(bin_op.rhs); | | |
| 9798 | const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm(); | | |
| 9799 | const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty); | | |
| 9800 | const access_kind: Builder.MemoryAccessKind = | | |
| 9801 | if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | | |
| 9802 | | | |
| 9803 | self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); | | |
| 9804 | | | |
| 9805 | if (try self.air.value(bin_op.rhs, pt)) |elem_val| { | | |
| 9806 | if (elem_val.isUndef(zcu)) { | | |
| 9807 | // Even if safety is disabled, we still emit a memset to undefined since it conveys | | |
| 9808 | // extra information to LLVM. However, safety makes the difference between using | | |
| 9809 | // 0xaa or actual undefined for the fill byte. | | |
| 9810 | const fill_byte = if (safety) | | |
| 9811 | try o.builder.intValue(.i8, 0xaa) | | |
| 9812 | else | | |
| 9813 | try o.builder.undefValue(.i8); | | |
| 9814 | const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); | | |
| 9815 | _ = try self.wip.callMemSet( | | |
| 9816 | dest_ptr, | | |
| 9817 | dest_ptr_align, | | |
| 9818 | fill_byte, | | |
| 9819 | len, | | |
| 9820 | access_kind, | | |
| 9821 | self.disable_intrinsics, | | |
| 9822 | ); | | |
| 9823 | const owner_mod = self.ng.ownerModule(); | | |
| 9824 | if (safety and owner_mod.valgrind) { | | |
| 9825 | try self.valgrindMarkUndef(dest_ptr, len); | | |
| 9826 | } | | |
| 9827 | return .none; | | |
| 9828 | } | | |
| 9829 | | | |
| 9830 | // Test if the element value is compile-time known to be a | | |
| 9831 | // repeating byte pattern, for example, `@as(u64, 0)` has a | | |
| 9832 | // repeating byte pattern of 0 bytes. In such case, the memset | | |
| 9833 | // intrinsic can be used. | | |
| 9834 | if (try elem_val.hasRepeatedByteRepr(pt)) |byte_val| { | | |
| 9835 | const fill_byte = try o.builder.intValue(.i8, byte_val); | | |
| 9836 | const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); | | |
| 9837 | _ = try self.wip.callMemSet( | | |
| 9838 | dest_ptr, | | |
| 9839 | dest_ptr_align, | | |
| 9840 | fill_byte, | | |
| 9841 | len, | | |
| 9842 | access_kind, | | |
| 9843 | self.disable_intrinsics, | | |
| 9844 | ); | | |
| 9845 | return .none; | | |
| 9846 | } | | |
| 9847 | } | | |
| 9848 | | | |
| 9849 | const value = try self.resolveInst(bin_op.rhs); | | |
| 9850 | const elem_abi_size = elem_ty.abiSize(zcu); | | |
| 9851 | | | |
| 9852 | if (elem_abi_size == 1) { | | |
| 9853 | // In this case we can take advantage of LLVM's intrinsic. | | |
| 9854 | const fill_byte = try self.bitCast(value, elem_ty, Type.u8); | | |
| 9855 | const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); | | |
| 9856 | | | |
| 9857 | _ = try self.wip.callMemSet( | | |
| 9858 | dest_ptr, | | |
| 9859 | dest_ptr_align, | | |
| 9860 | fill_byte, | | |
| 9861 | len, | | |
| 9862 | access_kind, | | |
| 9863 | self.disable_intrinsics, | | |
| 9864 | ); | | |
| 9865 | return .none; | | |
| 9866 | } | | |
| 9867 | | | |
| 9868 | // non-byte-sized element. lower with a loop. something like this: | | |
| 9869 | | | |
| 9870 | // entry: | | |
| 9871 | // ... | | |
| 9872 | // %end_ptr = getelementptr %ptr, %len | | |
| 9873 | // br %loop | | |
| 9874 | // loop: | | |
| 9875 | // %it_ptr = phi body %next_ptr, entry %ptr | | |
| 9876 | // %end = cmp eq %it_ptr, %end_ptr | | |
| 9877 | // br %end, %body, %end | | |
| 9878 | // body: | | |
| 9879 | // store %it_ptr, %value | | |
| 9880 | // %next_ptr = getelementptr %it_ptr, 1 | | |
| 9881 | // br %loop | | |
| 9882 | // end: | | |
| 9883 | // ... | | |
| 9884 | const entry_block = self.wip.cursor.block; | | |
| 9885 | const loop_block = try self.wip.block(2, "InlineMemsetLoop"); | | |
| 9886 | const body_block = try self.wip.block(1, "InlineMemsetBody"); | | |
| 9887 | const end_block = try self.wip.block(1, "InlineMemsetEnd"); | | |
| 9888 | | | |
| 9889 | const llvm_usize_ty = try o.lowerType(pt, Type.usize); | | |
| 9890 | const len = switch (ptr_ty.ptrSize(zcu)) { | | |
| 9891 | .slice => try self.wip.extractValue(dest_slice, &.{1}, ""), | | |
| 9892 | .one => try o.builder.intValue(llvm_usize_ty, ptr_ty.childType(zcu).arrayLen(zcu)), | | |
| 9893 | .many, .c => unreachable, | | |
| 9894 | }; | | |
| 9895 | const elem_llvm_ty = try o.lowerType(pt, elem_ty); | | |
| 9896 | const end_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, dest_ptr, &.{len}, ""); | | |
| 9897 | _ = try self.wip.br(loop_block); | | |
| 9898 | | | |
| 9899 | self.wip.cursor = .{ .block = loop_block }; | | |
| 9900 | const it_ptr = try self.wip.phi(.ptr, ""); | | |
| 9901 | const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, ""); | | |
| 9902 | _ = try self.wip.brCond(end, body_block, end_block, .none); | | |
| 9903 | | | |
| 9904 | self.wip.cursor = .{ .block = body_block }; | | |
| 9905 | const elem_abi_align = elem_ty.abiAlignment(zcu); | | |
| 9906 | const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm(); | | |
| 9907 | if (isByRef(elem_ty, zcu)) { | | |
| 9908 | _ = try self.wip.callMemCpy( | | |
| 9909 | it_ptr.toValue(), | | |
| 9910 | it_ptr_align, | | |
| 9911 | value, | | |
| 9912 | elem_abi_align.toLlvm(), | | |
| 9913 | try o.builder.intValue(llvm_usize_ty, elem_abi_size), | | |
| 9914 | access_kind, | | |
| 9915 | self.disable_intrinsics, | | |
| 9916 | ); | | |
| 9917 | } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align); | | |
| 9918 | const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{ | | |
| 9919 | try o.builder.intValue(llvm_usize_ty, 1), | | |
| 9920 | }, ""); | | |
| 9921 | _ = try self.wip.br(loop_block); | | |
| 9922 | | | |
| 9923 | self.wip.cursor = .{ .block = end_block }; | | |
| 9924 | it_ptr.finish(&.{ next_ptr, dest_ptr }, &.{ body_block, entry_block }, &self.wip); | | |
| 9925 | return .none; | | |
| 9926 | } | | |
| 9927 | | | |
| 9928 | fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9929 | const pt = self.ng.pt; | | |
| 9930 | const zcu = pt.zcu; | | |
| 9931 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 9932 | const dest_slice = try self.resolveInst(bin_op.lhs); | | |
| 9933 | const dest_ptr_ty = self.typeOf(bin_op.lhs); | | |
| 9934 | const src_slice = try self.resolveInst(bin_op.rhs); | | |
| 9935 | const src_ptr_ty = self.typeOf(bin_op.rhs); | | |
| 9936 | const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty); | | |
| 9937 | const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty); | | |
| 9938 | const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty); | | |
| 9939 | const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or | | |
| 9940 | dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | | |
| 9941 | | | |
| 9942 | self.maybeMarkAllowZeroAccess(dest_ptr_ty.ptrInfo(zcu)); | | |
| 9943 | self.maybeMarkAllowZeroAccess(src_ptr_ty.ptrInfo(zcu)); | | |
| 9944 | | | |
| 9945 | _ = try self.wip.callMemCpy( | | |
| 9946 | dest_ptr, | | |
| 9947 | dest_ptr_ty.ptrAlignment(zcu).toLlvm(), | | |
| 9948 | src_ptr, | | |
| 9949 | src_ptr_ty.ptrAlignment(zcu).toLlvm(), | | |
| 9950 | len, | | |
| 9951 | access_kind, | | |
| 9952 | self.disable_intrinsics, | | |
| 9953 | ); | | |
| 9954 | return .none; | | |
| 9955 | } | | |
| 9956 | | | |
| 9957 | fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9958 | const pt = self.ng.pt; | | |
| 9959 | const zcu = pt.zcu; | | |
| 9960 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 9961 | const dest_slice = try self.resolveInst(bin_op.lhs); | | |
| 9962 | const dest_ptr_ty = self.typeOf(bin_op.lhs); | | |
| 9963 | const src_slice = try self.resolveInst(bin_op.rhs); | | |
| 9964 | const src_ptr_ty = self.typeOf(bin_op.rhs); | | |
| 9965 | const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty); | | |
| 9966 | const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty); | | |
| 9967 | const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty); | | |
| 9968 | const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or | | |
| 9969 | dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | | |
| 9970 | | | |
| 9971 | _ = try self.wip.callMemMove( | | |
| 9972 | dest_ptr, | | |
| 9973 | dest_ptr_ty.ptrAlignment(zcu).toLlvm(), | | |
| 9974 | src_ptr, | | |
| 9975 | src_ptr_ty.ptrAlignment(zcu).toLlvm(), | | |
| 9976 | len, | | |
| 9977 | access_kind, | | |
| 9978 | ); | | |
| 9979 | return .none; | | |
| 9980 | } | | |
| 9981 | | | |
| 9982 | fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 9983 | const o = self.ng.object; | | |
| 9984 | const pt = self.ng.pt; | | |
| 9985 | const zcu = pt.zcu; | | |
| 9986 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 9987 | const un_ptr_ty = self.typeOf(bin_op.lhs); | | |
| 9988 | const un_ty = un_ptr_ty.childType(zcu); | | |
| 9989 | const layout = un_ty.unionGetLayout(zcu); | | |
| 9990 | if (layout.tag_size == 0) return .none; | | |
| 9991 | | | |
| 9992 | const access_kind: Builder.MemoryAccessKind = | | |
| 9993 | if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | | |
| 9994 | | | |
| 9995 | self.maybeMarkAllowZeroAccess(un_ptr_ty.ptrInfo(zcu)); | | |
| 9996 | | | |
| 9997 | const union_ptr = try self.resolveInst(bin_op.lhs); | | |
| 9998 | const new_tag = try self.resolveInst(bin_op.rhs); | | |
| 9999 | const union_ptr_align = un_ptr_ty.ptrAlignment(zcu); | | |
| 10000 | if (layout.payload_size == 0) { | | |
| 10001 | _ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm()); | | |
| 10002 | return .none; | | |
| 10003 | } | | |
| 10004 | const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); | | |
| 10005 | const tag_field_ptr = try self.wip.gepStruct(try o.lowerType(pt, un_ty), union_ptr, tag_index, ""); | | |
| 10006 | const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) { | | |
| 10007 | 0 => union_ptr_align, | | |
| 10008 | else => |off| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))), | | |
| 10009 | }; | | |
| 10010 | _ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm()); | | |
| 10011 | return .none; | | |
| 10012 | } | | |
| 10013 | | | |
| 10014 | fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10015 | const o = self.ng.object; | | |
| 10016 | const pt = self.ng.pt; | | |
| 10017 | const zcu = pt.zcu; | | |
| 10018 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 10019 | const un_ty = self.typeOf(ty_op.operand); | | |
| 10020 | const layout = un_ty.unionGetLayout(zcu); | | |
| 10021 | if (layout.tag_size == 0) return .none; | | |
| 10022 | const union_handle = try self.resolveInst(ty_op.operand); | | |
| 10023 | if (isByRef(un_ty, zcu)) { | | |
| 10024 | const llvm_un_ty = try o.lowerType(pt, un_ty); | | |
| 10025 | if (layout.payload_size == 0) | | |
| 10026 | return self.wip.load(.normal, llvm_un_ty, union_handle, .default, ""); | | |
| 10027 | const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); | | |
| 10028 | const tag_field_ptr = try self.wip.gepStruct(llvm_un_ty, union_handle, tag_index, ""); | | |
| 10029 | const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index]; | | |
| 10030 | return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, ""); | | |
| 10031 | } else { | | |
| 10032 | if (layout.payload_size == 0) return union_handle; | | |
| 10033 | const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); | | |
| 10034 | return self.wip.extractValue(union_handle, &.{tag_index}, ""); | | |
| 10035 | } | | |
| 10036 | } | | |
| 10037 | | | |
| 10038 | fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) !Builder.Value { | | |
| 10039 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 10040 | const operand = try self.resolveInst(un_op); | | |
| 10041 | const operand_ty = self.typeOf(un_op); | | |
| 10042 | | | |
| 10043 | return self.buildFloatOp(op, .normal, operand_ty, 1, .{operand}); | | |
| 10044 | } | | |
| 10045 | | | |
| 10046 | fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 10047 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 10048 | const operand = try self.resolveInst(un_op); | | |
| 10049 | const operand_ty = self.typeOf(un_op); | | |
| 10050 | | | |
| 10051 | return self.buildFloatOp(.neg, fast, operand_ty, 1, .{operand}); | | |
| 10052 | } | | |
| 10053 | | | |
| 10054 | fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value { | | |
| 10055 | const o = self.ng.object; | | |
| 10056 | const pt = self.ng.pt; | | |
| 10057 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 10058 | const inst_ty = self.typeOfIndex(inst); | | |
| 10059 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 10060 | const operand = try self.resolveInst(ty_op.operand); | | |
| 10061 | | | |
| 10062 | const result = try self.wip.callIntrinsic( | | |
| 10063 | .normal, | | |
| 10064 | .none, | | |
| 10065 | intrinsic, | | |
| 10066 | &.{try o.lowerType(pt, operand_ty)}, | | |
| 10067 | &.{ operand, .false }, | | |
| 10068 | "", | | |
| 10069 | ); | | |
| 10070 | return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); | | |
| 10071 | } | | |
| 10072 | | | |
| 10073 | fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value { | | |
| 10074 | const o = self.ng.object; | | |
| 10075 | const pt = self.ng.pt; | | |
| 10076 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 10077 | const inst_ty = self.typeOfIndex(inst); | | |
| 10078 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 10079 | const operand = try self.resolveInst(ty_op.operand); | | |
| 10080 | | | |
| 10081 | const result = try self.wip.callIntrinsic( | | |
| 10082 | .normal, | | |
| 10083 | .none, | | |
| 10084 | intrinsic, | | |
| 10085 | &.{try o.lowerType(pt, operand_ty)}, | | |
| 10086 | &.{operand}, | | |
| 10087 | "", | | |
| 10088 | ); | | |
| 10089 | return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); | | |
| 10090 | } | | |
| 10091 | | | |
| 10092 | fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10093 | const o = self.ng.object; | | |
| 10094 | const pt = self.ng.pt; | | |
| 10095 | const zcu = pt.zcu; | | |
| 10096 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 10097 | const operand_ty = self.typeOf(ty_op.operand); | | |
| 10098 | var bits = operand_ty.intInfo(zcu).bits; | | |
| 10099 | assert(bits % 8 == 0); | | |
| 10100 | | | |
| 10101 | const inst_ty = self.typeOfIndex(inst); | | |
| 10102 | var operand = try self.resolveInst(ty_op.operand); | | |
| 10103 | var llvm_operand_ty = try o.lowerType(pt, operand_ty); | | |
| 10104 | | | |
| 10105 | if (bits % 16 == 8) { | | |
| 10106 | // If not an even byte-multiple, we need zero-extend + shift-left 1 byte | | |
| 10107 | // The truncated result at the end will be the correct bswap | | |
| 10108 | const scalar_ty = try o.builder.intType(@intCast(bits + 8)); | | |
| 10109 | if (operand_ty.zigTypeTag(zcu) == .vector) { | | |
| 10110 | const vec_len = operand_ty.vectorLen(zcu); | | |
| 10111 | llvm_operand_ty = try o.builder.vectorType(.normal, vec_len, scalar_ty); | | |
| 10112 | } else llvm_operand_ty = scalar_ty; | | |
| 10113 | | | |
| 10114 | const shift_amt = | | |
| 10115 | try o.builder.splatValue(llvm_operand_ty, try o.builder.intConst(scalar_ty, 8)); | | |
| 10116 | const extended = try self.wip.cast(.zext, operand, llvm_operand_ty, ""); | | |
| 10117 | operand = try self.wip.bin(.shl, extended, shift_amt, ""); | | |
| 10118 | | | |
| 10119 | bits = bits + 8; | | |
| 10120 | } | | |
| 10121 | | | |
| 10122 | const result = | | |
| 10123 | try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, ""); | | |
| 10124 | return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); | | |
| 10125 | } | | |
| 10126 | | | |
| 10127 | fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10128 | const o = self.ng.object; | | |
| 10129 | const pt = self.ng.pt; | | |
| 10130 | const zcu = pt.zcu; | | |
| 10131 | const ip = &zcu.intern_pool; | | |
| 10132 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 10133 | const operand = try self.resolveInst(ty_op.operand); | | |
| 10134 | const error_set_ty = ty_op.ty.toType(); | | |
| 10135 | | | |
| 10136 | const names = error_set_ty.errorSetNames(zcu); | | |
| 10137 | const valid_block = try self.wip.block(@intCast(names.len), "Valid"); | | |
| 10138 | const invalid_block = try self.wip.block(1, "Invalid"); | | |
| 10139 | const end_block = try self.wip.block(2, "End"); | | |
| 10140 | var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none); | | |
| 10141 | defer wip_switch.finish(&self.wip); | | |
| 10142 | | | |
| 10143 | for (0..names.len) |name_index| { | | |
| 10144 | const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?; | | |
| 10145 | const this_tag_int_value = try o.builder.intConst(try o.errorIntType(pt), err_int); | | |
| 10146 | try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip); | | |
| 10147 | } | | |
| 10148 | self.wip.cursor = .{ .block = valid_block }; | | |
| 10149 | _ = try self.wip.br(end_block); | | |
| 10150 | | | |
| 10151 | self.wip.cursor = .{ .block = invalid_block }; | | |
| 10152 | _ = try self.wip.br(end_block); | | |
| 10153 | | | |
| 10154 | self.wip.cursor = .{ .block = end_block }; | | |
| 10155 | const phi = try self.wip.phi(.i1, ""); | | |
| 10156 | phi.finish(&.{ .true, .false }, &.{ valid_block, invalid_block }, &self.wip); | | |
| 10157 | return phi.toValue(); | | |
| 10158 | } | | |
| 10159 | | | |
| 10160 | fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10161 | const o = self.ng.object; | | |
| 10162 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 10163 | const operand = try self.resolveInst(un_op); | | |
| 10164 | const enum_ty = self.typeOf(un_op); | | |
| 10165 | | | |
| 10166 | const llvm_fn = try self.getIsNamedEnumValueFunction(enum_ty); | | |
| 10167 | return self.wip.call( | | |
| 10168 | .normal, | | |
| 10169 | .fastcc, | | |
| 10170 | .none, | | |
| 10171 | llvm_fn.typeOf(&o.builder), | | |
| 10172 | llvm_fn.toValue(&o.builder), | | |
| 10173 | &.{operand}, | | |
| 10174 | "", | | |
| 10175 | ); | | |
| 10176 | } | | |
| 10177 | | | |
| 10178 | fn getIsNamedEnumValueFunction(self: *FuncGen, enum_ty: Type) !Builder.Function.Index { | | |
| 10179 | const o = self.ng.object; | | |
| 10180 | const pt = self.ng.pt; | | |
| 10181 | const zcu = pt.zcu; | | |
| 10182 | const ip = &zcu.intern_pool; | | |
| 10183 | | | |
| 10184 | const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern()); | | |
| 10185 | if (gop.found_existing) return gop.value_ptr.*; | | |
| 10186 | errdefer assert(o.named_enum_map.remove(enum_ty.toIntern())); | | |
| 10187 | const function_index = try o.builder.addFunction( | | |
| 10188 | // Dummy function type; `updateIsNamedEnumValue` will replace it with the correct type. | | |
| 10189 | // TODO: change the builder API so we don't need to do this. | | |
| 10190 | try o.builder.fnType(.void, &.{}, .normal), | | |
| 10191 | try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}), | | |
| 10192 | toLlvmAddressSpace(.generic, zcu.getTarget()), | | |
| 10193 | ); | | |
| 10194 | gop.value_ptr.* = function_index; | | |
| 10195 | try o.updateIsNamedEnumValueFunction(pt, enum_ty, function_index); | | |
| 10196 | return function_index; | | |
| 10197 | } | | |
| 10198 | | | |
| 10199 | fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10200 | const o = self.ng.object; | | |
| 10201 | const pt = self.ng.pt; | | |
| 10202 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 10203 | const operand = try self.resolveInst(un_op); | | |
| 10204 | const enum_ty = self.typeOf(un_op); | | |
| 10205 | | | |
| 10206 | const llvm_fn = try o.getEnumTagNameFunction(pt, enum_ty); | | |
| 10207 | return self.wip.call( | | |
| 10208 | .normal, | | |
| 10209 | .fastcc, | | |
| 10210 | .none, | | |
| 10211 | llvm_fn.typeOf(&o.builder), | | |
| 10212 | llvm_fn.toValue(&o.builder), | | |
| 10213 | &.{operand}, | | |
| 10214 | "", | | |
| 10215 | ); | | |
| 10216 | } | | |
| 10217 | | | |
| 10218 | fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10219 | const o = self.ng.object; | | |
| 10220 | const pt = self.ng.pt; | | |
| 10221 | const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; | | |
| 10222 | const operand = try self.resolveInst(un_op); | | |
| 10223 | const slice_ty = self.typeOfIndex(inst); | | |
| 10224 | const slice_llvm_ty = try o.lowerType(pt, slice_ty); | | |
| 10225 | | | |
| 10226 | // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed. | | |
| 10227 | const extended_operand = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), ""); | | |
| 10228 | | | |
| 10229 | const error_name_table_ptr = try self.getErrorNameTable(); | | |
| 10230 | const error_name_table = | | |
| 10231 | try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, ""); | | |
| 10232 | const error_name_ptr = | | |
| 10233 | try self.wip.gep(.inbounds, slice_llvm_ty, error_name_table, &.{extended_operand}, ""); | | |
| 10234 | return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, ""); | | |
| 10235 | } | | |
| 10236 | | | |
| 10237 | fn airSplat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10238 | const o = self.ng.object; | | |
| 10239 | const pt = self.ng.pt; | | |
| 10240 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 10241 | const scalar = try self.resolveInst(ty_op.operand); | | |
| 10242 | const vector_ty = self.typeOfIndex(inst); | | |
| 10243 | return self.wip.splatVector(try o.lowerType(pt, vector_ty), scalar, ""); | | |
| 10244 | } | | |
| 10245 | | | |
| 10246 | fn airSelect(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10247 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | | |
| 10248 | const extra = self.air.extraData(Air.Bin, pl_op.payload).data; | | |
| 10249 | const pred = try self.resolveInst(pl_op.operand); | | |
| 10250 | const a = try self.resolveInst(extra.lhs); | | |
| 10251 | const b = try self.resolveInst(extra.rhs); | | |
| 10252 | | | |
| 10253 | return self.wip.select(.normal, pred, a, b, ""); | | |
| 10254 | } | | |
| 10255 | | | |
| 10256 | fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10257 | const o = fg.ng.object; | | |
| 10258 | const pt = fg.ng.pt; | | |
| 10259 | const zcu = pt.zcu; | | |
| 10260 | const gpa = zcu.gpa; | | |
| 10261 | | | |
| 10262 | const unwrapped = fg.air.unwrapShuffleOne(zcu, inst); | | |
| 10263 | | | |
| 10264 | const operand = try fg.resolveInst(unwrapped.operand); | | |
| 10265 | const mask = unwrapped.mask; | | |
| 10266 | const operand_ty = fg.typeOf(unwrapped.operand); | | |
| 10267 | const llvm_operand_ty = try o.lowerType(pt, operand_ty); | | |
| 10268 | const llvm_result_ty = try o.lowerType(pt, unwrapped.result_ty); | | |
| 10269 | const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu)); | | |
| 10270 | const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty); | | |
| 10271 | const llvm_poison_mask_elem = try o.builder.poisonConst(.i32); | | |
| 10272 | const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32); | | |
| 10273 | | | |
| 10274 | // LLVM requires that the two input vectors have the same length, so lowering isn't trivial. | | |
| 10275 | // And, in the words of jacobly0: "llvm sucks at shuffles so we do have to hold its hand at | | |
| 10276 | // least a bit". So, there are two cases here. | | |
| 10277 | // | | |
| 10278 | // If the operand length equals the mask length, we do just the one `shufflevector`, where | | |
| 10279 | // the second operand is a constant vector with comptime-known elements at the right indices | | |
| 10280 | // and poison values elsewhere (in the indices which won't be selected). | | |
| 10281 | // | | |
| 10282 | // Otherwise, we lower to *two* `shufflevector` instructions. The first shuffles the runtime | | |
| 10283 | // operand with an all-poison vector to extract and correctly position all of the runtime | | |
| 10284 | // elements. We also make a constant vector with all of the comptime elements correctly | | |
| 10285 | // positioned. Then, our second instruction selects elements from those "runtime-or-poison" | | |
| 10286 | // and "comptime-or-poison" vectors to compute the result. | | |
| 10287 | | | |
| 10288 | // This buffer is used primarily for the mask constants. | | |
| 10289 | const llvm_elem_buf = try gpa.alloc(Builder.Constant, mask.len); | | |
| 10290 | defer gpa.free(llvm_elem_buf); | | |
| 10291 | | | |
| 10292 | // ...but first, we'll collect all of the comptime-known values. | | |
| 10293 | var any_defined_comptime_value = false; | | |
| 10294 | for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| { | | |
| 10295 | llvm_elem.* = switch (mask_elem.unwrap()) { | | |
| 10296 | .elem => llvm_poison_elem, | | |
| 10297 | .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) elem: { | | |
| 10298 | any_defined_comptime_value = true; | | |
| 10299 | break :elem try o.lowerValue(pt, val); | | |
| 10300 | } else llvm_poison_elem, | | |
| 10301 | }; | | |
| 10302 | } | | |
| 10303 | // This vector is like the result, but runtime elements are replaced with poison. | | |
| 10304 | const comptime_and_poison: Builder.Value = if (any_defined_comptime_value) vec: { | | |
| 10305 | break :vec try o.builder.vectorValue(llvm_result_ty, llvm_elem_buf); | | |
| 10306 | } else try o.builder.poisonValue(llvm_result_ty); | | |
| 10307 | | | |
| 10308 | if (operand_ty.vectorLen(zcu) == mask.len) { | | |
| 10309 | // input length equals mask/output length, so we lower to one instruction | | |
| 10310 | for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| { | | |
| 10311 | llvm_elem.* = switch (mask_elem.unwrap()) { | | |
| 10312 | .elem => |idx| try o.builder.intConst(.i32, idx), | | |
| 10313 | .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: { | | |
| 10314 | break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx); | | |
| 10315 | } else llvm_poison_mask_elem, | | |
| 10316 | }; | | |
| 10317 | } | | |
| 10318 | return fg.wip.shuffleVector( | | |
| 10319 | operand, | | |
| 10320 | comptime_and_poison, | | |
| 10321 | try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf), | | |
| 10322 | "", | | |
| 10323 | ); | | |
| 10324 | } | | |
| 10325 | | | |
| 10326 | for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| { | | |
| 10327 | llvm_elem.* = switch (mask_elem.unwrap()) { | | |
| 10328 | .elem => |idx| try o.builder.intConst(.i32, idx), | | |
| 10329 | .value => llvm_poison_mask_elem, | | |
| 10330 | }; | | |
| 10331 | } | | |
| 10332 | // This vector is like our result, but all comptime-known elements are poison. | | |
| 10333 | const runtime_and_poison = try fg.wip.shuffleVector( | | |
| 10334 | operand, | | |
| 10335 | try o.builder.poisonValue(llvm_operand_ty), | | |
| 10336 | try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf), | | |
| 10337 | "", | | |
| 10338 | ); | | |
| 10339 | | | |
| 10340 | if (!any_defined_comptime_value) { | | |
| 10341 | // `comptime_and_poison` is just poison; a second shuffle would be a nop. | | |
| 10342 | return runtime_and_poison; | | |
| 10343 | } | | |
| 10344 | | | |
| 10345 | // In this second shuffle, the inputs, the mask, and the output all have the same length. | | |
| 10346 | for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| { | | |
| 10347 | llvm_elem.* = switch (mask_elem.unwrap()) { | | |
| 10348 | .elem => try o.builder.intConst(.i32, elem_idx), | | |
| 10349 | .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: { | | |
| 10350 | break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx); | | |
| 10351 | } else llvm_poison_mask_elem, | | |
| 10352 | }; | | |
| 10353 | } | | |
| 10354 | // Merge the runtime and comptime elements with the mask we just built. | | |
| 10355 | return fg.wip.shuffleVector( | | |
| 10356 | runtime_and_poison, | | |
| 10357 | comptime_and_poison, | | |
| 10358 | try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf), | | |
| 10359 | "", | | |
| 10360 | ); | | |
| 10361 | } | | |
| 10362 | | | |
| 10363 | fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10364 | const o = fg.ng.object; | | |
| 10365 | const pt = fg.ng.pt; | | |
| 10366 | const zcu = pt.zcu; | | |
| 10367 | const gpa = zcu.gpa; | | |
| 10368 | | | |
| 10369 | const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst); | | |
| 10370 | | | |
| 10371 | const mask = unwrapped.mask; | | |
| 10372 | const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu)); | | |
| 10373 | const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32); | | |
| 10374 | const llvm_poison_mask_elem = try o.builder.poisonConst(.i32); | | |
| 10375 | | | |
| 10376 | // This is kind of simpler than in `airShuffleOne`. We extend the shorter vector to the | | |
| 10377 | // length of the longer one with an initial `shufflevector` if necessary, and then do the | | |
| 10378 | // actual computation with a second `shufflevector`. | | |
| 10379 | | | |
| 10380 | const operand_a_len = fg.typeOf(unwrapped.operand_a).vectorLen(zcu); | | |
| 10381 | const operand_b_len = fg.typeOf(unwrapped.operand_b).vectorLen(zcu); | | |
| 10382 | const operand_len: u32 = @max(operand_a_len, operand_b_len); | | |
| 10383 | | | |
| 10384 | // If we need to extend an operand, this is the type that mask will have. | | |
| 10385 | const llvm_operand_mask_ty = try o.builder.vectorType(.normal, operand_len, .i32); | | |
| 10386 | | | |
| 10387 | const llvm_elem_buf = try gpa.alloc(Builder.Constant, @max(mask.len, operand_len)); | | |
| 10388 | defer gpa.free(llvm_elem_buf); | | |
| 10389 | | | |
| 10390 | const operand_a: Builder.Value = extend: { | | |
| 10391 | const raw = try fg.resolveInst(unwrapped.operand_a); | | |
| 10392 | if (operand_a_len == operand_len) break :extend raw; | | |
| 10393 | // Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>` | | |
| 10394 | const mask_elems = llvm_elem_buf[0..operand_len]; | | |
| 10395 | for (mask_elems[0..operand_a_len], 0..) |*llvm_elem, elem_idx| { | | |
| 10396 | llvm_elem.* = try o.builder.intConst(.i32, elem_idx); | | |
| 10397 | } | | |
| 10398 | @memset(mask_elems[operand_a_len..], llvm_poison_mask_elem); | | |
| 10399 | const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_a_len, llvm_elem_ty); | | |
| 10400 | break :extend try fg.wip.shuffleVector( | | |
| 10401 | raw, | | |
| 10402 | try o.builder.poisonValue(llvm_this_operand_ty), | | |
| 10403 | try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems), | | |
| 10404 | "", | | |
| 10405 | ); | | |
| 10406 | }; | | |
| 10407 | const operand_b: Builder.Value = extend: { | | |
| 10408 | const raw = try fg.resolveInst(unwrapped.operand_b); | | |
| 10409 | if (operand_b_len == operand_len) break :extend raw; | | |
| 10410 | // Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>` | | |
| 10411 | const mask_elems = llvm_elem_buf[0..operand_len]; | | |
| 10412 | for (mask_elems[0..operand_b_len], 0..) |*llvm_elem, elem_idx| { | | |
| 10413 | llvm_elem.* = try o.builder.intConst(.i32, elem_idx); | | |
| 10414 | } | | |
| 10415 | @memset(mask_elems[operand_b_len..], llvm_poison_mask_elem); | | |
| 10416 | const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_b_len, llvm_elem_ty); | | |
| 10417 | break :extend try fg.wip.shuffleVector( | | |
| 10418 | raw, | | |
| 10419 | try o.builder.poisonValue(llvm_this_operand_ty), | | |
| 10420 | try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems), | | |
| 10421 | "", | | |
| 10422 | ); | | |
| 10423 | }; | | |
| 10424 | | | |
| 10425 | // `operand_a` and `operand_b` now have the same length (we've extended the shorter one with | | |
| 10426 | // an initial shuffle if necessary). Now for the easy bit. | | |
| 10427 | | | |
| 10428 | const mask_elems = llvm_elem_buf[0..mask.len]; | | |
| 10429 | for (mask, mask_elems) |mask_elem, *llvm_mask_elem| { | | |
| 10430 | llvm_mask_elem.* = switch (mask_elem.unwrap()) { | | |
| 10431 | .a_elem => |idx| try o.builder.intConst(.i32, idx), | | |
| 10432 | .b_elem => |idx| try o.builder.intConst(.i32, operand_len + idx), | | |
| 10433 | .undef => llvm_poison_mask_elem, | | |
| 10434 | }; | | |
| 10435 | } | | |
| 10436 | return fg.wip.shuffleVector( | | |
| 10437 | operand_a, | | |
| 10438 | operand_b, | | |
| 10439 | try o.builder.vectorValue(llvm_mask_ty, mask_elems), | | |
| 10440 | "", | | |
| 10441 | ); | | |
| 10442 | } | | |
| 10443 | | | |
| 10444 | /// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result. | | |
| 10445 | /// | | |
| 10446 | /// Equivalent to: | | |
| 10447 | /// reduce: { | | |
| 10448 | /// var i: usize = 0; | | |
| 10449 | /// var accum: T = init; | | |
| 10450 | /// while (i < vec.len) : (i += 1) { | | |
| 10451 | /// accum = llvm_fn(accum, vec[i]); | | |
| 10452 | /// } | | |
| 10453 | /// break :reduce accum; | | |
| 10454 | /// } | | |
| 10455 | /// | | |
| 10456 | fn buildReducedCall( | | |
| 10457 | self: *FuncGen, | | |
| 10458 | llvm_fn: Builder.Function.Index, | | |
| 10459 | operand_vector: Builder.Value, | | |
| 10460 | vector_len: usize, | | |
| 10461 | accum_init: Builder.Value, | | |
| 10462 | ) !Builder.Value { | | |
| 10463 | const o = self.ng.object; | | |
| 10464 | const pt = self.ng.pt; | | |
| 10465 | const usize_ty = try o.lowerType(pt, Type.usize); | | |
| 10466 | const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len); | | |
| 10467 | const llvm_result_ty = accum_init.typeOfWip(&self.wip); | | |
| 10468 | | | |
| 10469 | // Allocate and initialize our mutable variables | | |
| 10470 | const i_ptr = try self.buildAlloca(usize_ty, .default); | | |
| 10471 | _ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default); | | |
| 10472 | const accum_ptr = try self.buildAlloca(llvm_result_ty, .default); | | |
| 10473 | _ = try self.wip.store(.normal, accum_init, accum_ptr, .default); | | |
| 10474 | | | |
| 10475 | // Setup the loop | | |
| 10476 | const loop = try self.wip.block(2, "ReduceLoop"); | | |
| 10477 | const loop_exit = try self.wip.block(1, "AfterReduce"); | | |
| 10478 | _ = try self.wip.br(loop); | | |
| 10479 | { | | |
| 10480 | self.wip.cursor = .{ .block = loop }; | | |
| 10481 | | | |
| 10482 | // while (i < vec.len) | | |
| 10483 | const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, ""); | | |
| 10484 | const cond = try self.wip.icmp(.ult, i, llvm_vector_len, ""); | | |
| 10485 | const loop_then = try self.wip.block(1, "ReduceLoopThen"); | | |
| 10486 | | | |
| 10487 | _ = try self.wip.brCond(cond, loop_then, loop_exit, .none); | | |
| 10488 | | | |
| 10489 | { | | |
| 10490 | self.wip.cursor = .{ .block = loop_then }; | | |
| 10491 | | | |
| 10492 | // accum = f(accum, vec[i]); | | |
| 10493 | const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, ""); | | |
| 10494 | const element = try self.wip.extractElement(operand_vector, i, ""); | | |
| 10495 | const new_accum = try self.wip.call( | | |
| 10496 | .normal, | | |
| 10497 | .ccc, | | |
| 10498 | .none, | | |
| 10499 | llvm_fn.typeOf(&o.builder), | | |
| 10500 | llvm_fn.toValue(&o.builder), | | |
| 10501 | &.{ accum, element }, | | |
| 10502 | "", | | |
| 10503 | ); | | |
| 10504 | _ = try self.wip.store(.normal, new_accum, accum_ptr, .default); | | |
| 10505 | | | |
| 10506 | // i += 1 | | |
| 10507 | const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), ""); | | |
| 10508 | _ = try self.wip.store(.normal, new_i, i_ptr, .default); | | |
| 10509 | _ = try self.wip.br(loop); | | |
| 10510 | } | | |
| 10511 | } | | |
| 10512 | | | |
| 10513 | self.wip.cursor = .{ .block = loop_exit }; | | |
| 10514 | return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, ""); | | |
| 10515 | } | | |
| 10516 | | | |
| 10517 | fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { | | |
| 10518 | const o = self.ng.object; | | |
| 10519 | const pt = self.ng.pt; | | |
| 10520 | const zcu = pt.zcu; | | |
| 10521 | const target = zcu.getTarget(); | | |
| 10522 | | | |
| 10523 | const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce; | | |
| 10524 | const operand = try self.resolveInst(reduce.operand); | | |
| 10525 | const operand_ty = self.typeOf(reduce.operand); | | |
| 10526 | const llvm_operand_ty = try o.lowerType(pt, operand_ty); | | |
| 10527 | const scalar_ty = self.typeOfIndex(inst); | | |
| 10528 | const llvm_scalar_ty = try o.lowerType(pt, scalar_ty); | | |
| 10529 | | | |
| 10530 | switch (reduce.operation) { | | |
| 10531 | .And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { | | |
| 10532 | .And => .@"vector.reduce.and", | | |
| 10533 | .Or => .@"vector.reduce.or", | | |
| 10534 | .Xor => .@"vector.reduce.xor", | | |
| 10535 | else => unreachable, | | |
| 10536 | }, &.{llvm_operand_ty}, &.{operand}, ""), | | |
| 10537 | .Min, .Max => switch (scalar_ty.zigTypeTag(zcu)) { | | |
| 10538 | .int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { | | |
| 10539 | .Min => if (scalar_ty.isSignedInt(zcu)) | | |
| 10540 | .@"vector.reduce.smin" | | |
| 10541 | else | | |
| 10542 | .@"vector.reduce.umin", | | |
| 10543 | .Max => if (scalar_ty.isSignedInt(zcu)) | | |
| 10544 | .@"vector.reduce.smax" | | |
| 10545 | else | | |
| 10546 | .@"vector.reduce.umax", | | |
| 10547 | else => unreachable, | | |
| 10548 | }, &.{llvm_operand_ty}, &.{operand}, ""), | | |
| 10549 | .float => if (intrinsicsAllowed(scalar_ty, target)) | | |
| 10550 | return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) { | | |
| 10551 | .Min => .@"vector.reduce.fmin", | | |
| 10552 | .Max => .@"vector.reduce.fmax", | | |
| 10553 | else => unreachable, | | |
| 10554 | }, &.{llvm_operand_ty}, &.{operand}, ""), | | |
| 10555 | else => unreachable, | | |
| 10556 | }, | | |
| 10557 | .Add, .Mul => switch (scalar_ty.zigTypeTag(zcu)) { | | |
| 10558 | .int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { | | |
| 10559 | .Add => .@"vector.reduce.add", | | |
| 10560 | .Mul => .@"vector.reduce.mul", | | |
| 10561 | else => unreachable, | | |
| 10562 | }, &.{llvm_operand_ty}, &.{operand}, ""), | | |
| 10563 | .float => if (intrinsicsAllowed(scalar_ty, target)) | | |
| 10564 | return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) { | | |
| 10565 | .Add => .@"vector.reduce.fadd", | | |
| 10566 | .Mul => .@"vector.reduce.fmul", | | |
| 10567 | else => unreachable, | | |
| 10568 | }, &.{llvm_operand_ty}, &.{ switch (reduce.operation) { | | |
| 10569 | .Add => try o.builder.fpValue(llvm_scalar_ty, -0.0), | | |
| 10570 | .Mul => try o.builder.fpValue(llvm_scalar_ty, 1.0), | | |
| 10571 | else => unreachable, | | |
| 10572 | }, operand }, ""), | | |
| 10573 | else => unreachable, | | |
| 10574 | }, | | |
| 10575 | } | | |
| 10576 | | | |
| 10577 | // Reduction could not be performed with intrinsics. | | |
| 10578 | // Use a manual loop over a softfloat call instead. | | |
| 10579 | const float_bits = scalar_ty.floatBits(target); | | |
| 10580 | const fn_name = switch (reduce.operation) { | | |
| 10581 | .Min => try o.builder.strtabStringFmt("{s}fmin{s}", .{ | | |
| 10582 | libcFloatPrefix(float_bits), libcFloatSuffix(float_bits), | | |
| 10583 | }), | | |
| 10584 | .Max => try o.builder.strtabStringFmt("{s}fmax{s}", .{ | | |
| 10585 | libcFloatPrefix(float_bits), libcFloatSuffix(float_bits), | | |
| 10586 | }), | | |
| 10587 | .Add => try o.builder.strtabStringFmt("__add{s}f3", .{ | | |
| 10588 | compilerRtFloatAbbrev(float_bits), | | |
| 10589 | }), | | |
| 10590 | .Mul => try o.builder.strtabStringFmt("__mul{s}f3", .{ | | |
| 10591 | compilerRtFloatAbbrev(float_bits), | | |
| 10592 | }), | | |
| 10593 | else => unreachable, | | |
| 10594 | }; | | |
| 10595 | | | |
| 10596 | const libc_fn = | | |
| 10597 | try self.getLibcFunction(fn_name, &.{ llvm_scalar_ty, llvm_scalar_ty }, llvm_scalar_ty); | | |
| 10598 | const init_val = switch (llvm_scalar_ty) { | | |
| 10599 | .i16 => try o.builder.intValue(.i16, @as(i16, @bitCast( | | |
| 10600 | @as(f16, switch (reduce.operation) { | | |
| 10601 | .Min, .Max => std.math.nan(f16), | | |
| 10602 | .Add => -0.0, | | |
| 10603 | .Mul => 1.0, | | |
| 10604 | else => unreachable, | | |
| 10605 | }), | | |
| 10606 | ))), | | |
| 10607 | .i80 => try o.builder.intValue(.i80, @as(i80, @bitCast( | | |
| 10608 | @as(f80, switch (reduce.operation) { | | |
| 10609 | .Min, .Max => std.math.nan(f80), | | |
| 10610 | .Add => -0.0, | | |
| 10611 | .Mul => 1.0, | | |
| 10612 | else => unreachable, | | |
| 10613 | }), | | |
| 10614 | ))), | | |
| 10615 | .i128 => try o.builder.intValue(.i128, @as(i128, @bitCast( | | |
| 10616 | @as(f128, switch (reduce.operation) { | | |
| 10617 | .Min, .Max => std.math.nan(f128), | | |
| 10618 | .Add => -0.0, | | |
| 10619 | .Mul => 1.0, | | |
| 10620 | else => unreachable, | | |
| 10621 | }), | | |
| 10622 | ))), | | |
| 10623 | else => unreachable, | | |
| 10624 | }; | | |
| 10625 | return self.buildReducedCall(libc_fn, operand, operand_ty.vectorLen(zcu), init_val); | | |
| 10626 | } | | |
| 10627 | | | |
| 10628 | fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10629 | const o = self.ng.object; | | |
| 10630 | const pt = self.ng.pt; | | |
| 10631 | const zcu = pt.zcu; | | |
| 10632 | const ip = &zcu.intern_pool; | | |
| 10633 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 10634 | const result_ty = self.typeOfIndex(inst); | | |
| 10635 | const len: usize = @intCast(result_ty.arrayLen(zcu)); | | |
| 10636 | const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); | | |
| 10637 | const llvm_result_ty = try o.lowerType(pt, result_ty); | | |
| 10638 | | | |
| 10639 | switch (result_ty.zigTypeTag(zcu)) { | | |
| 10640 | .vector => { | | |
| 10641 | var vector = try o.builder.poisonValue(llvm_result_ty); | | |
| 10642 | for (elements, 0..) |elem, i| { | | |
| 10643 | const index_u32 = try o.builder.intValue(.i32, i); | | |
| 10644 | const llvm_elem = try self.resolveInst(elem); | | |
| 10645 | vector = try self.wip.insertElement(vector, llvm_elem, index_u32, ""); | | |
| 10646 | } | | |
| 10647 | return vector; | | |
| 10648 | }, | | |
| 10649 | .@"struct" => { | | |
| 10650 | if (zcu.typeToPackedStruct(result_ty)) |struct_type| { | | |
| 10651 | const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type); | | |
| 10652 | const big_bits = backing_int_ty.bitSize(zcu); | | |
| 10653 | const int_ty = try o.builder.intType(@intCast(big_bits)); | | |
| 10654 | comptime assert(Type.packed_struct_layout_version == 2); | | |
| 10655 | var running_int = try o.builder.intValue(int_ty, 0); | | |
| 10656 | var running_bits: u16 = 0; | | |
| 10657 | for (elements, struct_type.field_types.get(ip)) |elem, field_ty| { | | |
| 10658 | if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue; | | |
| 10659 | | | |
| 10660 | const non_int_val = try self.resolveInst(elem); | | |
| 10661 | const ty_bit_size: u16 = @intCast(Type.fromInterned(field_ty).bitSize(zcu)); | | |
| 10662 | const small_int_ty = try o.builder.intType(ty_bit_size); | | |
| 10663 | const small_int_val = if (Type.fromInterned(field_ty).isPtrAtRuntime(zcu)) | | |
| 10664 | try self.wip.cast(.ptrtoint, non_int_val, small_int_ty, "") | | |
| 10665 | else | | |
| 10666 | try self.wip.cast(.bitcast, non_int_val, small_int_ty, ""); | | |
| 10667 | const shift_rhs = try o.builder.intValue(int_ty, running_bits); | | |
| 10668 | const extended_int_val = | | |
| 10669 | try self.wip.conv(.unsigned, small_int_val, int_ty, ""); | | |
| 10670 | const shifted = try self.wip.bin(.shl, extended_int_val, shift_rhs, ""); | | |
| 10671 | running_int = try self.wip.bin(.@"or", running_int, shifted, ""); | | |
| 10672 | running_bits += ty_bit_size; | | |
| 10673 | } | | |
| 10674 | return running_int; | | |
| 10675 | } | | |
| 10676 | | | |
| 10677 | assert(result_ty.containerLayout(zcu) != .@"packed"); | | |
| 10678 | | | |
| 10679 | if (isByRef(result_ty, zcu)) { | | |
| 10680 | // TODO in debug builds init to undef so that the padding will be 0xaa | | |
| 10681 | // even if we fully populate the fields. | | |
| 10682 | const alignment = result_ty.abiAlignment(zcu).toLlvm(); | | |
| 10683 | const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment); | | |
| 10684 | | | |
| 10685 | for (elements, 0..) |elem, i| { | | |
| 10686 | if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue; | | |
| 10687 | | | |
| 10688 | const llvm_elem = try self.resolveInst(elem); | | |
| 10689 | const llvm_i = o.llvmFieldIndex(result_ty, i).?; | | |
| 10690 | const field_ptr = try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, ""); | | |
| 10691 | | | |
| 10692 | const field_ptr_ty = try pt.ptrType(.{ | | |
| 10693 | .child = self.typeOf(elem).toIntern(), | | |
| 10694 | .flags = .{ | | |
| 10695 | .alignment = result_ty.explicitFieldAlignment(i, zcu), | | |
| 10696 | }, | | |
| 10697 | }); | | |
| 10698 | try self.store(field_ptr, field_ptr_ty, llvm_elem, .none); | | |
| 10699 | } | | |
| 10700 | | | |
| 10701 | return alloca_inst; | | |
| 10702 | } else { | | |
| 10703 | var result = try o.builder.poisonValue(llvm_result_ty); | | |
| 10704 | for (elements, 0..) |elem, i| { | | |
| 10705 | if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue; | | |
| 10706 | | | |
| 10707 | const llvm_elem = try self.resolveInst(elem); | | |
| 10708 | const llvm_i = o.llvmFieldIndex(result_ty, i).?; | | |
| 10709 | result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, ""); | | |
| 10710 | } | | |
| 10711 | return result; | | |
| 10712 | } | | |
| 10713 | }, | | |
| 10714 | .array => { | | |
| 10715 | assert(isByRef(result_ty, zcu)); | | |
| 10716 | | | |
| 10717 | const llvm_usize = try o.lowerType(pt, Type.usize); | | |
| 10718 | const usize_zero = try o.builder.intValue(llvm_usize, 0); | | |
| 10719 | const alignment = result_ty.abiAlignment(zcu).toLlvm(); | | |
| 10720 | const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment); | | |
| 10721 | | | |
| 10722 | const array_info = result_ty.arrayInfo(zcu); | | |
| 10723 | const elem_ptr_ty = try pt.ptrType(.{ | | |
| 10724 | .child = array_info.elem_type.toIntern(), | | |
| 10725 | }); | | |
| 10726 | | | |
| 10727 | for (elements, 0..) |elem, i| { | | |
| 10728 | const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{ | | |
| 10729 | usize_zero, try o.builder.intValue(llvm_usize, i), | | |
| 10730 | }, ""); | | |
| 10731 | const llvm_elem = try self.resolveInst(elem); | | |
| 10732 | try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none); | | |
| 10733 | } | | |
| 10734 | if (array_info.sentinel) |sent_val| { | | |
| 10735 | const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{ | | |
| 10736 | usize_zero, try o.builder.intValue(llvm_usize, array_info.len), | | |
| 10737 | }, ""); | | |
| 10738 | const llvm_elem = try self.resolveValue(sent_val); | | |
| 10739 | try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none); | | |
| 10740 | } | | |
| 10741 | | | |
| 10742 | return alloca_inst; | | |
| 10743 | }, | | |
| 10744 | else => unreachable, | | |
| 10745 | } | | |
| 10746 | } | | |
| 10747 | | | |
| 10748 | fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10749 | const o = self.ng.object; | | |
| 10750 | const pt = self.ng.pt; | | |
| 10751 | const zcu = pt.zcu; | | |
| 10752 | const ip = &zcu.intern_pool; | | |
| 10753 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | | |
| 10754 | const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; | | |
| 10755 | const union_ty = self.typeOfIndex(inst); | | |
| 10756 | const union_llvm_ty = try o.lowerType(pt, union_ty); | | |
| 10757 | const union_obj = zcu.typeToUnion(union_ty).?; | | |
| 10758 | | | |
| 10759 | assert(union_obj.layout != .@"packed"); | | |
| 10760 | | | |
| 10761 | const layout = Type.getUnionLayout(union_obj, zcu); | | |
| 10762 | | | |
| 10763 | const tag_int_val = blk: { | | |
| 10764 | const tag_ty = union_ty.unionTagTypeHypothetical(zcu); | | |
| 10765 | const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index); | | |
| 10766 | break :blk tag_val.intFromEnum(zcu); | | |
| 10767 | }; | | |
| 10768 | if (layout.payload_size == 0) { | | |
| 10769 | if (layout.tag_size == 0) { | | |
| 10770 | return .none; | | |
| 10771 | } | | |
| 10772 | assert(!isByRef(union_ty, zcu)); | | |
| 10773 | var big_int_space: Value.BigIntSpace = undefined; | | |
| 10774 | const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu); | | |
| 10775 | return try o.builder.bigIntValue(union_llvm_ty, tag_big_int); | | |
| 10776 | } | | |
| 10777 | assert(isByRef(union_ty, zcu)); | | |
| 10778 | // The llvm type of the alloca will be the named LLVM union type, and will not | | |
| 10779 | // necessarily match the format that we need, depending on which tag is active. | | |
| 10780 | // We must construct the correct unnamed struct type here, in order to then set | | |
| 10781 | // the fields appropriately. | | |
| 10782 | const alignment = layout.abi_align.toLlvm(); | | |
| 10783 | const result_ptr = try self.buildAlloca(union_llvm_ty, alignment); | | |
| 10784 | const llvm_payload = try self.resolveInst(extra.init); | | |
| 10785 | const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]); | | |
| 10786 | const field_llvm_ty = try o.lowerType(pt, field_ty); | | |
| 10787 | const field_size = field_ty.abiSize(zcu); | | |
| 10788 | const field_align = union_ty.explicitFieldAlignment(extra.field_index, zcu); | | |
| 10789 | const llvm_usize = try o.lowerType(pt, Type.usize); | | |
| 10790 | const usize_zero = try o.builder.intValue(llvm_usize, 0); | | |
| 10791 | | | |
| 10792 | assert(field_ty.hasRuntimeBits(zcu)); | | |
| 10793 | | | |
| 10794 | const llvm_union_ty = t: { | | |
| 10795 | const payload_ty = p: { | | |
| 10796 | if (field_size == layout.payload_size) { | | |
| 10797 | break :p field_llvm_ty; | | |
| 10798 | } | | |
| 10799 | const padding_len = layout.payload_size - field_size; | | |
| 10800 | break :p try o.builder.structType(.@"packed", &.{ | | |
| 10801 | field_llvm_ty, try o.builder.arrayType(padding_len, .i8), | | |
| 10802 | }); | | |
| 10803 | }; | | |
| 10804 | if (layout.tag_size == 0) break :t try o.builder.structType(.normal, &.{payload_ty}); | | |
| 10805 | const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); | | |
| 10806 | var fields: [3]Builder.Type = undefined; | | |
| 10807 | var fields_len: usize = 2; | | |
| 10808 | if (layout.tag_align.compare(.gte, layout.payload_align)) { | | |
| 10809 | fields = .{ tag_ty, payload_ty, undefined }; | | |
| 10810 | } else { | | |
| 10811 | fields = .{ payload_ty, tag_ty, undefined }; | | |
| 10812 | } | | |
| 10813 | if (layout.padding != 0) { | | |
| 10814 | fields[fields_len] = try o.builder.arrayType(layout.padding, .i8); | | |
| 10815 | fields_len += 1; | | |
| 10816 | } | | |
| 10817 | break :t try o.builder.structType(.normal, fields[0..fields_len]); | | |
| 10818 | }; | | |
| 10819 | | | |
| 10820 | // Now we follow the layout as expressed above with GEP instructions to set the | | |
| 10821 | // tag and the payload. | | |
| 10822 | const field_ptr_ty = try pt.ptrType(.{ | | |
| 10823 | .child = field_ty.toIntern(), | | |
| 10824 | .flags = .{ .alignment = field_align }, | | |
| 10825 | }); | | |
| 10826 | if (layout.tag_size == 0) { | | |
| 10827 | const indices = [3]Builder.Value{ usize_zero, .@"0", .@"0" }; | | |
| 10828 | const len: usize = if (field_size == layout.payload_size) 2 else 3; | | |
| 10829 | const field_ptr = | | |
| 10830 | try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], ""); | | |
| 10831 | try self.store(field_ptr, field_ptr_ty, llvm_payload, .none); | | |
| 10832 | return result_ptr; | | |
| 10833 | } | | |
| 10834 | | | |
| 10835 | { | | |
| 10836 | const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align)); | | |
| 10837 | const indices: [3]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, payload_index), .@"0" }; | | |
| 10838 | const len: usize = if (field_size == layout.payload_size) 2 else 3; | | |
| 10839 | const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], ""); | | |
| 10840 | try self.store(field_ptr, field_ptr_ty, llvm_payload, .none); | | |
| 10841 | } | | |
| 10842 | { | | |
| 10843 | const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); | | |
| 10844 | const indices: [2]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, tag_index) }; | | |
| 10845 | const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, &indices, ""); | | |
| 10846 | const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); | | |
| 10847 | var big_int_space: Value.BigIntSpace = undefined; | | |
| 10848 | const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu); | | |
| 10849 | const llvm_tag = try o.builder.bigIntValue(tag_ty, tag_big_int); | | |
| 10850 | const tag_alignment = Type.fromInterned(union_obj.enum_tag_type).abiAlignment(zcu).toLlvm(); | | |
| 10851 | _ = try self.wip.store(.normal, llvm_tag, field_ptr, tag_alignment); | | |
| 10852 | } | | |
| 10853 | | | |
| 10854 | return result_ptr; | | |
| 10855 | } | | |
| 10856 | | | |
| 10857 | fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10858 | const o = self.ng.object; | | |
| 10859 | const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch; | | |
| 10860 | | | |
| 10861 | comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.read) == 0); | | |
| 10862 | comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.write) == 1); | | |
| 10863 | | | |
| 10864 | comptime assert(prefetch.locality >= 0); | | |
| 10865 | comptime assert(prefetch.locality <= 3); | | |
| 10866 | | | |
| 10867 | comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.instruction) == 0); | | |
| 10868 | comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.data) == 1); | | |
| 10869 | | | |
| 10870 | // LLVM fails during codegen of instruction cache prefetchs for these architectures. | | |
| 10871 | // This is an LLVM bug as the prefetch intrinsic should be a noop if not supported | | |
| 10872 | // by the target. | | |
| 10873 | // To work around this, don't emit llvm.prefetch in this case. | | |
| 10874 | // See https://bugs.llvm.org/show_bug.cgi?id=21037 | | |
| 10875 | const zcu = self.ng.pt.zcu; | | |
| 10876 | const target = zcu.getTarget(); | | |
| 10877 | switch (prefetch.cache) { | | |
| 10878 | .instruction => switch (target.cpu.arch) { | | |
| 10879 | .x86_64, | | |
| 10880 | .x86, | | |
| 10881 | .powerpc, | | |
| 10882 | .powerpcle, | | |
| 10883 | .powerpc64, | | |
| 10884 | .powerpc64le, | | |
| 10885 | => return .none, | | |
| 10886 | .arm, .armeb, .thumb, .thumbeb => { | | |
| 10887 | switch (prefetch.rw) { | | |
| 10888 | .write => return .none, | | |
| 10889 | else => {}, | | |
| 10890 | } | | |
| 10891 | }, | | |
| 10892 | else => {}, | | |
| 10893 | }, | | |
| 10894 | .data => {}, | | |
| 10895 | } | | |
| 10896 | | | |
| 10897 | _ = try self.wip.callIntrinsic(.normal, .none, .prefetch, &.{.ptr}, &.{ | | |
| 10898 | try self.sliceOrArrayPtr(try self.resolveInst(prefetch.ptr), self.typeOf(prefetch.ptr)), | | |
| 10899 | try o.builder.intValue(.i32, prefetch.rw), | | |
| 10900 | try o.builder.intValue(.i32, prefetch.locality), | | |
| 10901 | try o.builder.intValue(.i32, prefetch.cache), | | |
| 10902 | }, ""); | | |
| 10903 | return .none; | | |
| 10904 | } | | |
| 10905 | | | |
| 10906 | fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10907 | const o = self.ng.object; | | |
| 10908 | const pt = self.ng.pt; | | |
| 10909 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 10910 | const inst_ty = self.typeOfIndex(inst); | | |
| 10911 | const operand = try self.resolveInst(ty_op.operand); | | |
| 10912 | | | |
| 10913 | return self.wip.cast(.addrspacecast, operand, try o.lowerType(pt, inst_ty), ""); | | |
| 10914 | } | | |
| 10915 | | | |
| 10916 | fn workIntrinsic( | | |
| 10917 | self: *FuncGen, | | |
| 10918 | dimension: u32, | | |
| 10919 | default: u32, | | |
| 10920 | comptime basename: []const u8, | | |
| 10921 | ) !Builder.Value { | | |
| 10922 | return self.wip.callIntrinsic(.normal, .none, switch (dimension) { | | |
| 10923 | 0 => @field(Builder.Intrinsic, basename ++ ".x"), | | |
| 10924 | 1 => @field(Builder.Intrinsic, basename ++ ".y"), | | |
| 10925 | 2 => @field(Builder.Intrinsic, basename ++ ".z"), | | |
| 10926 | else => return self.ng.object.builder.intValue(.i32, default), | | |
| 10927 | }, &.{}, &.{}, ""); | | |
| 10928 | } | | |
| 10929 | | | |
| 10930 | fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10931 | const target = self.ng.pt.zcu.getTarget(); | | |
| 10932 | | | |
| 10933 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | | |
| 10934 | const dimension = pl_op.payload; | | |
| 10935 | | | |
| 10936 | return switch (target.cpu.arch) { | | |
| 10937 | .amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workitem.id"), | | |
| 10938 | .nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.tid"), | | |
| 10939 | else => unreachable, | | |
| 10940 | }; | | |
| 10941 | } | | |
| 10942 | | | |
| 10943 | fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10944 | const o = self.ng.object; | | |
| 10945 | const pt = self.ng.pt; | | |
| 10946 | const target = pt.zcu.getTarget(); | | |
| 10947 | | | |
| 10948 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | | |
| 10949 | const dimension = pl_op.payload; | | |
| 10950 | | | |
| 10951 | switch (target.cpu.arch) { | | |
| 10952 | .amdgcn => { | | |
| 10953 | if (dimension >= 3) return .@"1"; | | |
| 10954 | | | |
| 10955 | // Fetch the dispatch pointer, which points to this structure: | | |
| 10956 | // https://github.com/RadeonOpenCompute/ROCR-Runtime/blob/adae6c61e10d371f7cbc3d0e94ae2c070cab18a4/src/inc/hsa.h#L2913 | | |
| 10957 | const dispatch_ptr = | | |
| 10958 | try self.wip.callIntrinsic(.normal, .none, .@"amdgcn.dispatch.ptr", &.{}, &.{}, ""); | | |
| 10959 | | | |
| 10960 | // Load the work_group_* member from the struct as u16. | | |
| 10961 | // Just treat the dispatch pointer as an array of u16 to keep things simple. | | |
| 10962 | const workgroup_size_ptr = try self.wip.gep(.inbounds, .i16, dispatch_ptr, &.{ | | |
| 10963 | try o.builder.intValue(try o.lowerType(pt, Type.usize), 2 + dimension), | | |
| 10964 | }, ""); | | |
| 10965 | const workgroup_size_alignment = comptime Builder.Alignment.fromByteUnits(2); | | |
| 10966 | return self.wip.load(.normal, .i16, workgroup_size_ptr, workgroup_size_alignment, ""); | | |
| 10967 | }, | | |
| 10968 | .nvptx, .nvptx64 => { | | |
| 10969 | return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid"); | | |
| 10970 | }, | | |
| 10971 | else => unreachable, | | |
| 10972 | } | | |
| 10973 | } | | |
| 10974 | | | |
| 10975 | fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | | |
| 10976 | const target = self.ng.pt.zcu.getTarget(); | | |
| 10977 | | | |
| 10978 | const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | | |
| 10979 | const dimension = pl_op.payload; | | |
| 10980 | | | |
| 10981 | return switch (target.cpu.arch) { | | |
| 10982 | .amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workgroup.id"), | | |
| 10983 | .nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.ctaid"), | | |
| 10984 | else => unreachable, | | |
| 10985 | }; | | |
| 10986 | } | | |
| 10987 | | | |
| 10988 | fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index { | | |
| 10989 | const o = self.ng.object; | | |
| 10990 | const pt = self.ng.pt; | | |
| 10991 | | | |
| 10992 | const table = o.error_name_table; | | |
| 10993 | if (table != .none) return table; | | |
| 10994 | | | |
| 10995 | // TODO: Address space | | |
| 10996 | const variable_index = | | |
| 10997 | try o.builder.addVariable(try o.builder.strtabString("__zig_err_name_table"), .ptr, .default); | | |
| 10998 | variable_index.setLinkage(.private, &o.builder); | | |
| 10999 | variable_index.setMutability(.constant, &o.builder); | | |
| 11000 | variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); | | |
| 11001 | variable_index.setAlignment( | | |
| 11002 | Type.slice_const_u8_sentinel_0.abiAlignment(pt.zcu).toLlvm(), | | |
| 11003 | &o.builder, | | |
| 11004 | ); | | |
| 11005 | | | |
| 11006 | o.error_name_table = variable_index; | | |
| 11007 | return variable_index; | | |
| 11008 | } | | |
| 11009 | | | |
| 11010 | /// Assumes the optional is not pointer-like and payload has bits. | | |
| 11011 | fn optCmpNull( | | |
| 11012 | self: *FuncGen, | | |
| 11013 | cond: Builder.IntegerCondition, | | |
| 11014 | opt_llvm_ty: Builder.Type, | | |
| 11015 | opt_handle: Builder.Value, | | |
| 11016 | is_by_ref: bool, | | |
| 11017 | access_kind: Builder.MemoryAccessKind, | | |
| 11018 | ) Allocator.Error!Builder.Value { | | |
| 11019 | const o = self.ng.object; | | |
| 11020 | const field = b: { | | |
| 11021 | if (is_by_ref) { | | |
| 11022 | const field_ptr = try self.wip.gepStruct(opt_llvm_ty, opt_handle, 1, ""); | | |
| 11023 | break :b try self.wip.load(access_kind, .i8, field_ptr, .default, ""); | | |
| 11024 | } | | |
| 11025 | break :b try self.wip.extractValue(opt_handle, &.{1}, ""); | | |
| 11026 | }; | | |
| 11027 | comptime assert(optional_layout_version == 3); | | |
| 11028 | | | |
| 11029 | return self.wip.icmp(cond, field, try o.builder.intValue(.i8, 0), ""); | | |
| 11030 | } | | |
| 11031 | | | |
| 11032 | /// Assumes the optional is not pointer-like and payload has bits. | | |
| 11033 | fn optPayloadHandle( | | |
| 11034 | fg: *FuncGen, | | |
| 11035 | opt_llvm_ty: Builder.Type, | | |
| 11036 | opt_handle: Builder.Value, | | |
| 11037 | opt_ty: Type, | | |
| 11038 | can_elide_load: bool, | | |
| 11039 | ) !Builder.Value { | | |
| 11040 | const pt = fg.ng.pt; | | |
| 11041 | const zcu = pt.zcu; | | |
| 11042 | const payload_ty = opt_ty.optionalChild(zcu); | | |
| 11043 | | | |
| 11044 | if (isByRef(opt_ty, zcu)) { | | |
| 11045 | // We have a pointer and we need to return a pointer to the first field. | | |
| 11046 | const payload_ptr = try fg.wip.gepStruct(opt_llvm_ty, opt_handle, 0, ""); | | |
| 11047 | | | |
| 11048 | const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm(); | | |
| 11049 | if (isByRef(payload_ty, zcu)) { | | |
| 11050 | if (can_elide_load) | | |
| 11051 | return payload_ptr; | | |
| 11052 | | | |
| 11053 | return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal); | | |
| 11054 | } | | |
| 11055 | return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_alignment); | | |
| 11056 | } | | |
| 11057 | | | |
| 11058 | assert(!isByRef(payload_ty, zcu)); | | |
| 11059 | return fg.wip.extractValue(opt_handle, &.{0}, ""); | | |
| 11060 | } | | |
| 11061 | | | |
| 11062 | fn buildOptional( | | |
| 11063 | self: *FuncGen, | | |
| 11064 | optional_ty: Type, | | |
| 11065 | payload: Builder.Value, | | |
| 11066 | non_null_bit: Builder.Value, | | |
| 11067 | ) !Builder.Value { | | |
| 11068 | const o = self.ng.object; | | |
| 11069 | const pt = self.ng.pt; | | |
| 11070 | const zcu = pt.zcu; | | |
| 11071 | const optional_llvm_ty = try o.lowerType(pt, optional_ty); | | |
| 11072 | const non_null_field = try self.wip.cast(.zext, non_null_bit, .i8, ""); | | |
| 11073 | | | |
| 11074 | if (isByRef(optional_ty, zcu)) { | | |
| 11075 | const payload_alignment = optional_ty.abiAlignment(pt.zcu).toLlvm(); | | |
| 11076 | const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment); | | |
| 11077 | | | |
| 11078 | { | | |
| 11079 | const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, ""); | | |
| 11080 | _ = try self.wip.store(.normal, payload, field_ptr, payload_alignment); | | |
| 11081 | } | | |
| 11082 | { | | |
| 11083 | const non_null_alignment = comptime Builder.Alignment.fromByteUnits(1); | | |
| 11084 | const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 1, ""); | | |
| 11085 | _ = try self.wip.store(.normal, non_null_field, field_ptr, non_null_alignment); | | |
| 11086 | } | | |
| 11087 | | | |
| 11088 | return alloca_inst; | | |
| 11089 | } | | |
| 11090 | | | |
| 11091 | return self.wip.buildAggregate(optional_llvm_ty, &.{ payload, non_null_field }, ""); | | |
| 11092 | } | | |
| 11093 | | | |
| 11094 | fn fieldPtr( | | |
| 11095 | self: *FuncGen, | | |
| 11096 | aggregate_ptr: Builder.Value, | | |
| 11097 | aggregate_ptr_ty: Type, | | |
| 11098 | field_index: u32, | | |
| 11099 | ) !Builder.Value { | | |
| 11100 | const o = self.ng.object; | | |
| 11101 | const pt = self.ng.pt; | | |
| 11102 | const zcu = pt.zcu; | | |
| 11103 | const aggregate_ty = aggregate_ptr_ty.childType(zcu); | | |
| 11104 | if (aggregate_ty.containerLayout(zcu) == .@"packed") { | | |
| 11105 | // A pointer to a bitpack field is equivalent to a pointer to the whole bitpack; the | | |
| 11106 | // bit offset is represented in the pointer *type*. | | |
| 11107 | return aggregate_ptr; | | |
| 11108 | } | | |
| 11109 | switch (aggregate_ty.zigTypeTag(zcu)) { | | |
| 11110 | .@"struct" => { | | |
| 11111 | if (!aggregate_ty.hasRuntimeBits(zcu)) { | | |
| 11112 | return aggregate_ptr; | | |
| 11113 | } | | |
| 11114 | const struct_llvm_ty = try o.lowerType(pt, aggregate_ty); | | |
| 11115 | if (o.llvmFieldIndex(aggregate_ty, field_index)) |llvm_field_index| { | | |
| 11116 | return self.wip.gepStruct(struct_llvm_ty, aggregate_ptr, llvm_field_index, ""); | | |
| 11117 | } else { | | |
| 11118 | // If we found no index then this means this is a zero sized field at the | | |
| 11119 | // end of the struct. Treat our struct pointer as an array of two and get | | |
| 11120 | // the index to the element at index `1` to get a pointer to the end of | | |
| 11121 | // the struct. | | |
| 11122 | const llvm_index = try o.builder.intValue( | | |
| 11123 | try o.lowerType(pt, Type.usize), | | |
| 11124 | @intFromBool(aggregate_ty.hasRuntimeBits(zcu)), | | |
| 11125 | ); | | |
| 11126 | return self.wip.gep(.inbounds, struct_llvm_ty, aggregate_ptr, &.{llvm_index}, ""); | | |
| 11127 | } | | |
| 11128 | }, | | |
| 11129 | .@"union" => { | | |
| 11130 | const layout = aggregate_ty.unionGetLayout(zcu); | | |
| 11131 | if (layout.payload_size == 0) return aggregate_ptr; | | |
| 11132 | const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align)); | | |
| 11133 | const union_llvm_ty = try o.lowerType(pt, aggregate_ty); | | |
| 11134 | return self.wip.gepStruct(union_llvm_ty, aggregate_ptr, payload_index, ""); | | |
| 11135 | }, | | |
| 11136 | else => unreachable, | | |
| 11137 | } | | |
| 11138 | } | | |
| 11139 | | | |
| 11140 | /// Load a value and, if needed, mask out padding bits for non byte-sized integer values. | | |
| 11141 | fn loadTruncate( | | |
| 11142 | fg: *FuncGen, | | |
| 11143 | access_kind: Builder.MemoryAccessKind, | | |
| 11144 | payload_ty: Type, | | |
| 11145 | payload_ptr: Builder.Value, | | |
| 11146 | payload_alignment: Builder.Alignment, | | |
| 11147 | ) !Builder.Value { | | |
| 11148 | // from https://llvm.org/docs/LangRef.html#load-instruction : | | |
| 11149 | // "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. " | | |
| 11150 | // => so load the byte aligned value and trunc the unwanted bits. | | |
| 11151 | | | |
| 11152 | const o = fg.ng.object; | | |
| 11153 | const pt = fg.ng.pt; | | |
| 11154 | const zcu = pt.zcu; | | |
| 11155 | const payload_llvm_ty = try o.lowerType(pt, payload_ty); | | |
| 11156 | const abi_size = payload_ty.abiSize(zcu); | | |
| 11157 | | | |
| 11158 | const load_llvm_ty = if (payload_ty.isAbiInt(zcu)) | | |
| 11159 | try o.builder.intType(@intCast(abi_size * 8)) | | |
| 11160 | else | | |
| 11161 | payload_llvm_ty; | | |
| 11162 | const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, ""); | | |
| 11163 | const shifted = if (payload_llvm_ty != load_llvm_ty and o.target.cpu.arch.endian() == .big) | | |
| 11164 | try fg.wip.bin(.lshr, loaded, try o.builder.intValue( | | |
| 11165 | load_llvm_ty, | | |
| 11166 | (payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8, | | |
| 11167 | ), "") | | |
| 11168 | else | | |
| 11169 | loaded; | | |
| 11170 | | | |
| 11171 | return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, ""); | | |
| 11172 | } | | |
| 11173 | | | |
| 11174 | /// Load a by-ref type by constructing a new alloca and performing a memcpy. | | |
| 11175 | fn loadByRef( | | |
| 11176 | fg: *FuncGen, | | |
| 11177 | ptr: Builder.Value, | | |
| 11178 | pointee_type: Type, | | |
| 11179 | ptr_alignment: Builder.Alignment, | | |
| 11180 | access_kind: Builder.MemoryAccessKind, | | |
| 11181 | ) !Builder.Value { | | |
| 11182 | const o = fg.ng.object; | | |
| 11183 | const pt = fg.ng.pt; | | |
| 11184 | const pointee_llvm_ty = try o.lowerType(pt, pointee_type); | | |
| 11185 | const result_align = InternPool.Alignment.fromLlvm(ptr_alignment) | | |
| 11186 | .max(pointee_type.abiAlignment(pt.zcu)).toLlvm(); | | |
| 11187 | const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align); | | |
| 11188 | const size_bytes = pointee_type.abiSize(pt.zcu); | | |
| 11189 | _ = try fg.wip.callMemCpy( | | |
| 11190 | result_ptr, | | |
| 11191 | result_align, | | |
| 11192 | ptr, | | |
| 11193 | ptr_alignment, | | |
| 11194 | try o.builder.intValue(try o.lowerType(pt, Type.usize), size_bytes), | | |
| 11195 | access_kind, | | |
| 11196 | fg.disable_intrinsics, | | |
| 11197 | ); | | |
| 11198 | return result_ptr; | | |
| 11199 | } | | |
| 11200 | | | |
| 11201 | /// This function always performs a copy. For isByRef=true types, it creates a new | | |
| 11202 | /// alloca and copies the value into it, then returns the alloca instruction. | | |
| 11203 | /// For isByRef=false types, it creates a load instruction and returns it. | | |
| 11204 | fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) !Builder.Value { | | |
| 11205 | const o = self.ng.object; | | |
| 11206 | const pt = self.ng.pt; | | |
| 11207 | const zcu = pt.zcu; | | |
| 11208 | const info = ptr_ty.ptrInfo(zcu); | | |
| 11209 | const elem_ty = Type.fromInterned(info.child); | | |
| 11210 | if (!elem_ty.hasRuntimeBits(zcu)) return .none; | | |
| 11211 | | | |
| 11212 | const ptr_alignment = (if (info.flags.alignment != .none) | | |
| 11213 | @as(InternPool.Alignment, info.flags.alignment) | | |
| 11214 | else | | |
| 11215 | elem_ty.abiAlignment(zcu)).toLlvm(); | | |
| 11216 | | | |
| 11217 | const access_kind: Builder.MemoryAccessKind = | | |
| 11218 | if (info.flags.is_volatile) .@"volatile" else .normal; | | |
| 11219 | | | |
| 11220 | if (info.flags.vector_index != .none) { | | |
| 11221 | const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); | | |
| 11222 | const vec_elem_ty = try o.lowerType(pt, elem_ty); | | |
| 11223 | const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty); | | |
| 11224 | | | |
| 11225 | const loaded_vector = try self.wip.load(access_kind, vec_ty, ptr, ptr_alignment, ""); | | |
| 11226 | return self.wip.extractElement(loaded_vector, index_u32, ""); | | |
| 11227 | } | | |
| 11228 | | | |
| 11229 | if (info.packed_offset.host_size == 0) { | | |
| 11230 | if (isByRef(elem_ty, zcu)) { | | |
| 11231 | return self.loadByRef(ptr, elem_ty, ptr_alignment, access_kind); | | |
| 11232 | } | | |
| 11233 | return self.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment); | | |
| 11234 | } | | |
| 11235 | | | |
| 11236 | const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8)); | | |
| 11237 | const containing_int = | | |
| 11238 | try self.wip.load(access_kind, containing_int_ty, ptr, ptr_alignment, ""); | | |
| 11239 | | | |
| 11240 | const elem_bits = ptr_ty.childType(zcu).bitSize(zcu); | | |
| 11241 | const shift_amt = try o.builder.intValue(containing_int_ty, info.packed_offset.bit_offset); | | |
| 11242 | const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, ""); | | |
| 11243 | const elem_llvm_ty = try o.lowerType(pt, elem_ty); | | |
| 11244 | | | |
| 11245 | if (isByRef(elem_ty, zcu)) { | | |
| 11246 | const result_align = elem_ty.abiAlignment(zcu).toLlvm(); | | |
| 11247 | const result_ptr = try self.buildAlloca(elem_llvm_ty, result_align); | | |
| 11248 | | | |
| 11249 | const same_size_int = try o.builder.intType(@intCast(elem_bits)); | | |
| 11250 | const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); | | |
| 11251 | _ = try self.wip.store(.normal, truncated_int, result_ptr, result_align); | | |
| 11252 | return result_ptr; | | |
| 11253 | } | | |
| 11254 | | | |
| 11255 | if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) { | | |
| 11256 | const same_size_int = try o.builder.intType(@intCast(elem_bits)); | | |
| 11257 | const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); | | |
| 11258 | return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); | | |
| 11259 | } | | |
| 11260 | | | |
| 11261 | if (elem_ty.isPtrAtRuntime(zcu)) { | | |
| 11262 | const same_size_int = try o.builder.intType(@intCast(elem_bits)); | | |
| 11263 | const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); | | |
| 11264 | return self.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, ""); | | |
| 11265 | } | | |
| 11266 | | | |
| 11267 | return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, ""); | | |
| 11268 | } | | |
| 11269 | | | |
| 11270 | fn store( | | |
| 11271 | self: *FuncGen, | | |
| 11272 | ptr: Builder.Value, | | |
| 11273 | ptr_ty: Type, | | |
| 11274 | elem: Builder.Value, | | |
| 11275 | ordering: Builder.AtomicOrdering, | | |
| 11276 | ) !void { | | |
| 11277 | const o = self.ng.object; | | |
| 11278 | const pt = self.ng.pt; | | |
| 11279 | const zcu = pt.zcu; | | |
| 11280 | const info = ptr_ty.ptrInfo(zcu); | | |
| 11281 | const elem_ty = Type.fromInterned(info.child); | | |
| 11282 | if (!elem_ty.hasRuntimeBits(zcu)) { | | |
| 11283 | return; | | |
| 11284 | } | | |
| 11285 | const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); | | |
| 11286 | const access_kind: Builder.MemoryAccessKind = | | |
| 11287 | if (info.flags.is_volatile) .@"volatile" else .normal; | | |
| 11288 | | | |
| 11289 | if (info.flags.vector_index != .none) { | | |
| 11290 | const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); | | |
| 11291 | const vec_elem_ty = try o.lowerType(pt, elem_ty); | | |
| 11292 | const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty); | | |
| 11293 | | | |
| 11294 | const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, ""); | | |
| 11295 | | | |
| 11296 | const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, ""); | | |
| 11297 | | | |
| 11298 | assert(ordering == .none); | | |
| 11299 | _ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment); | | |
| 11300 | return; | | |
| 11301 | } | | |
| 11302 | | | |
| 11303 | if (info.packed_offset.host_size != 0) { | | |
| 11304 | const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8)); | | |
| 11305 | assert(ordering == .none); | | |
| 11306 | const containing_int = | | |
| 11307 | try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, ""); | | |
| 11308 | const elem_bits = ptr_ty.childType(zcu).bitSize(zcu); | | |
| 11309 | const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset); | | |
| 11310 | // Convert to equally-sized integer type in order to perform the bit | | |
| 11311 | // operations on the value to store | | |
| 11312 | const value_bits_type = try o.builder.intType(@intCast(elem_bits)); | | |
| 11313 | const value_bits = if (elem_ty.isPtrAtRuntime(zcu)) | | |
| 11314 | try self.wip.cast(.ptrtoint, elem, value_bits_type, "") | | |
| 11315 | else | | |
| 11316 | try self.wip.cast(.bitcast, elem, value_bits_type, ""); | | |
| 11317 | | | |
| 11318 | const mask_val = blk: { | | |
| 11319 | const zext = try self.wip.cast( | | |
| 11320 | .zext, | | |
| 11321 | try o.builder.intValue(value_bits_type, -1), | | |
| 11322 | containing_int_ty, | | |
| 11323 | "", | | |
| 11324 | ); | | |
| 11325 | const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), ""); | | |
| 11326 | break :blk try self.wip.bin( | | |
| 11327 | .xor, | | |
| 11328 | shl, | | |
| 11329 | try o.builder.intValue(containing_int_ty, -1), | | |
| 11330 | "", | | |
| 11331 | ); | | |
| 11332 | }; | | |
| 11333 | | | |
| 11334 | const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, ""); | | |
| 11335 | const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, ""); | | |
| 11336 | const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), ""); | | |
| 11337 | const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, ""); | | |
| 11338 | | | |
| 11339 | assert(ordering == .none); | | |
| 11340 | _ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment); | | |
| 11341 | return; | | |
| 11342 | } | | |
| 11343 | if (!isByRef(elem_ty, zcu)) { | | |
| 11344 | _ = try self.wip.storeAtomic( | | |
| 11345 | access_kind, | | |
| 11346 | elem, | | |
| 11347 | ptr, | | |
| 11348 | self.sync_scope, | | |
| 11349 | ordering, | | |
| 11350 | ptr_alignment, | | |
| 11351 | ); | | |
| 11352 | return; | | |
| 11353 | } | | |
| 11354 | assert(ordering == .none); | | |
| 11355 | _ = try self.wip.callMemCpy( | | |
| 11356 | ptr, | | |
| 11357 | ptr_alignment, | | |
| 11358 | elem, | | |
| 11359 | elem_ty.abiAlignment(zcu).toLlvm(), | | |
| 11360 | try o.builder.intValue(try o.lowerType(pt, Type.usize), elem_ty.abiSize(zcu)), | | |
| 11361 | access_kind, | | |
| 11362 | self.disable_intrinsics, | | |
| 11363 | ); | | |
| 11364 | } | | |
| 11365 | | | |
| 11366 | fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void { | | |
| 11367 | const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545; | | |
| 11368 | const o = fg.ng.object; | | |
| 11369 | const pt = fg.ng.pt; | | |
| 11370 | const usize_ty = try o.lowerType(pt, Type.usize); | | |
| 11371 | const zero = try o.builder.intValue(usize_ty, 0); | | |
| 11372 | const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED); | | |
| 11373 | const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, ""); | | |
| 11374 | _ = try valgrindClientRequest(fg, zero, req, ptr_as_usize, len, zero, zero, zero); | | |
| 11375 | } | | |
| 11376 | | | |
| 11377 | fn valgrindClientRequest( | | |
| 11378 | fg: *FuncGen, | | |
| 11379 | default_value: Builder.Value, | | |
| 11380 | request: Builder.Value, | | |
| 11381 | a1: Builder.Value, | | |
| 11382 | a2: Builder.Value, | | |
| 11383 | a3: Builder.Value, | | |
| 11384 | a4: Builder.Value, | | |
| 11385 | a5: Builder.Value, | | |
| 11386 | ) Allocator.Error!Builder.Value { | | |
| 11387 | const o = fg.ng.object; | | |
| 11388 | const pt = fg.ng.pt; | | |
| 11389 | const zcu = pt.zcu; | | |
| 11390 | const target = zcu.getTarget(); | | |
| 11391 | if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value; | | |
| 11392 | | | |
| 11393 | const llvm_usize = try o.lowerType(pt, Type.usize); | | |
| 11394 | const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm(); | | |
| 11395 | | | |
| 11396 | const array_llvm_ty = try o.builder.arrayType(6, llvm_usize); | | |
| 11397 | const array_ptr = if (fg.valgrind_client_request_array == .none) a: { | | |
| 11398 | const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment); | | |
| 11399 | fg.valgrind_client_request_array = array_ptr; | | |
| 11400 | break :a array_ptr; | | |
| 11401 | } else fg.valgrind_client_request_array; | | |
| 11402 | const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 }; | | |
| 11403 | const zero = try o.builder.intValue(llvm_usize, 0); | | |
| 11404 | for (array_elements, 0..) |elem, i| { | | |
| 11405 | const elem_ptr = try fg.wip.gep(.inbounds, array_llvm_ty, array_ptr, &.{ | | |
| 11406 | zero, try o.builder.intValue(llvm_usize, i), | | |
| 11407 | }, ""); | | |
| 11408 | _ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment); | | |
| 11409 | } | | |
| 11410 | | | |
| 11411 | const arch_specific: struct { | | |
| 11412 | template: [:0]const u8, | | |
| 11413 | constraints: [:0]const u8, | | |
| 11414 | } = switch (target.cpu.arch) { | | |
| 11415 | .arm, .armeb, .thumb, .thumbeb => .{ | | |
| 11416 | .template = | | |
| 11417 | \\ mov r12, r12, ror #3 ; mov r12, r12, ror #13 | | |
| 11418 | \\ mov r12, r12, ror #29 ; mov r12, r12, ror #19 | | |
| 11419 | \\ orr r10, r10, r10 | | |
| 11420 | , | | |
| 11421 | .constraints = "={r3},{r4},{r3},~{cc},~{memory}", | | |
| 11422 | }, | | |
| 11423 | .aarch64, .aarch64_be => .{ | | |
| 11424 | .template = | | |
| 11425 | \\ ror x12, x12, #3 ; ror x12, x12, #13 | | |
| 11426 | \\ ror x12, x12, #51 ; ror x12, x12, #61 | | |
| 11427 | \\ orr x10, x10, x10 | | |
| 11428 | , | | |
| 11429 | .constraints = "={x3},{x4},{x3},~{cc},~{memory}", | | |
| 11430 | }, | | |
| 11431 | .mips, .mipsel => .{ | | |
| 11432 | .template = | | |
| 11433 | \\ srl $$0, $$0, 13 | | |
| 11434 | \\ srl $$0, $$0, 29 | | |
| 11435 | \\ srl $$0, $$0, 3 | | |
| 11436 | \\ srl $$0, $$0, 19 | | |
| 11437 | \\ or $$13, $$13, $$13 | | |
| 11438 | , | | |
| 11439 | .constraints = "={$11},{$12},{$11},~{memory},~{$1}", | | |
| 11440 | }, | | |
| 11441 | .mips64, .mips64el => .{ | | |
| 11442 | .template = | | |
| 11443 | \\ dsll $$0, $$0, 3 ; dsll $$0, $$0, 13 | | |
| 11444 | \\ dsll $$0, $$0, 29 ; dsll $$0, $$0, 19 | | |
| 11445 | \\ or $$13, $$13, $$13 | | |
| 11446 | , | | |
| 11447 | .constraints = "={$11},{$12},{$11},~{memory},~{$1}", | | |
| 11448 | }, | | |
| 11449 | .powerpc, .powerpcle => .{ | | |
| 11450 | .template = | | |
| 11451 | \\ rlwinm 0, 0, 3, 0, 31 ; rlwinm 0, 0, 13, 0, 31 | | |
| 11452 | \\ rlwinm 0, 0, 29, 0, 31 ; rlwinm 0, 0, 19, 0, 31 | | |
| 11453 | \\ or 1, 1, 1 | | |
| 11454 | , | | |
| 11455 | .constraints = "={r3},{r4},{r3},~{cc},~{memory}", | | |
| 11456 | }, | | |
| 11457 | .powerpc64, .powerpc64le => .{ | | |
| 11458 | .template = | | |
| 11459 | \\ rotldi 0, 0, 3 ; rotldi 0, 0, 13 | | |
| 11460 | \\ rotldi 0, 0, 61 ; rotldi 0, 0, 51 | | |
| 11461 | \\ or 1, 1, 1 | | |
| 11462 | , | | |
| 11463 | .constraints = "={r3},{r4},{r3},~{cc},~{memory}", | | |
| 11464 | }, | | |
| 11465 | .riscv64 => .{ | | |
| 11466 | .template = | | |
| 11467 | \\ .option push | | |
| 11468 | \\ .option norvc | | |
| 11469 | \\ srli zero, zero, 3 | | |
| 11470 | \\ srli zero, zero, 13 | | |
| 11471 | \\ srli zero, zero, 51 | | |
| 11472 | \\ srli zero, zero, 61 | | |
| 11473 | \\ or a0, a0, a0 | | |
| 11474 | \\ .option pop | | |
| 11475 | , | | |
| 11476 | .constraints = "={a3},{a4},{a3},~{cc},~{memory}", | | |
| 11477 | }, | | |
| 11478 | .s390x => .{ | | |
| 11479 | .template = | | |
| 11480 | \\ lr %r15, %r15 | | |
| 11481 | \\ lr %r1, %r1 | | |
| 11482 | \\ lr %r2, %r2 | | |
| 11483 | \\ lr %r3, %r3 | | |
| 11484 | \\ lr %r2, %r2 | | |
| 11485 | , | | |
| 11486 | .constraints = "={r3},{r2},{r3},~{cc},~{memory}", | | |
| 11487 | }, | | |
| 11488 | .x86 => .{ | | |
| 11489 | .template = | | |
| 11490 | \\ roll $$3, %edi ; roll $$13, %edi | | |
| 11491 | \\ roll $$61, %edi ; roll $$51, %edi | | |
| 11492 | \\ xchgl %ebx, %ebx | | |
| 11493 | , | | |
| 11494 | .constraints = "={edx},{eax},{edx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}", | | |
| 11495 | }, | | |
| 11496 | .x86_64 => .{ | | |
| 11497 | .template = | | |
| 11498 | \\ rolq $$3, %rdi ; rolq $$13, %rdi | | |
| 11499 | \\ rolq $$61, %rdi ; rolq $$51, %rdi | | |
| 11500 | \\ xchgq %rbx, %rbx | | |
| 11501 | , | | |
| 11502 | .constraints = "={rdx},{rax},{rdx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}", | | |
| 11503 | }, | | |
| 11504 | else => unreachable, | | |
| 11505 | }; | | |
| 11506 | | | |
| 11507 | return fg.wip.callAsm( | | |
| 11508 | .none, | | |
| 11509 | try o.builder.fnType(llvm_usize, &.{ llvm_usize, llvm_usize }, .normal), | | |
| 11510 | .{ .sideeffect = true }, | | |
| 11511 | try o.builder.string(arch_specific.template), | | |
| 11512 | try o.builder.string(arch_specific.constraints), | | |
| 11513 | &.{ try fg.wip.cast(.ptrtoint, array_ptr, llvm_usize, ""), default_value }, | | |
| 11514 | "", | | |
| 11515 | ); | | |
| 11516 | } | | |
| 11517 | | | |
| 11518 | fn typeOf(fg: *FuncGen, inst: Air.Inst.Ref) Type { | | |
| 11519 | const zcu = fg.ng.pt.zcu; | | |
| 11520 | return fg.air.typeOf(inst, &zcu.intern_pool); | | |
| 11521 | } | | |
| 11522 | | | |
| 11523 | fn typeOfIndex(fg: *FuncGen, inst: Air.Inst.Index) Type { | | |
| 11524 | const zcu = fg.ng.pt.zcu; | | |
| 11525 | return fg.air.typeOfIndex(inst, &zcu.intern_pool); | | |
| 11526 | } | | |
| 11527 | }; | | |
| 11528 | | | |
| 11529 | fn toLlvmAtomicOrdering(atomic_order: std.builtin.AtomicOrder) Builder.AtomicOrdering { | | |
| 11530 | return switch (atomic_order) { | | |
| 11531 | .unordered => .unordered, | | |
| 11532 | .monotonic => .monotonic, | | |
| 11533 | .acquire => .acquire, | | |
| 11534 | .release => .release, | | |
| 11535 | .acq_rel => .acq_rel, | | |
| 11536 | .seq_cst => .seq_cst, | | |
| 11537 | }; | | |
| 11538 | } | | |
| 11539 | | | |
| 11540 | fn toLlvmAtomicRmwBinOp( | | |
| 11541 | op: std.builtin.AtomicRmwOp, | | |
| 11542 | is_signed: bool, | | |
| 11543 | is_float: bool, | | |
| 11544 | ) Builder.Function.Instruction.AtomicRmw.Operation { | | |
| 11545 | return switch (op) { | | |
| 11546 | .Xchg => .xchg, | | |
| 11547 | .Add => if (is_float) .fadd else return .add, | | |
| 11548 | .Sub => if (is_float) .fsub else return .sub, | | |
| 11549 | .And => .@"and", | | |
| 11550 | .Nand => .nand, | | |
| 11551 | .Or => .@"or", | | |
| 11552 | .Xor => .xor, | | |
| 11553 | .Max => if (is_float) .fmax else if (is_signed) .max else return .umax, | | |
| 11554 | .Min => if (is_float) .fmin else if (is_signed) .min else return .umin, | | |
| 11555 | }; | | |
| 11556 | } | | |
| 11557 | | | |
| 11558 | const CallingConventionInfo = struct { | | |
| 11559 | /// The LLVM calling convention to use. | | |
| 11560 | llvm_cc: Builder.CallConv, | | |
| 11561 | /// Whether to use an `alignstack` attribute to forcibly re-align the stack pointer in the function's prologue. | | |
| 11562 | align_stack: bool, | | |
| 11563 | /// Whether the function needs a `naked` attribute. | | |
| 11564 | naked: bool, | | |
| 11565 | /// How many leading parameters to apply the `inreg` attribute to. | | |
| 11566 | inreg_param_count: u2 = 0, | | |
| 11567 | }; | | |
| 11568 | | | |
| 11569 | pub fn toLlvmCallConv(cc: std.builtin.CallingConvention, target: *const std.Target) ?CallingConventionInfo { | | |
| 11570 | const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null; | | |
| 11571 | const incoming_stack_alignment: ?u64, const register_params: u2 = switch (cc) { | | |
| 11572 | inline else => |pl| switch (@TypeOf(pl)) { | | |
| 11573 | void => .{ null, 0 }, | | |
| 11574 | std.builtin.CallingConvention.ArcInterruptOptions, | | |
| 11575 | std.builtin.CallingConvention.ArmInterruptOptions, | | |
| 11576 | std.builtin.CallingConvention.RiscvInterruptOptions, | | |
| 11577 | std.builtin.CallingConvention.ShInterruptOptions, | | |
| 11578 | std.builtin.CallingConvention.MicroblazeInterruptOptions, | | |
| 11579 | std.builtin.CallingConvention.MipsInterruptOptions, | | |
| 11580 | std.builtin.CallingConvention.CommonOptions, | | |
| 11581 | => .{ pl.incoming_stack_alignment, 0 }, | | |
| 11582 | std.builtin.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params }, | | |
| 11583 | else => @compileError("TODO: toLlvmCallConv" ++ @tagName(pl)), | | |
| 11584 | }, | | |
| 11585 | }; | | |
| 11586 | return .{ | | |
| 11587 | .llvm_cc = llvm_cc, | | |
| 11588 | .align_stack = if (incoming_stack_alignment) |a| need_align: { | | |
| 11589 | const normal_stack_align = target.stackAlignment(); | | |
| 11590 | break :need_align a < normal_stack_align; | | |
| 11591 | } else false, | | |
| 11592 | .naked = cc == .naked, | | |
| 11593 | .inreg_param_count = register_params, | | |
| 11594 | }; | | |
| 11595 | } | | |
| 11596 | fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv { | | |
| 11597 | if (target.cCallingConvention()) |default_c| { | | |
| 11598 | if (cc_tag == default_c) { | | |
| 11599 | return .ccc; | | |
| 11600 | } | | |
| 11601 | } | | |
| 11602 | return switch (cc_tag) { | | |
| 11603 | .@"inline" => unreachable, | | |
| 11604 | .auto, .async => .fastcc, | | |
| 11605 | .naked => .ccc, | | |
| 11606 | .x86_64_sysv => .x86_64_sysvcc, | | |
| 11607 | .x86_64_win => .win64cc, | | |
| 11608 | .x86_64_regcall_v3_sysv => if (target.cpu.arch == .x86_64 and target.os.tag != .windows) | | |
| 11609 | .x86_regcallcc | | |
| 11610 | else | | |
| 11611 | null, | | |
| 11612 | .x86_64_regcall_v4_win => if (target.cpu.arch == .x86_64 and target.os.tag == .windows) | | |
| 11613 | .x86_regcallcc // we use the "RegCallv4" module flag to make this correct | | |
| 11614 | else | | |
| 11615 | null, | | |
| 11616 | .x86_64_vectorcall => .x86_vectorcallcc, | | |
| 11617 | .x86_64_interrupt => .x86_intrcc, | | |
| 11618 | .x86_stdcall => .x86_stdcallcc, | | |
| 11619 | .x86_fastcall => .x86_fastcallcc, | | |
| 11620 | .x86_thiscall => .x86_thiscallcc, | | |
| 11621 | .x86_regcall_v3 => if (target.cpu.arch == .x86 and target.os.tag != .windows) | | |
| 11622 | .x86_regcallcc | | |
| 11623 | else | | |
| 11624 | null, | | |
| 11625 | .x86_regcall_v4_win => if (target.cpu.arch == .x86 and target.os.tag == .windows) | | |
| 11626 | .x86_regcallcc // we use the "RegCallv4" module flag to make this correct | | |
| 11627 | else | | |
| 11628 | null, | | |
| 11629 | .x86_vectorcall => .x86_vectorcallcc, | | |
| 11630 | .x86_interrupt => .x86_intrcc, | | |
| 11631 | .aarch64_vfabi => .aarch64_vector_pcs, | | |
| 11632 | .aarch64_vfabi_sve => .aarch64_sve_vector_pcs, | | |
| 11633 | .arm_aapcs => .arm_aapcscc, | | |
| 11634 | .arm_aapcs_vfp => .arm_aapcs_vfpcc, | | |
| 11635 | .riscv64_lp64_v => .riscv_vectorcallcc, | | |
| 11636 | .riscv32_ilp32_v => .riscv_vectorcallcc, | | |
| 11637 | .avr_builtin => .avr_builtincc, | | |
| 11638 | .avr_signal => .avr_signalcc, | | |
| 11639 | .avr_interrupt => .avr_intrcc, | | |
| 11640 | .m68k_rtd => .m68k_rtdcc, | | |
| 11641 | .m68k_interrupt => .m68k_intrcc, | | |
| 11642 | .msp430_interrupt => .msp430_intrcc, | | |
| 11643 | .amdgcn_kernel => .amdgpu_kernel, | | |
| 11644 | .amdgcn_cs => .amdgpu_cs, | | |
| 11645 | .nvptx_device => .ptx_device, | | |
| 11646 | .nvptx_kernel => .ptx_kernel, | | |
| 11647 | | | |
| 11648 | // Calling conventions which LLVM uses function attributes for. | | |
| 11649 | .riscv64_interrupt, | | |
| 11650 | .riscv32_interrupt, | | |
| 11651 | .arm_interrupt, | | |
| 11652 | .mips64_interrupt, | | |
| 11653 | .mips_interrupt, | | |
| 11654 | .csky_interrupt, | | |
| 11655 | => .ccc, | | |
| 11656 | | | |
| 11657 | // All the calling conventions which LLVM does not have a general representation for. | | |
| 11658 | // Note that these are often still supported through the `cCallingConvention` path above via `ccc`. | | |
| 11659 | .x86_16_cdecl, | | |
| 11660 | .x86_16_stdcall, | | |
| 11661 | .x86_16_regparmcall, | | |
| 11662 | .x86_16_interrupt, | | |
| 11663 | .x86_sysv, | | |
| 11664 | .x86_win, | | |
| 11665 | .x86_thiscall_mingw, | | |
| 11666 | .x86_64_x32, | | |
| 11667 | .aarch64_aapcs, | | |
| 11668 | .aarch64_aapcs_darwin, | | |
| 11669 | .aarch64_aapcs_win, | | |
| 11670 | .alpha_osf, | | |
| 11671 | .microblaze_std, | | |
| 11672 | .microblaze_interrupt, | | |
| 11673 | .mips64_n64, | | |
| 11674 | .mips64_n32, | | |
| 11675 | .mips_o32, | | |
| 11676 | .riscv64_lp64, | | |
| 11677 | .riscv32_ilp32, | | |
| 11678 | .sparc64_sysv, | | |
| 11679 | .sparc_sysv, | | |
| 11680 | .powerpc64_elf, | | |
| 11681 | .powerpc64_elf_altivec, | | |
| 11682 | .powerpc64_elf_v2, | | |
| 11683 | .powerpc_sysv, | | |
| 11684 | .powerpc_sysv_altivec, | | |
| 11685 | .powerpc_aix, | | |
| 11686 | .powerpc_aix_altivec, | | |
| 11687 | .wasm_mvp, | | |
| 11688 | .arc_sysv, | | |
| 11689 | .arc_interrupt, | | |
| 11690 | .avr_gnu, | | |
| 11691 | .bpf_std, | | |
| 11692 | .csky_sysv, | | |
| 11693 | .hexagon_sysv, | | |
| 11694 | .hexagon_sysv_hvx, | | |
| 11695 | .hppa_elf, | | |
| 11696 | .hppa64_elf, | | |
| 11697 | .kvx_lp64, | | |
| 11698 | .kvx_ilp32, | | |
| 11699 | .lanai_sysv, | | |
| 11700 | .loongarch64_lp64, | | |
| 11701 | .loongarch32_ilp32, | | |
| 11702 | .m68k_sysv, | | |
| 11703 | .m68k_gnu, | | |
| 11704 | .msp430_eabi, | | |
| 11705 | .or1k_sysv, | | |
| 11706 | .propeller_sysv, | | |
| 11707 | .s390x_sysv, | | |
| 11708 | .s390x_sysv_vx, | | |
| 11709 | .sh_gnu, | | |
| 11710 | .sh_renesas, | | |
| 11711 | .sh_interrupt, | | |
| 11712 | .ve_sysv, | | |
| 11713 | .xcore_xs1, | | |
| 11714 | .xcore_xs2, | | |
| 11715 | .xtensa_call0, | | |
| 11716 | .xtensa_windowed, | | |
| 11717 | .amdgcn_device, | | |
| 11718 | .spirv_device, | | |
| 11719 | .spirv_kernel, | | |
| 11720 | .spirv_fragment, | | |
| 11721 | .spirv_vertex, | | |
| 11722 | => null, | | |
| 11723 | }; | | |
| 11724 | } | | |
| 11725 | | | |
| 11726 | /// Convert a zig-address space to an llvm address space. | | |
| 11727 | fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace { | | |
| 11728 | for (llvmAddrSpaceInfo(target)) |info| if (info.zig == address_space) return info.llvm; | | |
| 11729 | unreachable; | | |
| 11730 | } | | |
| 11731 | | | |
| 11732 | const AddrSpaceInfo = struct { | | |
| 11733 | zig: ?std.builtin.AddressSpace, | | |
| 11734 | llvm: Builder.AddrSpace, | | |
| 11735 | non_integral: bool = false, | | |
| 11736 | size: ?u16 = null, | | |
| 11737 | abi: ?u16 = null, | | |
| 11738 | pref: ?u16 = null, | | |
| 11739 | idx: ?u16 = null, | | |
| 11740 | force_in_data_layout: bool = false, | | |
| 11741 | }; | | |
| 11742 | fn llvmAddrSpaceInfo(target: *const std.Target) []const AddrSpaceInfo { | | |
| 11743 | return switch (target.cpu.arch) { | | |
| 11744 | .x86, .x86_64 => &.{ | | |
| 11745 | .{ .zig = .generic, .llvm = .default }, | | |
| 11746 | .{ .zig = .gs, .llvm = Builder.AddrSpace.x86.gs }, | | |
| 11747 | .{ .zig = .fs, .llvm = Builder.AddrSpace.x86.fs }, | | |
| 11748 | .{ .zig = .ss, .llvm = Builder.AddrSpace.x86.ss }, | | |
| 11749 | .{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_sptr, .size = 32, .abi = 32, .force_in_data_layout = true }, | | |
| 11750 | .{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_uptr, .size = 32, .abi = 32, .force_in_data_layout = true }, | | |
| 11751 | .{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr64, .size = 64, .abi = 64, .force_in_data_layout = true }, | | |
| 11752 | }, | | |
| 11753 | .nvptx, .nvptx64 => &.{ | | |
| 11754 | .{ .zig = .generic, .llvm = Builder.AddrSpace.nvptx.generic }, | | |
| 11755 | .{ .zig = .global, .llvm = Builder.AddrSpace.nvptx.global }, | | |
| 11756 | .{ .zig = .constant, .llvm = Builder.AddrSpace.nvptx.constant }, | | |
| 11757 | .{ .zig = .param, .llvm = Builder.AddrSpace.nvptx.param }, | | |
| 11758 | .{ .zig = .shared, .llvm = Builder.AddrSpace.nvptx.shared }, | | |
| 11759 | .{ .zig = .local, .llvm = Builder.AddrSpace.nvptx.local }, | | |
| 11760 | }, | | |
| 11761 | .amdgcn => &.{ | | |
| 11762 | .{ .zig = .generic, .llvm = Builder.AddrSpace.amdgpu.flat, .force_in_data_layout = true }, | | |
| 11763 | .{ .zig = .global, .llvm = Builder.AddrSpace.amdgpu.global, .force_in_data_layout = true }, | | |
| 11764 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.region, .size = 32, .abi = 32 }, | | |
| 11765 | .{ .zig = .shared, .llvm = Builder.AddrSpace.amdgpu.local, .size = 32, .abi = 32 }, | | |
| 11766 | .{ .zig = .constant, .llvm = Builder.AddrSpace.amdgpu.constant, .force_in_data_layout = true }, | | |
| 11767 | .{ .zig = .local, .llvm = Builder.AddrSpace.amdgpu.private, .size = 32, .abi = 32 }, | | |
| 11768 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_32bit, .size = 32, .abi = 32 }, | | |
| 11769 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_fat_pointer, .non_integral = true, .size = 160, .abi = 256, .idx = 32 }, | | |
| 11770 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_resource, .non_integral = true, .size = 128, .abi = 128 }, | | |
| 11771 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_strided_pointer, .non_integral = true, .size = 192, .abi = 256, .idx = 32 }, | | |
| 11772 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_0 }, | | |
| 11773 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_1 }, | | |
| 11774 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_2 }, | | |
| 11775 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_3 }, | | |
| 11776 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_4 }, | | |
| 11777 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_5 }, | | |
| 11778 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_6 }, | | |
| 11779 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_7 }, | | |
| 11780 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_8 }, | | |
| 11781 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_9 }, | | |
| 11782 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_10 }, | | |
| 11783 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_11 }, | | |
| 11784 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_12 }, | | |
| 11785 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_13 }, | | |
| 11786 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_14 }, | | |
| 11787 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_15 }, | | |
| 11788 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.streamout_register }, | | |
| 11789 | }, | | |
| 11790 | .avr => &.{ | | |
| 11791 | .{ .zig = .generic, .llvm = Builder.AddrSpace.avr.data, .abi = 8 }, | | |
| 11792 | .{ .zig = .flash, .llvm = Builder.AddrSpace.avr.program, .abi = 8 }, | | |
| 11793 | .{ .zig = .flash1, .llvm = Builder.AddrSpace.avr.program1, .abi = 8 }, | | |
| 11794 | .{ .zig = .flash2, .llvm = Builder.AddrSpace.avr.program2, .abi = 8 }, | | |
| 11795 | .{ .zig = .flash3, .llvm = Builder.AddrSpace.avr.program3, .abi = 8 }, | | |
| 11796 | .{ .zig = .flash4, .llvm = Builder.AddrSpace.avr.program4, .abi = 8 }, | | |
| 11797 | .{ .zig = .flash5, .llvm = Builder.AddrSpace.avr.program5, .abi = 8 }, | | |
| 11798 | }, | | |
| 11799 | .wasm32, .wasm64 => &.{ | | |
| 11800 | .{ .zig = .generic, .llvm = Builder.AddrSpace.wasm.default, .force_in_data_layout = true }, | | |
| 11801 | .{ .zig = null, .llvm = Builder.AddrSpace.wasm.variable, .non_integral = true }, | | |
| 11802 | .{ .zig = null, .llvm = Builder.AddrSpace.wasm.externref, .non_integral = true, .size = 8, .abi = 8 }, | | |
| 11803 | .{ .zig = null, .llvm = Builder.AddrSpace.wasm.funcref, .non_integral = true, .size = 8, .abi = 8 }, | | |
| 11804 | }, | | |
| 11805 | .m68k => &.{ | | |
| 11806 | .{ .zig = .generic, .llvm = .default, .abi = 16, .pref = 32 }, | | |
| 11807 | }, | | |
| 11808 | else => &.{ | | |
| 11809 | .{ .zig = .generic, .llvm = .default }, | | |
| 11810 | }, | 4401 | }, |
| 11811 | }; | 4402 | }; |
| 11812 | } | 4403 | return .{ |
| 11813 | | 4404 | .llvm_cc = llvm_cc, |
| 11814 | /// On some targets, local values that are in the generic address space must be generated into a | 4405 | .align_stack = if (incoming_stack_alignment) |a| need_align: { |
| 11815 | /// different address, space and then cast back to the generic address space. | 4406 | const normal_stack_align = target.stackAlignment(); |
| 11816 | /// For example, on GPUs local variable declarations must be generated into the local address space. | 4407 | break :need_align a < normal_stack_align; |
| 11817 | /// This function returns the address space local values should be generated into. | 4408 | } else false, |
| 11818 | fn llvmAllocaAddressSpace(target: *const std.Target) Builder.AddrSpace { | 4409 | .naked = cc == .naked, |
| 11819 | return switch (target.cpu.arch) { | 4410 | .inreg_param_count = register_params, |
| 11820 | // On amdgcn, locals should be generated into the private address space. | | |
| 11821 | // To make Zig not impossible to use, these are then converted to addresses in the | | |
| 11822 | // generic address space and treates as regular pointers. This is the way that HIP also does it. | | |
| 11823 | .amdgcn => Builder.AddrSpace.amdgpu.private, | | |
| 11824 | else => .default, | | |
| 11825 | }; | | |
| 11826 | } | | |
| 11827 | | | |
| 11828 | /// On some targets, global values that are in the generic address space must be generated into a | | |
| 11829 | /// different address space, and then cast back to the generic address space. | | |
| 11830 | fn llvmDefaultGlobalAddressSpace(target: *const std.Target) Builder.AddrSpace { | | |
| 11831 | return switch (target.cpu.arch) { | | |
| 11832 | // On amdgcn, globals must be explicitly allocated and uploaded so that the program can access | | |
| 11833 | // them. | | |
| 11834 | .amdgcn => Builder.AddrSpace.amdgpu.global, | | |
| 11835 | else => .default, | | |
| 11836 | }; | | |
| 11837 | } | | |
| 11838 | | | |
| 11839 | /// Return the actual address space that a value should be stored in if its a global address space. | | |
| 11840 | /// When a value is placed in the resulting address space, it needs to be cast back into wanted_address_space. | | |
| 11841 | fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace { | | |
| 11842 | return switch (wanted_address_space) { | | |
| 11843 | .generic => llvmDefaultGlobalAddressSpace(target), | | |
| 11844 | else => |as| toLlvmAddressSpace(as, target), | | |
| 11845 | }; | | |
| 11846 | } | | |
| 11847 | | | |
| 11848 | fn returnTypeByRef(zcu: *Zcu, target: *const std.Target, ty: Type) bool { | | |
| 11849 | if (isByRef(ty, zcu)) { | | |
| 11850 | return true; | | |
| 11851 | } else if (target.cpu.arch.isX86() and | | |
| 11852 | !target.cpu.has(.x86, .evex512) and | | |
| 11853 | ty.totalVectorBits(zcu) >= 512) | | |
| 11854 | { | | |
| 11855 | // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns | | |
| 11856 | // "512-bit vector arguments require 'evex512' for AVX512" | | |
| 11857 | return true; | | |
| 11858 | } else { | | |
| 11859 | return false; | | |
| 11860 | } | | |
| 11861 | } | | |
| 11862 | | | |
| 11863 | fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: *Zcu, target: *const std.Target) bool { | | |
| 11864 | const return_type = Type.fromInterned(fn_info.return_type); | | |
| 11865 | if (!return_type.hasRuntimeBits(zcu)) return false; | | |
| 11866 | | | |
| 11867 | return switch (fn_info.cc) { | | |
| 11868 | .auto => returnTypeByRef(zcu, target, return_type), | | |
| 11869 | .x86_64_sysv => firstParamSRetSystemV(return_type, zcu, target), | | |
| 11870 | .x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory, | | |
| 11871 | .x86_sysv, .x86_win => isByRef(return_type, zcu), | | |
| 11872 | .x86_stdcall => !isScalar(zcu, return_type), | | |
| 11873 | .wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect, | | |
| 11874 | .aarch64_aapcs, | | |
| 11875 | .aarch64_aapcs_darwin, | | |
| 11876 | .aarch64_aapcs_win, | | |
| 11877 | => aarch64_c_abi.classifyType(return_type, zcu) == .memory, | | |
| 11878 | .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) { | | |
| 11879 | .memory, .i64_array => true, | | |
| 11880 | .i32_array => |size| size != 1, | | |
| 11881 | .byval => false, | | |
| 11882 | }, | | |
| 11883 | .riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory, | | |
| 11884 | .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) { | | |
| 11885 | .memory, .i32_array => true, | | |
| 11886 | .byval => false, | | |
| 11887 | }, | | |
| 11888 | else => false, // TODO: investigate other targets/callconvs | | |
| 11889 | }; | 4411 | }; |
| 11890 | } | 4412 | } |
| 11891 | | 4413 | pub fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv { |
| 11892 | fn firstParamSRetSystemV(ty: Type, zcu: *Zcu, target: *const std.Target) bool { | 4414 | if (target.cCallingConvention()) |default_c| { |
| 11893 | const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret); | 4415 | if (cc_tag == default_c) { |
| 11894 | if (class[0] == .memory) return true; | 4416 | return .ccc; |
| 11895 | if (class[0] == .x87 and class[2] != .none) return true; | | |
| 11896 | return false; | | |
| 11897 | } | | |
| 11898 | | | |
| 11899 | /// In order to support the C calling convention, some return types need to be lowered | | |
| 11900 | /// completely differently in the function prototype to honor the C ABI, and then | | |
| 11901 | /// be effectively bitcasted to the actual return type. | | |
| 11902 | fn lowerFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { | | |
| 11903 | const zcu = pt.zcu; | | |
| 11904 | const return_type = Type.fromInterned(fn_info.return_type); | | |
| 11905 | if (!return_type.hasRuntimeBits(zcu)) { | | |
| 11906 | assert(!return_type.isError(zcu)); | | |
| 11907 | return .void; | | |
| 11908 | } | | |
| 11909 | const target = zcu.getTarget(); | | |
| 11910 | switch (fn_info.cc) { | | |
| 11911 | .@"inline" => unreachable, | | |
| 11912 | .auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(pt, return_type), | | |
| 11913 | | | |
| 11914 | .x86_64_sysv => return lowerSystemVFnRetTy(o, pt, fn_info), | | |
| 11915 | .x86_64_win => return lowerWin64FnRetTy(o, pt, fn_info), | | |
| 11916 | .x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(pt, return_type) else .void, | | |
| 11917 | .x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(pt, return_type), | | |
| 11918 | .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) { | | |
| 11919 | .memory => return .void, | | |
| 11920 | .float_array => return o.lowerType(pt, return_type), | | |
| 11921 | .byval => return o.lowerType(pt, return_type), | | |
| 11922 | .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))), | | |
| 11923 | .double_integer => return o.builder.arrayType(2, .i64), | | |
| 11924 | }, | | |
| 11925 | .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) { | | |
| 11926 | .memory, .i64_array => return .void, | | |
| 11927 | .i32_array => |len| return if (len == 1) .i32 else .void, | | |
| 11928 | .byval => return o.lowerType(pt, return_type), | | |
| 11929 | }, | | |
| 11930 | .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) { | | |
| 11931 | .memory, .i32_array => return .void, | | |
| 11932 | .byval => return o.lowerType(pt, return_type), | | |
| 11933 | }, | | |
| 11934 | .riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) { | | |
| 11935 | .memory => return .void, | | |
| 11936 | .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))), | | |
| 11937 | .double_integer => { | | |
| 11938 | const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) { | | |
| 11939 | .riscv64, .riscv64be => .i64, | | |
| 11940 | .riscv32, .riscv32be => .i32, | | |
| 11941 | else => unreachable, | | |
| 11942 | }; | | |
| 11943 | return o.builder.structType(.normal, &.{ integer, integer }); | | |
| 11944 | }, | | |
| 11945 | .byval => return o.lowerType(pt, return_type), | | |
| 11946 | .fields => { | | |
| 11947 | var types_len: usize = 0; | | |
| 11948 | var types: [8]Builder.Type = undefined; | | |
| 11949 | for (0..return_type.structFieldCount(zcu)) |field_index| { | | |
| 11950 | const field_ty = return_type.fieldType(field_index, zcu); | | |
| 11951 | if (!field_ty.hasRuntimeBits(zcu)) continue; | | |
| 11952 | types[types_len] = try o.lowerType(pt, field_ty); | | |
| 11953 | types_len += 1; | | |
| 11954 | } | | |
| 11955 | return o.builder.structType(.normal, types[0..types_len]); | | |
| 11956 | }, | | |
| 11957 | }, | | |
| 11958 | .wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) { | | |
| 11959 | .direct => |scalar_ty| return o.lowerType(pt, scalar_ty), | | |
| 11960 | .indirect => return .void, | | |
| 11961 | }, | | |
| 11962 | // TODO investigate other callconvs | | |
| 11963 | else => return o.lowerType(pt, return_type), | | |
| 11964 | } | | |
| 11965 | } | | |
| 11966 | | | |
| 11967 | fn lowerWin64FnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { | | |
| 11968 | const zcu = pt.zcu; | | |
| 11969 | const return_type = Type.fromInterned(fn_info.return_type); | | |
| 11970 | switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) { | | |
| 11971 | .integer => { | | |
| 11972 | if (isScalar(zcu, return_type)) { | | |
| 11973 | return o.lowerType(pt, return_type); | | |
| 11974 | } else { | | |
| 11975 | return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8)); | | |
| 11976 | } | | |
| 11977 | }, | | |
| 11978 | .win_i128 => return o.builder.vectorType(.normal, 2, .i64), | | |
| 11979 | .memory => return .void, | | |
| 11980 | .sse => return o.lowerType(pt, return_type), | | |
| 11981 | else => unreachable, | | |
| 11982 | } | | |
| 11983 | } | | |
| 11984 | | | |
| 11985 | fn lowerSystemVFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { | | |
| 11986 | const zcu = pt.zcu; | | |
| 11987 | const ip = &zcu.intern_pool; | | |
| 11988 | const return_type = Type.fromInterned(fn_info.return_type); | | |
| 11989 | return_type.assertHasLayout(zcu); | | |
| 11990 | if (isScalar(zcu, return_type)) { | | |
| 11991 | return o.lowerType(pt, return_type); | | |
| 11992 | } | | |
| 11993 | const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret); | | |
| 11994 | var types_index: u32 = 0; | | |
| 11995 | var types_buffer: [8]Builder.Type = undefined; | | |
| 11996 | for (classes) |class| { | | |
| 11997 | switch (class) { | | |
| 11998 | .integer => { | | |
| 11999 | types_buffer[types_index] = .i64; | | |
| 12000 | types_index += 1; | | |
| 12001 | }, | | |
| 12002 | .sse => { | | |
| 12003 | types_buffer[types_index] = .double; | | |
| 12004 | types_index += 1; | | |
| 12005 | }, | | |
| 12006 | .sseup => { | | |
| 12007 | if (types_buffer[types_index - 1] == .double) { | | |
| 12008 | types_buffer[types_index - 1] = .fp128; | | |
| 12009 | } else { | | |
| 12010 | types_buffer[types_index] = .double; | | |
| 12011 | types_index += 1; | | |
| 12012 | } | | |
| 12013 | }, | | |
| 12014 | .float => { | | |
| 12015 | types_buffer[types_index] = .float; | | |
| 12016 | types_index += 1; | | |
| 12017 | }, | | |
| 12018 | .float_combine => { | | |
| 12019 | types_buffer[types_index] = try o.builder.vectorType(.normal, 2, .float); | | |
| 12020 | types_index += 1; | | |
| 12021 | }, | | |
| 12022 | .x87 => { | | |
| 12023 | if (types_index != 0 or classes[2] != .none) return .void; | | |
| 12024 | types_buffer[types_index] = .x86_fp80; | | |
| 12025 | types_index += 1; | | |
| 12026 | }, | | |
| 12027 | .x87up => continue, | | |
| 12028 | .none => break, | | |
| 12029 | .memory, .integer_per_element => return .void, | | |
| 12030 | .win_i128 => unreachable, // windows only | | |
| 12031 | } | | |
| 12032 | } | | |
| 12033 | const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer}); | | |
| 12034 | if (first_non_integer == null or classes[first_non_integer.?] == .none) { | | |
| 12035 | assert(first_non_integer orelse classes.len == types_index); | | |
| 12036 | switch (ip.indexToKey(return_type.toIntern())) { | | |
| 12037 | .struct_type => { | | |
| 12038 | const size = return_type.abiSize(zcu); | | |
| 12039 | assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index); | | |
| 12040 | if (size % 8 > 0) { | | |
| 12041 | types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8)); | | |
| 12042 | } | | |
| 12043 | }, | | |
| 12044 | else => {}, | | |
| 12045 | } | | |
| 12046 | if (types_index == 1) return types_buffer[0]; | | |
| 12047 | } | | |
| 12048 | return o.builder.structType(.normal, types_buffer[0..types_index]); | | |
| 12049 | } | | |
| 12050 | | | |
| 12051 | const ParamTypeIterator = struct { | | |
| 12052 | object: *Object, | | |
| 12053 | pt: Zcu.PerThread, | | |
| 12054 | fn_info: InternPool.Key.FuncType, | | |
| 12055 | zig_index: u32, | | |
| 12056 | llvm_index: u32, | | |
| 12057 | types_len: u32, | | |
| 12058 | types_buffer: [8]Builder.Type, | | |
| 12059 | byval_attr: bool, | | |
| 12060 | | | |
| 12061 | const Lowering = union(enum) { | | |
| 12062 | no_bits, | | |
| 12063 | byval, | | |
| 12064 | byref, | | |
| 12065 | byref_mut, | | |
| 12066 | abi_sized_int, | | |
| 12067 | multiple_llvm_types, | | |
| 12068 | slice, | | |
| 12069 | float_array: u8, | | |
| 12070 | i32_array: u8, | | |
| 12071 | i64_array: u8, | | |
| 12072 | }; | | |
| 12073 | | | |
| 12074 | fn next(it: *ParamTypeIterator) Allocator.Error!?Lowering { | | |
| 12075 | if (it.zig_index >= it.fn_info.param_types.len) return null; | | |
| 12076 | const ip = &it.pt.zcu.intern_pool; | | |
| 12077 | const ty = it.fn_info.param_types.get(ip)[it.zig_index]; | | |
| 12078 | it.byval_attr = false; | | |
| 12079 | return nextInner(it, Type.fromInterned(ty)); | | |
| 12080 | } | | |
| 12081 | | | |
| 12082 | /// `airCall` uses this instead of `next` so that it can take into account variadic functions. | | |
| 12083 | fn nextCall(it: *ParamTypeIterator, fg: *FuncGen, args: []const Air.Inst.Ref) Allocator.Error!?Lowering { | | |
| 12084 | assert(std.meta.eql(it.pt, fg.ng.pt)); | | |
| 12085 | const ip = &it.pt.zcu.intern_pool; | | |
| 12086 | if (it.zig_index >= it.fn_info.param_types.len) { | | |
| 12087 | if (it.zig_index >= args.len) { | | |
| 12088 | return null; | | |
| 12089 | } else { | | |
| 12090 | return nextInner(it, fg.typeOf(args[it.zig_index])); | | |
| 12091 | } | | |
| 12092 | } else { | | |
| 12093 | return nextInner(it, Type.fromInterned(it.fn_info.param_types.get(ip)[it.zig_index])); | | |
| 12094 | } | | |
| 12095 | } | | |
| 12096 | | | |
| 12097 | fn nextInner(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering { | | |
| 12098 | const pt = it.pt; | | |
| 12099 | const zcu = pt.zcu; | | |
| 12100 | const target = zcu.getTarget(); | | |
| 12101 | | | |
| 12102 | if (!ty.hasRuntimeBits(zcu)) { | | |
| 12103 | it.zig_index += 1; | | |
| 12104 | return .no_bits; | | |
| 12105 | } | | |
| 12106 | switch (it.fn_info.cc) { | | |
| 12107 | .@"inline" => unreachable, | | |
| 12108 | .auto => { | | |
| 12109 | it.zig_index += 1; | | |
| 12110 | it.llvm_index += 1; | | |
| 12111 | if (ty.isSlice(zcu) or | | |
| 12112 | (ty.zigTypeTag(zcu) == .optional and ty.optionalChild(zcu).isSlice(zcu) and !ty.ptrAllowsZero(zcu))) | | |
| 12113 | { | | |
| 12114 | it.llvm_index += 1; | | |
| 12115 | return .slice; | | |
| 12116 | } else if (isByRef(ty, zcu)) { | | |
| 12117 | return .byref; | | |
| 12118 | } else if (target.cpu.arch.isX86() and | | |
| 12119 | !target.cpu.has(.x86, .evex512) and | | |
| 12120 | ty.totalVectorBits(zcu) >= 512) | | |
| 12121 | { | | |
| 12122 | // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns | | |
| 12123 | // "512-bit vector arguments require 'evex512' for AVX512" | | |
| 12124 | return .byref; | | |
| 12125 | } else { | | |
| 12126 | return .byval; | | |
| 12127 | } | | |
| 12128 | }, | | |
| 12129 | .async => { | | |
| 12130 | @panic("TODO implement async function lowering in the LLVM backend"); | | |
| 12131 | }, | | |
| 12132 | .x86_64_sysv => return it.nextSystemV(ty), | | |
| 12133 | .x86_64_win => return it.nextWin64(ty), | | |
| 12134 | .x86_stdcall => { | | |
| 12135 | it.zig_index += 1; | | |
| 12136 | it.llvm_index += 1; | | |
| 12137 | | | |
| 12138 | if (isScalar(zcu, ty)) { | | |
| 12139 | return .byval; | | |
| 12140 | } else { | | |
| 12141 | it.byval_attr = true; | | |
| 12142 | return .byref; | | |
| 12143 | } | | |
| 12144 | }, | | |
| 12145 | .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => { | | |
| 12146 | it.zig_index += 1; | | |
| 12147 | it.llvm_index += 1; | | |
| 12148 | switch (aarch64_c_abi.classifyType(ty, zcu)) { | | |
| 12149 | .memory => return .byref_mut, | | |
| 12150 | .float_array => |len| return Lowering{ .float_array = len }, | | |
| 12151 | .byval => return .byval, | | |
| 12152 | .integer => { | | |
| 12153 | it.types_len = 1; | | |
| 12154 | it.types_buffer[0] = .i64; | | |
| 12155 | return .multiple_llvm_types; | | |
| 12156 | }, | | |
| 12157 | .double_integer => return Lowering{ .i64_array = 2 }, | | |
| 12158 | } | | |
| 12159 | }, | | |
| 12160 | .arm_aapcs, .arm_aapcs_vfp => { | | |
| 12161 | it.zig_index += 1; | | |
| 12162 | it.llvm_index += 1; | | |
| 12163 | switch (arm_c_abi.classifyType(ty, zcu, .arg)) { | | |
| 12164 | .memory => { | | |
| 12165 | it.byval_attr = true; | | |
| 12166 | return .byref; | | |
| 12167 | }, | | |
| 12168 | .byval => return .byval, | | |
| 12169 | .i32_array => |size| return Lowering{ .i32_array = size }, | | |
| 12170 | .i64_array => |size| return Lowering{ .i64_array = size }, | | |
| 12171 | } | | |
| 12172 | }, | | |
| 12173 | .mips_o32 => { | | |
| 12174 | it.zig_index += 1; | | |
| 12175 | it.llvm_index += 1; | | |
| 12176 | switch (mips_c_abi.classifyType(ty, zcu, .arg)) { | | |
| 12177 | .memory => { | | |
| 12178 | it.byval_attr = true; | | |
| 12179 | return .byref; | | |
| 12180 | }, | | |
| 12181 | .byval => return .byval, | | |
| 12182 | .i32_array => |size| return Lowering{ .i32_array = size }, | | |
| 12183 | } | | |
| 12184 | }, | | |
| 12185 | .riscv64_lp64, .riscv32_ilp32 => { | | |
| 12186 | it.zig_index += 1; | | |
| 12187 | it.llvm_index += 1; | | |
| 12188 | switch (riscv_c_abi.classifyType(ty, zcu)) { | | |
| 12189 | .memory => return .byref_mut, | | |
| 12190 | .byval => return .byval, | | |
| 12191 | .integer => return .abi_sized_int, | | |
| 12192 | .double_integer => return Lowering{ .i64_array = 2 }, | | |
| 12193 | .fields => { | | |
| 12194 | it.types_len = 0; | | |
| 12195 | for (0..ty.structFieldCount(zcu)) |field_index| { | | |
| 12196 | const field_ty = ty.fieldType(field_index, zcu); | | |
| 12197 | if (!field_ty.hasRuntimeBits(zcu)) continue; | | |
| 12198 | it.types_buffer[it.types_len] = try it.object.lowerType(pt, field_ty); | | |
| 12199 | it.types_len += 1; | | |
| 12200 | } | | |
| 12201 | it.llvm_index += it.types_len - 1; | | |
| 12202 | return .multiple_llvm_types; | | |
| 12203 | }, | | |
| 12204 | } | | |
| 12205 | }, | | |
| 12206 | .wasm_mvp => switch (wasm_c_abi.classifyType(ty, zcu)) { | | |
| 12207 | .direct => |scalar_ty| { | | |
| 12208 | if (isScalar(zcu, ty)) { | | |
| 12209 | it.zig_index += 1; | | |
| 12210 | it.llvm_index += 1; | | |
| 12211 | return .byval; | | |
| 12212 | } else { | | |
| 12213 | var types_buffer: [8]Builder.Type = undefined; | | |
| 12214 | types_buffer[0] = try it.object.lowerType(pt, scalar_ty); | | |
| 12215 | it.types_buffer = types_buffer; | | |
| 12216 | it.types_len = 1; | | |
| 12217 | it.llvm_index += 1; | | |
| 12218 | it.zig_index += 1; | | |
| 12219 | return .multiple_llvm_types; | | |
| 12220 | } | | |
| 12221 | }, | | |
| 12222 | .indirect => { | | |
| 12223 | it.zig_index += 1; | | |
| 12224 | it.llvm_index += 1; | | |
| 12225 | it.byval_attr = true; | | |
| 12226 | return .byref; | | |
| 12227 | }, | | |
| 12228 | }, | | |
| 12229 | // TODO investigate other callconvs | | |
| 12230 | else => { | | |
| 12231 | it.zig_index += 1; | | |
| 12232 | it.llvm_index += 1; | | |
| 12233 | return .byval; | | |
| 12234 | }, | | |
| 12235 | } | | |
| 12236 | } | | |
| 12237 | | | |
| 12238 | fn nextWin64(it: *ParamTypeIterator, ty: Type) ?Lowering { | | |
| 12239 | const zcu = it.pt.zcu; | | |
| 12240 | switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget(), .arg)) { | | |
| 12241 | .integer => { | | |
| 12242 | if (isScalar(zcu, ty)) { | | |
| 12243 | it.zig_index += 1; | | |
| 12244 | it.llvm_index += 1; | | |
| 12245 | return .byval; | | |
| 12246 | } else { | | |
| 12247 | it.zig_index += 1; | | |
| 12248 | it.llvm_index += 1; | | |
| 12249 | return .abi_sized_int; | | |
| 12250 | } | | |
| 12251 | }, | | |
| 12252 | .win_i128 => { | | |
| 12253 | it.zig_index += 1; | | |
| 12254 | it.llvm_index += 1; | | |
| 12255 | return .byref; | | |
| 12256 | }, | | |
| 12257 | .memory => { | | |
| 12258 | it.zig_index += 1; | | |
| 12259 | it.llvm_index += 1; | | |
| 12260 | return .byref_mut; | | |
| 12261 | }, | | |
| 12262 | .sse => { | | |
| 12263 | it.zig_index += 1; | | |
| 12264 | it.llvm_index += 1; | | |
| 12265 | return .byval; | | |
| 12266 | }, | | |
| 12267 | else => unreachable, | | |
| 12268 | } | 4417 | } |
| 12269 | } | 4418 | } |
| | 4419 | return switch (cc_tag) { |
| | 4420 | .@"inline" => unreachable, |
| | 4421 | .auto, .async => .fastcc, |
| | 4422 | .naked => .ccc, |
| | 4423 | .x86_64_sysv => .x86_64_sysvcc, |
| | 4424 | .x86_64_win => .win64cc, |
| | 4425 | .x86_64_regcall_v3_sysv => if (target.cpu.arch == .x86_64 and target.os.tag != .windows) |
| | 4426 | .x86_regcallcc |
| | 4427 | else |
| | 4428 | null, |
| | 4429 | .x86_64_regcall_v4_win => if (target.cpu.arch == .x86_64 and target.os.tag == .windows) |
| | 4430 | .x86_regcallcc // we use the "RegCallv4" module flag to make this correct |
| | 4431 | else |
| | 4432 | null, |
| | 4433 | .x86_64_vectorcall => .x86_vectorcallcc, |
| | 4434 | .x86_64_interrupt => .x86_intrcc, |
| | 4435 | .x86_stdcall => .x86_stdcallcc, |
| | 4436 | .x86_fastcall => .x86_fastcallcc, |
| | 4437 | .x86_thiscall => .x86_thiscallcc, |
| | 4438 | .x86_regcall_v3 => if (target.cpu.arch == .x86 and target.os.tag != .windows) |
| | 4439 | .x86_regcallcc |
| | 4440 | else |
| | 4441 | null, |
| | 4442 | .x86_regcall_v4_win => if (target.cpu.arch == .x86 and target.os.tag == .windows) |
| | 4443 | .x86_regcallcc // we use the "RegCallv4" module flag to make this correct |
| | 4444 | else |
| | 4445 | null, |
| | 4446 | .x86_vectorcall => .x86_vectorcallcc, |
| | 4447 | .x86_interrupt => .x86_intrcc, |
| | 4448 | .aarch64_vfabi => .aarch64_vector_pcs, |
| | 4449 | .aarch64_vfabi_sve => .aarch64_sve_vector_pcs, |
| | 4450 | .arm_aapcs => .arm_aapcscc, |
| | 4451 | .arm_aapcs_vfp => .arm_aapcs_vfpcc, |
| | 4452 | .riscv64_lp64_v => .riscv_vectorcallcc, |
| | 4453 | .riscv32_ilp32_v => .riscv_vectorcallcc, |
| | 4454 | .avr_builtin => .avr_builtincc, |
| | 4455 | .avr_signal => .avr_signalcc, |
| | 4456 | .avr_interrupt => .avr_intrcc, |
| | 4457 | .m68k_rtd => .m68k_rtdcc, |
| | 4458 | .m68k_interrupt => .m68k_intrcc, |
| | 4459 | .msp430_interrupt => .msp430_intrcc, |
| | 4460 | .amdgcn_kernel => .amdgpu_kernel, |
| | 4461 | .amdgcn_cs => .amdgpu_cs, |
| | 4462 | .nvptx_device => .ptx_device, |
| | 4463 | .nvptx_kernel => .ptx_kernel, |
| 12270 | | 4464 | |
| 12271 | fn nextSystemV(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering { | 4465 | // Calling conventions which LLVM uses function attributes for. |
| 12272 | const zcu = it.pt.zcu; | 4466 | .riscv64_interrupt, |
| 12273 | const ip = &zcu.intern_pool; | 4467 | .riscv32_interrupt, |
| 12274 | ty.assertHasLayout(zcu); | 4468 | .arm_interrupt, |
| 12275 | const classes = x86_64_abi.classifySystemV(ty, zcu, zcu.getTarget(), .arg); | 4469 | .mips64_interrupt, |
| 12276 | if (classes[0] == .memory) { | 4470 | .mips_interrupt, |
| 12277 | it.zig_index += 1; | 4471 | .csky_interrupt, |
| 12278 | it.llvm_index += 1; | 4472 | => .ccc, |
| 12279 | it.byval_attr = true; | | |
| 12280 | return .byref; | | |
| 12281 | } | | |
| 12282 | if (isScalar(zcu, ty)) { | | |
| 12283 | it.zig_index += 1; | | |
| 12284 | it.llvm_index += 1; | | |
| 12285 | return .byval; | | |
| 12286 | } | | |
| 12287 | var types_index: u32 = 0; | | |
| 12288 | var types_buffer: [8]Builder.Type = undefined; | | |
| 12289 | for (classes) |class| { | | |
| 12290 | switch (class) { | | |
| 12291 | .integer => { | | |
| 12292 | types_buffer[types_index] = .i64; | | |
| 12293 | types_index += 1; | | |
| 12294 | }, | | |
| 12295 | .sse => { | | |
| 12296 | types_buffer[types_index] = .double; | | |
| 12297 | types_index += 1; | | |
| 12298 | }, | | |
| 12299 | .sseup => { | | |
| 12300 | if (types_buffer[types_index - 1] == .double) { | | |
| 12301 | types_buffer[types_index - 1] = .fp128; | | |
| 12302 | } else { | | |
| 12303 | types_buffer[types_index] = .double; | | |
| 12304 | types_index += 1; | | |
| 12305 | } | | |
| 12306 | }, | | |
| 12307 | .float => { | | |
| 12308 | types_buffer[types_index] = .float; | | |
| 12309 | types_index += 1; | | |
| 12310 | }, | | |
| 12311 | .float_combine => { | | |
| 12312 | types_buffer[types_index] = try it.object.builder.vectorType(.normal, 2, .float); | | |
| 12313 | types_index += 1; | | |
| 12314 | }, | | |
| 12315 | .x87 => { | | |
| 12316 | it.zig_index += 1; | | |
| 12317 | it.llvm_index += 1; | | |
| 12318 | it.byval_attr = true; | | |
| 12319 | return .byref; | | |
| 12320 | }, | | |
| 12321 | .x87up => unreachable, | | |
| 12322 | .none => break, | | |
| 12323 | .memory => unreachable, // handled above | | |
| 12324 | .win_i128 => unreachable, // windows only | | |
| 12325 | .integer_per_element => { | | |
| 12326 | @panic("TODO"); | | |
| 12327 | }, | | |
| 12328 | } | | |
| 12329 | } | | |
| 12330 | const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer}); | | |
| 12331 | if (first_non_integer == null or classes[first_non_integer.?] == .none) { | | |
| 12332 | assert(first_non_integer orelse classes.len == types_index); | | |
| 12333 | if (types_index == 1) { | | |
| 12334 | it.zig_index += 1; | | |
| 12335 | it.llvm_index += 1; | | |
| 12336 | return .abi_sized_int; | | |
| 12337 | } | | |
| 12338 | if (it.llvm_index + types_index > 6) { | | |
| 12339 | it.zig_index += 1; | | |
| 12340 | it.llvm_index += 1; | | |
| 12341 | it.byval_attr = true; | | |
| 12342 | return .byref; | | |
| 12343 | } | | |
| 12344 | switch (ip.indexToKey(ty.toIntern())) { | | |
| 12345 | .struct_type => { | | |
| 12346 | const size = ty.abiSize(zcu); | | |
| 12347 | assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index); | | |
| 12348 | if (size % 8 > 0) { | | |
| 12349 | types_buffer[types_index - 1] = | | |
| 12350 | try it.object.builder.intType(@intCast(size % 8 * 8)); | | |
| 12351 | } | | |
| 12352 | }, | | |
| 12353 | else => {}, | | |
| 12354 | } | | |
| 12355 | } | | |
| 12356 | it.types_len = types_index; | | |
| 12357 | it.types_buffer = types_buffer; | | |
| 12358 | it.llvm_index += types_index; | | |
| 12359 | it.zig_index += 1; | | |
| 12360 | return .multiple_llvm_types; | | |
| 12361 | } | | |
| 12362 | }; | | |
| 12363 | | 4473 | |
| 12364 | fn iterateParamTypes(object: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) ParamTypeIterator { | 4474 | // All the calling conventions which LLVM does not have a general representation for. |
| 12365 | return .{ | 4475 | // Note that these are often still supported through the `cCallingConvention` path above via `ccc`. |
| 12366 | .object = object, | 4476 | .x86_16_cdecl, |
| 12367 | .pt = pt, | 4477 | .x86_16_stdcall, |
| 12368 | .fn_info = fn_info, | 4478 | .x86_16_regparmcall, |
| 12369 | .zig_index = 0, | 4479 | .x86_16_interrupt, |
| 12370 | .llvm_index = 0, | 4480 | .x86_sysv, |
| 12371 | .types_len = 0, | 4481 | .x86_win, |
| 12372 | .types_buffer = undefined, | 4482 | .x86_thiscall_mingw, |
| 12373 | .byval_attr = false, | 4483 | .x86_64_x32, |
| | 4484 | .aarch64_aapcs, |
| | 4485 | .aarch64_aapcs_darwin, |
| | 4486 | .aarch64_aapcs_win, |
| | 4487 | .alpha_osf, |
| | 4488 | .microblaze_std, |
| | 4489 | .microblaze_interrupt, |
| | 4490 | .mips64_n64, |
| | 4491 | .mips64_n32, |
| | 4492 | .mips_o32, |
| | 4493 | .riscv64_lp64, |
| | 4494 | .riscv32_ilp32, |
| | 4495 | .sparc64_sysv, |
| | 4496 | .sparc_sysv, |
| | 4497 | .powerpc64_elf, |
| | 4498 | .powerpc64_elf_altivec, |
| | 4499 | .powerpc64_elf_v2, |
| | 4500 | .powerpc_sysv, |
| | 4501 | .powerpc_sysv_altivec, |
| | 4502 | .powerpc_aix, |
| | 4503 | .powerpc_aix_altivec, |
| | 4504 | .wasm_mvp, |
| | 4505 | .arc_sysv, |
| | 4506 | .arc_interrupt, |
| | 4507 | .avr_gnu, |
| | 4508 | .bpf_std, |
| | 4509 | .csky_sysv, |
| | 4510 | .hexagon_sysv, |
| | 4511 | .hexagon_sysv_hvx, |
| | 4512 | .hppa_elf, |
| | 4513 | .hppa64_elf, |
| | 4514 | .kvx_lp64, |
| | 4515 | .kvx_ilp32, |
| | 4516 | .lanai_sysv, |
| | 4517 | .loongarch64_lp64, |
| | 4518 | .loongarch32_ilp32, |
| | 4519 | .m68k_sysv, |
| | 4520 | .m68k_gnu, |
| | 4521 | .msp430_eabi, |
| | 4522 | .or1k_sysv, |
| | 4523 | .propeller_sysv, |
| | 4524 | .s390x_sysv, |
| | 4525 | .s390x_sysv_vx, |
| | 4526 | .sh_gnu, |
| | 4527 | .sh_renesas, |
| | 4528 | .sh_interrupt, |
| | 4529 | .ve_sysv, |
| | 4530 | .xcore_xs1, |
| | 4531 | .xcore_xs2, |
| | 4532 | .xtensa_call0, |
| | 4533 | .xtensa_windowed, |
| | 4534 | .amdgcn_device, |
| | 4535 | .spirv_device, |
| | 4536 | .spirv_kernel, |
| | 4537 | .spirv_fragment, |
| | 4538 | .spirv_vertex, |
| | 4539 | => null, |
| 12374 | }; | 4540 | }; |
| 12375 | } | 4541 | } |
| 12376 | | 4542 | |
| 12377 | /// This function deliberately does not handle `_BitInt` because it typically | 4543 | /// Convert a zig-address space to an llvm address space. |
| 12378 | /// has different ABI than regular integer types, and there is no currently no | 4544 | pub fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace { |
| 12379 | /// way to determine whether a Zig integer type is meant to represent e.g. `int` | 4545 | for (llvmAddrSpaceInfo(target)) |info| if (info.zig == address_space) return info.llvm; |
| 12380 | /// or `_BitInt(32)`. | 4546 | unreachable; |
| 12381 | fn ccAbiPromoteInt(cc: std.builtin.CallingConvention, zcu: *Zcu, ty: Type) ?std.builtin.Signedness { | 4547 | } |
| 12382 | switch (cc) { | | |
| 12383 | .auto, .@"inline", .async => return null, | | |
| 12384 | else => {}, | | |
| 12385 | } | | |
| 12386 | | | |
| 12387 | const int_info = switch (ty.zigTypeTag(zcu)) { | | |
| 12388 | .bool => Type.u1.intInfo(zcu), | | |
| 12389 | else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return null, | | |
| 12390 | }; | | |
| 12391 | assert(int_info.bits >= 0); | | |
| 12392 | | 4548 | |
| 12393 | const target = zcu.getTarget(); | 4549 | const AddrSpaceInfo = struct { |
| | 4550 | zig: ?std.builtin.AddressSpace, |
| | 4551 | llvm: Builder.AddrSpace, |
| | 4552 | non_integral: bool = false, |
| | 4553 | size: ?u16 = null, |
| | 4554 | abi: ?u16 = null, |
| | 4555 | pref: ?u16 = null, |
| | 4556 | idx: ?u16 = null, |
| | 4557 | force_in_data_layout: bool = false, |
| | 4558 | }; |
| | 4559 | fn llvmAddrSpaceInfo(target: *const std.Target) []const AddrSpaceInfo { |
| 12394 | return switch (target.cpu.arch) { | 4560 | return switch (target.cpu.arch) { |
| 12395 | .aarch64, | 4561 | .x86, .x86_64 => &.{ |
| 12396 | .aarch64_be, | 4562 | .{ .zig = .generic, .llvm = .default }, |
| 12397 | => switch (target.os.tag) { | 4563 | .{ .zig = .gs, .llvm = Builder.AddrSpace.x86.gs }, |
| 12398 | .driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => switch (int_info.bits) { | 4564 | .{ .zig = .fs, .llvm = Builder.AddrSpace.x86.fs }, |
| 12399 | 8, 16 => int_info.signedness, | 4565 | .{ .zig = .ss, .llvm = Builder.AddrSpace.x86.ss }, |
| 12400 | else => null, | 4566 | .{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_sptr, .size = 32, .abi = 32, .force_in_data_layout = true }, |
| 12401 | }, | 4567 | .{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_uptr, .size = 32, .abi = 32, .force_in_data_layout = true }, |
| 12402 | else => null, | 4568 | .{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr64, .size = 64, .abi = 64, .force_in_data_layout = true }, |
| 12403 | }, | | |
| 12404 | | | |
| 12405 | .avr, | | |
| 12406 | => switch (int_info.bits) { | | |
| 12407 | 8 => int_info.signedness, | | |
| 12408 | else => null, | | |
| 12409 | }, | | |
| 12410 | | | |
| 12411 | .lanai, | | |
| 12412 | => null, | | |
| 12413 | | | |
| 12414 | .loongarch64, | | |
| 12415 | .riscv64, | | |
| 12416 | .riscv64be, | | |
| 12417 | => switch (int_info.bits) { | | |
| 12418 | 8, 16 => int_info.signedness, | | |
| 12419 | 32 => .signed, | | |
| 12420 | else => null, | | |
| 12421 | }, | | |
| 12422 | | | |
| 12423 | .mips, | | |
| 12424 | .mipsel, | | |
| 12425 | .mips64, | | |
| 12426 | .mips64el, | | |
| 12427 | => switch (int_info.bits) { | | |
| 12428 | 8, 16, 64 => int_info.signedness, | | |
| 12429 | // https://github.com/llvm/llvm-project/issues/179088 | | |
| 12430 | // 32 => .signed, | | |
| 12431 | else => null, | | |
| 12432 | }, | 4569 | }, |
| 12433 | | 4570 | .nvptx, .nvptx64 => &.{ |
| 12434 | .powerpc64, | 4571 | .{ .zig = .generic, .llvm = Builder.AddrSpace.nvptx.generic }, |
| 12435 | .powerpc64le, | 4572 | .{ .zig = .global, .llvm = Builder.AddrSpace.nvptx.global }, |
| 12436 | .s390x, | 4573 | .{ .zig = .constant, .llvm = Builder.AddrSpace.nvptx.constant }, |
| 12437 | .sparc64, | 4574 | .{ .zig = .param, .llvm = Builder.AddrSpace.nvptx.param }, |
| 12438 | .ve, | 4575 | .{ .zig = .shared, .llvm = Builder.AddrSpace.nvptx.shared }, |
| 12439 | => switch (int_info.bits) { | 4576 | .{ .zig = .local, .llvm = Builder.AddrSpace.nvptx.local }, |
| 12440 | 8, 16, 32 => int_info.signedness, | | |
| 12441 | else => null, | | |
| 12442 | }, | 4577 | }, |
| 12443 | | 4578 | .amdgcn => &.{ |
| 12444 | else => switch (int_info.bits) { | 4579 | .{ .zig = .generic, .llvm = Builder.AddrSpace.amdgpu.flat, .force_in_data_layout = true }, |
| 12445 | 8, 16 => int_info.signedness, | 4580 | .{ .zig = .global, .llvm = Builder.AddrSpace.amdgpu.global, .force_in_data_layout = true }, |
| 12446 | else => null, | 4581 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.region, .size = 32, .abi = 32 }, |
| | 4582 | .{ .zig = .shared, .llvm = Builder.AddrSpace.amdgpu.local, .size = 32, .abi = 32 }, |
| | 4583 | .{ .zig = .constant, .llvm = Builder.AddrSpace.amdgpu.constant, .force_in_data_layout = true }, |
| | 4584 | .{ .zig = .local, .llvm = Builder.AddrSpace.amdgpu.private, .size = 32, .abi = 32 }, |
| | 4585 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_32bit, .size = 32, .abi = 32 }, |
| | 4586 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_fat_pointer, .non_integral = true, .size = 160, .abi = 256, .idx = 32 }, |
| | 4587 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_resource, .non_integral = true, .size = 128, .abi = 128 }, |
| | 4588 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_strided_pointer, .non_integral = true, .size = 192, .abi = 256, .idx = 32 }, |
| | 4589 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_0 }, |
| | 4590 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_1 }, |
| | 4591 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_2 }, |
| | 4592 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_3 }, |
| | 4593 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_4 }, |
| | 4594 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_5 }, |
| | 4595 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_6 }, |
| | 4596 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_7 }, |
| | 4597 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_8 }, |
| | 4598 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_9 }, |
| | 4599 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_10 }, |
| | 4600 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_11 }, |
| | 4601 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_12 }, |
| | 4602 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_13 }, |
| | 4603 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_14 }, |
| | 4604 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_15 }, |
| | 4605 | .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.streamout_register }, |
| 12447 | }, | 4606 | }, |
| 12448 | }; | 4607 | .avr => &.{ |
| 12449 | } | 4608 | .{ .zig = .generic, .llvm = Builder.AddrSpace.avr.data, .abi = 8 }, |
| 12450 | | 4609 | .{ .zig = .flash, .llvm = Builder.AddrSpace.avr.program, .abi = 8 }, |
| 12451 | /// This is the one source of truth for whether a type is passed around as an LLVM pointer, | 4610 | .{ .zig = .flash1, .llvm = Builder.AddrSpace.avr.program1, .abi = 8 }, |
| 12452 | /// or as an LLVM value. | 4611 | .{ .zig = .flash2, .llvm = Builder.AddrSpace.avr.program2, .abi = 8 }, |
| 12453 | fn isByRef(ty: Type, zcu: *Zcu) bool { | 4612 | .{ .zig = .flash3, .llvm = Builder.AddrSpace.avr.program3, .abi = 8 }, |
| 12454 | // For tuples and structs, if there are more than this many non-void | 4613 | .{ .zig = .flash4, .llvm = Builder.AddrSpace.avr.program4, .abi = 8 }, |
| 12455 | // fields, then we make it byref, otherwise byval. | 4614 | .{ .zig = .flash5, .llvm = Builder.AddrSpace.avr.program5, .abi = 8 }, |
| 12456 | const max_fields_byval = 0; | | |
| 12457 | const ip = &zcu.intern_pool; | | |
| 12458 | | | |
| 12459 | switch (ty.zigTypeTag(zcu)) { | | |
| 12460 | .type, | | |
| 12461 | .comptime_int, | | |
| 12462 | .comptime_float, | | |
| 12463 | .enum_literal, | | |
| 12464 | .undefined, | | |
| 12465 | .null, | | |
| 12466 | .@"opaque", | | |
| 12467 | => unreachable, | | |
| 12468 | | | |
| 12469 | .noreturn, | | |
| 12470 | .void, | | |
| 12471 | .bool, | | |
| 12472 | .int, | | |
| 12473 | .float, | | |
| 12474 | .pointer, | | |
| 12475 | .error_set, | | |
| 12476 | .@"fn", | | |
| 12477 | .@"enum", | | |
| 12478 | .vector, | | |
| 12479 | .@"anyframe", | | |
| 12480 | => return false, | | |
| 12481 | | | |
| 12482 | .array, .frame => return ty.hasRuntimeBits(zcu), | | |
| 12483 | .@"struct" => { | | |
| 12484 | const struct_type = switch (ip.indexToKey(ty.toIntern())) { | | |
| 12485 | .tuple_type => |tuple| { | | |
| 12486 | var count: usize = 0; | | |
| 12487 | for (tuple.types.get(ip), tuple.values.get(ip)) |field_ty, field_val| { | | |
| 12488 | if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue; | | |
| 12489 | | | |
| 12490 | count += 1; | | |
| 12491 | if (count > max_fields_byval) return true; | | |
| 12492 | if (isByRef(Type.fromInterned(field_ty), zcu)) return true; | | |
| 12493 | } | | |
| 12494 | return false; | | |
| 12495 | }, | | |
| 12496 | .struct_type => ip.loadStructType(ty.toIntern()), | | |
| 12497 | else => unreachable, | | |
| 12498 | }; | | |
| 12499 | | | |
| 12500 | // Packed structs are represented to LLVM as integers. | | |
| 12501 | if (struct_type.layout == .@"packed") return false; | | |
| 12502 | | | |
| 12503 | const field_types = struct_type.field_types.get(ip); | | |
| 12504 | var it = struct_type.iterateRuntimeOrder(ip); | | |
| 12505 | var count: usize = 0; | | |
| 12506 | while (it.next()) |field_index| { | | |
| 12507 | count += 1; | | |
| 12508 | if (count > max_fields_byval) return true; | | |
| 12509 | const field_ty = Type.fromInterned(field_types[field_index]); | | |
| 12510 | if (isByRef(field_ty, zcu)) return true; | | |
| 12511 | } | | |
| 12512 | return false; | | |
| 12513 | }, | 4615 | }, |
| 12514 | .@"union" => switch (ty.containerLayout(zcu)) { | 4616 | .wasm32, .wasm64 => &.{ |
| 12515 | .@"packed" => return false, | 4617 | .{ .zig = .generic, .llvm = Builder.AddrSpace.wasm.default, .force_in_data_layout = true }, |
| 12516 | else => return ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu), | 4618 | .{ .zig = null, .llvm = Builder.AddrSpace.wasm.variable, .non_integral = true }, |
| | 4619 | .{ .zig = null, .llvm = Builder.AddrSpace.wasm.externref, .non_integral = true, .size = 8, .abi = 8 }, |
| | 4620 | .{ .zig = null, .llvm = Builder.AddrSpace.wasm.funcref, .non_integral = true, .size = 8, .abi = 8 }, |
| 12517 | }, | 4621 | }, |
| 12518 | .error_union => { | 4622 | .m68k => &.{ |
| 12519 | const payload_ty = ty.errorUnionPayload(zcu); | 4623 | .{ .zig = .generic, .llvm = .default, .abi = 16, .pref = 32 }, |
| 12520 | if (!payload_ty.hasRuntimeBits(zcu)) { | | |
| 12521 | return false; | | |
| 12522 | } | | |
| 12523 | return true; | | |
| 12524 | }, | 4624 | }, |
| 12525 | .optional => { | 4625 | else => &.{ |
| 12526 | const payload_ty = ty.optionalChild(zcu); | 4626 | .{ .zig = .generic, .llvm = .default }, |
| 12527 | if (!payload_ty.hasRuntimeBits(zcu)) { | | |
| 12528 | return false; | | |
| 12529 | } | | |
| 12530 | if (ty.optionalReprIsPayload(zcu)) { | | |
| 12531 | return false; | | |
| 12532 | } | | |
| 12533 | return true; | | |
| 12534 | }, | 4627 | }, |
| 12535 | } | 4628 | }; |
| 12536 | } | 4629 | } |
| 12537 | | 4630 | |
| 12538 | fn isScalar(zcu: *Zcu, ty: Type) bool { | 4631 | /// On some targets, global values that are in the generic address space must be generated into a |
| 12539 | return switch (ty.zigTypeTag(zcu)) { | 4632 | /// different address space, and then cast back to the generic address space. |
| 12540 | .void, | 4633 | fn llvmDefaultGlobalAddressSpace(target: *const std.Target) Builder.AddrSpace { |
| 12541 | .bool, | 4634 | return switch (target.cpu.arch) { |
| 12542 | .noreturn, | 4635 | // On amdgcn, globals must be explicitly allocated and uploaded so that the program can access |
| 12543 | .int, | 4636 | // them. |
| 12544 | .float, | 4637 | .amdgcn => Builder.AddrSpace.amdgpu.global, |
| 12545 | .pointer, | 4638 | else => .default, |
| 12546 | .optional, | | |
| 12547 | .error_set, | | |
| 12548 | .@"enum", | | |
| 12549 | .@"anyframe", | | |
| 12550 | .vector, | | |
| 12551 | => true, | | |
| 12552 | | | |
| 12553 | .@"struct" => ty.containerLayout(zcu) == .@"packed", | | |
| 12554 | .@"union" => ty.containerLayout(zcu) == .@"packed", | | |
| 12555 | else => false, | | |
| 12556 | }; | 4639 | }; |
| 12557 | } | 4640 | } |
| 12558 | | 4641 | |
| 12559 | /// This function returns true if we expect LLVM to lower x86_fp80 correctly | 4642 | /// Return the actual address space that a value should be stored in if its a global address space. |
| 12560 | /// and false if we expect LLVM to crash if it encounters an x86_fp80 type, | 4643 | /// When a value is placed in the resulting address space, it needs to be cast back into wanted_address_space. |
| 12561 | /// or if it produces miscompilations. | 4644 | fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace { |
| 12562 | fn backendSupportsF80(target: *const std.Target) bool { | 4645 | return switch (wanted_address_space) { |
| 12563 | return switch (target.cpu.arch) { | 4646 | .generic => llvmDefaultGlobalAddressSpace(target), |
| 12564 | .x86, .x86_64 => !target.cpu.has(.x86, .soft_float), | 4647 | else => |as| toLlvmAddressSpace(as, target), |
| 12565 | else => false, | | |
| 12566 | }; | 4648 | }; |
| 12567 | } | 4649 | } |
| 12568 | | 4650 | |
| 12569 | /// This function returns true if we expect LLVM to lower f16 correctly | 4651 | /// This function returns true if we expect LLVM to lower f16 correctly |
| 12570 | /// and false if we expect LLVM to crash if it encounters an f16 type, | 4652 | /// and false if we expect LLVM to crash if it encounters an f16 type, |
| 12571 | /// or if it produces miscompilations. | 4653 | /// or if it produces miscompilations. |
| 12572 | fn backendSupportsF16(target: *const std.Target) bool { | 4654 | pub fn backendSupportsF16(target: *const std.Target) bool { |
| 12573 | return switch (target.cpu.arch) { | 4655 | return switch (target.cpu.arch) { |
| 12574 | // https://github.com/llvm/llvm-project/issues/97981 | 4656 | // https://github.com/llvm/llvm-project/issues/97981 |
| 12575 | .csky, | 4657 | .csky, |
| ... | @@ -12594,10 +4676,20 @@ fn backendSupportsF16(target: *const std.Target) bool { | ... | @@ -12594,10 +4676,20 @@ fn backendSupportsF16(target: *const std.Target) bool { |
| 12594 | }; | 4676 | }; |
| 12595 | } | 4677 | } |
| 12596 | | 4678 | |
| | 4679 | /// This function returns true if we expect LLVM to lower x86_fp80 correctly |
| | 4680 | /// and false if we expect LLVM to crash if it encounters an x86_fp80 type, |
| | 4681 | /// or if it produces miscompilations. |
| | 4682 | pub fn backendSupportsF80(target: *const std.Target) bool { |
| | 4683 | return switch (target.cpu.arch) { |
| | 4684 | .x86, .x86_64 => !target.cpu.has(.x86, .soft_float), |
| | 4685 | else => false, |
| | 4686 | }; |
| | 4687 | } |
| | 4688 | |
| 12597 | /// This function returns true if we expect LLVM to lower f128 correctly, | 4689 | /// This function returns true if we expect LLVM to lower f128 correctly, |
| 12598 | /// and false if we expect LLVM to crash if it encounters an f128 type, | 4690 | /// and false if we expect LLVM to crash if it encounters an f128 type, |
| 12599 | /// or if it produces miscompilations. | 4691 | /// or if it produces miscompilations. |
| 12600 | fn backendSupportsF128(target: *const std.Target) bool { | 4692 | pub fn backendSupportsF128(target: *const std.Target) bool { |
| 12601 | return switch (target.cpu.arch) { | 4693 | return switch (target.cpu.arch) { |
| 12602 | // https://github.com/llvm/llvm-project/issues/121122 | 4694 | // https://github.com/llvm/llvm-project/issues/121122 |
| 12603 | .amdgcn, | 4695 | .amdgcn, |
| ... | @@ -12616,17 +4708,6 @@ fn backendSupportsF128(target: *const std.Target) bool { | ... | @@ -12616,17 +4708,6 @@ fn backendSupportsF128(target: *const std.Target) bool { |
| 12616 | }; | 4708 | }; |
| 12617 | } | 4709 | } |
| 12618 | | 4710 | |
| 12619 | /// LLVM does not support all relevant intrinsics for all targets, so we | | |
| 12620 | /// may need to manually generate a compiler-rt call. | | |
| 12621 | fn intrinsicsAllowed(scalar_ty: Type, target: *const std.Target) bool { | | |
| 12622 | return switch (scalar_ty.toIntern()) { | | |
| 12623 | .f16_type => backendSupportsF16(target), | | |
| 12624 | .f80_type => (target.cTypeBitSize(.longdouble) == 80) and backendSupportsF80(target), | | |
| 12625 | .f128_type => (target.cTypeBitSize(.longdouble) == 128) and backendSupportsF128(target), | | |
| 12626 | else => true, | | |
| 12627 | }; | | |
| 12628 | } | | |
| 12629 | | | |
| 12630 | /// We need to insert extra padding if LLVM's isn't enough. | 4711 | /// We need to insert extra padding if LLVM's isn't enough. |
| 12631 | /// However we don't want to ever call LLVMABIAlignmentOfType or | 4712 | /// However we don't want to ever call LLVMABIAlignmentOfType or |
| 12632 | /// LLVMABISizeOfType because these functions will trip assertions | 4713 | /// LLVMABISizeOfType because these functions will trip assertions |
| ... | @@ -12638,264 +4719,186 @@ const struct_layout_version = 2; | ... | @@ -12638,264 +4719,186 @@ const struct_layout_version = 2; |
| 12638 | | 4719 | |
| 12639 | // TODO: Restore the non_null field to i1 once | 4720 | // TODO: Restore the non_null field to i1 once |
| 12640 | // https://github.com/llvm/llvm-project/issues/56585/ is fixed | 4721 | // https://github.com/llvm/llvm-project/issues/56585/ is fixed |
| 12641 | const optional_layout_version = 3; | 4722 | pub const optional_layout_version = 3; |
| 12642 | | | |
| 12643 | const lt_errors_fn_name = "__zig_lt_errors_len"; | | |
| 12644 | | | |
| 12645 | fn compilerRtIntBits(bits: u16) ?u16 { | | |
| 12646 | inline for (.{ 32, 64, 128 }) |b| { | | |
| 12647 | if (bits <= b) { | | |
| 12648 | return b; | | |
| 12649 | } | | |
| 12650 | } | | |
| 12651 | return null; | | |
| 12652 | } | | |
| 12653 | | | |
| 12654 | fn buildAllocaInner( | | |
| 12655 | wip: *Builder.WipFunction, | | |
| 12656 | llvm_ty: Builder.Type, | | |
| 12657 | alignment: Builder.Alignment, | | |
| 12658 | target: *const std.Target, | | |
| 12659 | ) Allocator.Error!Builder.Value { | | |
| 12660 | const address_space = llvmAllocaAddressSpace(target); | | |
| 12661 | | | |
| 12662 | const alloca = blk: { | | |
| 12663 | const prev_cursor = wip.cursor; | | |
| 12664 | const prev_debug_location = wip.debug_location; | | |
| 12665 | defer { | | |
| 12666 | wip.cursor = prev_cursor; | | |
| 12667 | if (wip.cursor.block == .entry) wip.cursor.instruction += 1; | | |
| 12668 | wip.debug_location = prev_debug_location; | | |
| 12669 | } | | |
| 12670 | | | |
| 12671 | wip.cursor = .{ .block = .entry }; | | |
| 12672 | wip.debug_location = .no_location; | | |
| 12673 | break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, ""); | | |
| 12674 | }; | | |
| 12675 | | | |
| 12676 | // The pointer returned from this function should have the generic address space, | | |
| 12677 | // if this isn't the case then cast it to the generic address space. | | |
| 12678 | return wip.conv(.unneeded, alloca, .ptr, ""); | | |
| 12679 | } | | |
| 12680 | | | |
| 12681 | fn errUnionPayloadOffset(payload_ty: Type, pt: Zcu.PerThread) !u1 { | | |
| 12682 | const zcu = pt.zcu; | | |
| 12683 | const err_int_ty = try pt.errorIntType(); | | |
| 12684 | return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.gt, payload_ty.abiAlignment(zcu))); | | |
| 12685 | } | | |
| 12686 | | | |
| 12687 | fn errUnionErrorOffset(payload_ty: Type, pt: Zcu.PerThread) !u1 { | | |
| 12688 | const zcu = pt.zcu; | | |
| 12689 | const err_int_ty = try pt.errorIntType(); | | |
| 12690 | return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.lte, payload_ty.abiAlignment(zcu))); | | |
| 12691 | } | | |
| 12692 | | | |
| 12693 | /// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a memory location | | |
| 12694 | /// | | |
| 12695 | /// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang | | |
| 12696 | fn constraintAllowsMemory(constraint: []const u8) bool { | | |
| 12697 | // TODO: This implementation is woefully incomplete. | | |
| 12698 | for (constraint) |byte| { | | |
| 12699 | switch (byte) { | | |
| 12700 | '=', '*', ',', '&' => {}, | | |
| 12701 | 'm', 'o', 'X', 'g' => return true, | | |
| 12702 | else => {}, | | |
| 12703 | } | | |
| 12704 | } else return false; | | |
| 12705 | } | | |
| 12706 | | | |
| 12707 | /// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a register | | |
| 12708 | /// | | |
| 12709 | /// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang | | |
| 12710 | fn constraintAllowsRegister(constraint: []const u8) bool { | | |
| 12711 | // TODO: This implementation is woefully incomplete. | | |
| 12712 | for (constraint) |byte| { | | |
| 12713 | switch (byte) { | | |
| 12714 | '=', '*', ',', '&' => {}, | | |
| 12715 | 'm', 'o' => {}, | | |
| 12716 | else => return true, | | |
| 12717 | } | | |
| 12718 | } else return false; | | |
| 12719 | } | | |
| 12720 | | 4723 | |
| 12721 | pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void { | 4724 | pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void { |
| 12722 | switch (arch) { | 4725 | switch (arch) { |
| 12723 | .aarch64, .aarch64_be => { | 4726 | .aarch64, .aarch64_be => { |
| 12724 | llvm.LLVMInitializeAArch64Target(); | 4727 | bindings.LLVMInitializeAArch64Target(); |
| 12725 | llvm.LLVMInitializeAArch64TargetInfo(); | 4728 | bindings.LLVMInitializeAArch64TargetInfo(); |
| 12726 | llvm.LLVMInitializeAArch64TargetMC(); | 4729 | bindings.LLVMInitializeAArch64TargetMC(); |
| 12727 | llvm.LLVMInitializeAArch64AsmPrinter(); | 4730 | bindings.LLVMInitializeAArch64AsmPrinter(); |
| 12728 | llvm.LLVMInitializeAArch64AsmParser(); | 4731 | bindings.LLVMInitializeAArch64AsmParser(); |
| 12729 | }, | 4732 | }, |
| 12730 | .amdgcn => { | 4733 | .amdgcn => { |
| 12731 | llvm.LLVMInitializeAMDGPUTarget(); | 4734 | bindings.LLVMInitializeAMDGPUTarget(); |
| 12732 | llvm.LLVMInitializeAMDGPUTargetInfo(); | 4735 | bindings.LLVMInitializeAMDGPUTargetInfo(); |
| 12733 | llvm.LLVMInitializeAMDGPUTargetMC(); | 4736 | bindings.LLVMInitializeAMDGPUTargetMC(); |
| 12734 | llvm.LLVMInitializeAMDGPUAsmPrinter(); | 4737 | bindings.LLVMInitializeAMDGPUAsmPrinter(); |
| 12735 | llvm.LLVMInitializeAMDGPUAsmParser(); | 4738 | bindings.LLVMInitializeAMDGPUAsmParser(); |
| 12736 | }, | 4739 | }, |
| 12737 | .thumb, .thumbeb, .arm, .armeb => { | 4740 | .thumb, .thumbeb, .arm, .armeb => { |
| 12738 | llvm.LLVMInitializeARMTarget(); | 4741 | bindings.LLVMInitializeARMTarget(); |
| 12739 | llvm.LLVMInitializeARMTargetInfo(); | 4742 | bindings.LLVMInitializeARMTargetInfo(); |
| 12740 | llvm.LLVMInitializeARMTargetMC(); | 4743 | bindings.LLVMInitializeARMTargetMC(); |
| 12741 | llvm.LLVMInitializeARMAsmPrinter(); | 4744 | bindings.LLVMInitializeARMAsmPrinter(); |
| 12742 | llvm.LLVMInitializeARMAsmParser(); | 4745 | bindings.LLVMInitializeARMAsmParser(); |
| 12743 | }, | 4746 | }, |
| 12744 | .avr => { | 4747 | .avr => { |
| 12745 | llvm.LLVMInitializeAVRTarget(); | 4748 | bindings.LLVMInitializeAVRTarget(); |
| 12746 | llvm.LLVMInitializeAVRTargetInfo(); | 4749 | bindings.LLVMInitializeAVRTargetInfo(); |
| 12747 | llvm.LLVMInitializeAVRTargetMC(); | 4750 | bindings.LLVMInitializeAVRTargetMC(); |
| 12748 | llvm.LLVMInitializeAVRAsmPrinter(); | 4751 | bindings.LLVMInitializeAVRAsmPrinter(); |
| 12749 | llvm.LLVMInitializeAVRAsmParser(); | 4752 | bindings.LLVMInitializeAVRAsmParser(); |
| 12750 | }, | 4753 | }, |
| 12751 | .bpfel, .bpfeb => { | 4754 | .bpfel, .bpfeb => { |
| 12752 | llvm.LLVMInitializeBPFTarget(); | 4755 | bindings.LLVMInitializeBPFTarget(); |
| 12753 | llvm.LLVMInitializeBPFTargetInfo(); | 4756 | bindings.LLVMInitializeBPFTargetInfo(); |
| 12754 | llvm.LLVMInitializeBPFTargetMC(); | 4757 | bindings.LLVMInitializeBPFTargetMC(); |
| 12755 | llvm.LLVMInitializeBPFAsmPrinter(); | 4758 | bindings.LLVMInitializeBPFAsmPrinter(); |
| 12756 | llvm.LLVMInitializeBPFAsmParser(); | 4759 | bindings.LLVMInitializeBPFAsmParser(); |
| 12757 | }, | 4760 | }, |
| 12758 | .hexagon => { | 4761 | .hexagon => { |
| 12759 | llvm.LLVMInitializeHexagonTarget(); | 4762 | bindings.LLVMInitializeHexagonTarget(); |
| 12760 | llvm.LLVMInitializeHexagonTargetInfo(); | 4763 | bindings.LLVMInitializeHexagonTargetInfo(); |
| 12761 | llvm.LLVMInitializeHexagonTargetMC(); | 4764 | bindings.LLVMInitializeHexagonTargetMC(); |
| 12762 | llvm.LLVMInitializeHexagonAsmPrinter(); | 4765 | bindings.LLVMInitializeHexagonAsmPrinter(); |
| 12763 | llvm.LLVMInitializeHexagonAsmParser(); | 4766 | bindings.LLVMInitializeHexagonAsmParser(); |
| 12764 | }, | 4767 | }, |
| 12765 | .lanai => { | 4768 | .lanai => { |
| 12766 | llvm.LLVMInitializeLanaiTarget(); | 4769 | bindings.LLVMInitializeLanaiTarget(); |
| 12767 | llvm.LLVMInitializeLanaiTargetInfo(); | 4770 | bindings.LLVMInitializeLanaiTargetInfo(); |
| 12768 | llvm.LLVMInitializeLanaiTargetMC(); | 4771 | bindings.LLVMInitializeLanaiTargetMC(); |
| 12769 | llvm.LLVMInitializeLanaiAsmPrinter(); | 4772 | bindings.LLVMInitializeLanaiAsmPrinter(); |
| 12770 | llvm.LLVMInitializeLanaiAsmParser(); | 4773 | bindings.LLVMInitializeLanaiAsmParser(); |
| 12771 | }, | 4774 | }, |
| 12772 | .mips, .mipsel, .mips64, .mips64el => { | 4775 | .mips, .mipsel, .mips64, .mips64el => { |
| 12773 | llvm.LLVMInitializeMipsTarget(); | 4776 | bindings.LLVMInitializeMipsTarget(); |
| 12774 | llvm.LLVMInitializeMipsTargetInfo(); | 4777 | bindings.LLVMInitializeMipsTargetInfo(); |
| 12775 | llvm.LLVMInitializeMipsTargetMC(); | 4778 | bindings.LLVMInitializeMipsTargetMC(); |
| 12776 | llvm.LLVMInitializeMipsAsmPrinter(); | 4779 | bindings.LLVMInitializeMipsAsmPrinter(); |
| 12777 | llvm.LLVMInitializeMipsAsmParser(); | 4780 | bindings.LLVMInitializeMipsAsmParser(); |
| 12778 | }, | 4781 | }, |
| 12779 | .msp430 => { | 4782 | .msp430 => { |
| 12780 | llvm.LLVMInitializeMSP430Target(); | 4783 | bindings.LLVMInitializeMSP430Target(); |
| 12781 | llvm.LLVMInitializeMSP430TargetInfo(); | 4784 | bindings.LLVMInitializeMSP430TargetInfo(); |
| 12782 | llvm.LLVMInitializeMSP430TargetMC(); | 4785 | bindings.LLVMInitializeMSP430TargetMC(); |
| 12783 | llvm.LLVMInitializeMSP430AsmPrinter(); | 4786 | bindings.LLVMInitializeMSP430AsmPrinter(); |
| 12784 | llvm.LLVMInitializeMSP430AsmParser(); | 4787 | bindings.LLVMInitializeMSP430AsmParser(); |
| 12785 | }, | 4788 | }, |
| 12786 | .nvptx, .nvptx64 => { | 4789 | .nvptx, .nvptx64 => { |
| 12787 | llvm.LLVMInitializeNVPTXTarget(); | 4790 | bindings.LLVMInitializeNVPTXTarget(); |
| 12788 | llvm.LLVMInitializeNVPTXTargetInfo(); | 4791 | bindings.LLVMInitializeNVPTXTargetInfo(); |
| 12789 | llvm.LLVMInitializeNVPTXTargetMC(); | 4792 | bindings.LLVMInitializeNVPTXTargetMC(); |
| 12790 | llvm.LLVMInitializeNVPTXAsmPrinter(); | 4793 | bindings.LLVMInitializeNVPTXAsmPrinter(); |
| 12791 | // There is no LLVMInitializeNVPTXAsmParser function available. | 4794 | // There is no LLVMInitializeNVPTXAsmParser function available. |
| 12792 | }, | 4795 | }, |
| 12793 | .powerpc, .powerpcle, .powerpc64, .powerpc64le => { | 4796 | .powerpc, .powerpcle, .powerpc64, .powerpc64le => { |
| 12794 | llvm.LLVMInitializePowerPCTarget(); | 4797 | bindings.LLVMInitializePowerPCTarget(); |
| 12795 | llvm.LLVMInitializePowerPCTargetInfo(); | 4798 | bindings.LLVMInitializePowerPCTargetInfo(); |
| 12796 | llvm.LLVMInitializePowerPCTargetMC(); | 4799 | bindings.LLVMInitializePowerPCTargetMC(); |
| 12797 | llvm.LLVMInitializePowerPCAsmPrinter(); | 4800 | bindings.LLVMInitializePowerPCAsmPrinter(); |
| 12798 | llvm.LLVMInitializePowerPCAsmParser(); | 4801 | bindings.LLVMInitializePowerPCAsmParser(); |
| 12799 | }, | 4802 | }, |
| 12800 | .riscv32, .riscv32be, .riscv64, .riscv64be => { | 4803 | .riscv32, .riscv32be, .riscv64, .riscv64be => { |
| 12801 | llvm.LLVMInitializeRISCVTarget(); | 4804 | bindings.LLVMInitializeRISCVTarget(); |
| 12802 | llvm.LLVMInitializeRISCVTargetInfo(); | 4805 | bindings.LLVMInitializeRISCVTargetInfo(); |
| 12803 | llvm.LLVMInitializeRISCVTargetMC(); | 4806 | bindings.LLVMInitializeRISCVTargetMC(); |
| 12804 | llvm.LLVMInitializeRISCVAsmPrinter(); | 4807 | bindings.LLVMInitializeRISCVAsmPrinter(); |
| 12805 | llvm.LLVMInitializeRISCVAsmParser(); | 4808 | bindings.LLVMInitializeRISCVAsmParser(); |
| 12806 | }, | 4809 | }, |
| 12807 | .sparc, .sparc64 => { | 4810 | .sparc, .sparc64 => { |
| 12808 | llvm.LLVMInitializeSparcTarget(); | 4811 | bindings.LLVMInitializeSparcTarget(); |
| 12809 | llvm.LLVMInitializeSparcTargetInfo(); | 4812 | bindings.LLVMInitializeSparcTargetInfo(); |
| 12810 | llvm.LLVMInitializeSparcTargetMC(); | 4813 | bindings.LLVMInitializeSparcTargetMC(); |
| 12811 | llvm.LLVMInitializeSparcAsmPrinter(); | 4814 | bindings.LLVMInitializeSparcAsmPrinter(); |
| 12812 | llvm.LLVMInitializeSparcAsmParser(); | 4815 | bindings.LLVMInitializeSparcAsmParser(); |
| 12813 | }, | 4816 | }, |
| 12814 | .s390x => { | 4817 | .s390x => { |
| 12815 | llvm.LLVMInitializeSystemZTarget(); | 4818 | bindings.LLVMInitializeSystemZTarget(); |
| 12816 | llvm.LLVMInitializeSystemZTargetInfo(); | 4819 | bindings.LLVMInitializeSystemZTargetInfo(); |
| 12817 | llvm.LLVMInitializeSystemZTargetMC(); | 4820 | bindings.LLVMInitializeSystemZTargetMC(); |
| 12818 | llvm.LLVMInitializeSystemZAsmPrinter(); | 4821 | bindings.LLVMInitializeSystemZAsmPrinter(); |
| 12819 | llvm.LLVMInitializeSystemZAsmParser(); | 4822 | bindings.LLVMInitializeSystemZAsmParser(); |
| 12820 | }, | 4823 | }, |
| 12821 | .wasm32, .wasm64 => { | 4824 | .wasm32, .wasm64 => { |
| 12822 | llvm.LLVMInitializeWebAssemblyTarget(); | 4825 | bindings.LLVMInitializeWebAssemblyTarget(); |
| 12823 | llvm.LLVMInitializeWebAssemblyTargetInfo(); | 4826 | bindings.LLVMInitializeWebAssemblyTargetInfo(); |
| 12824 | llvm.LLVMInitializeWebAssemblyTargetMC(); | 4827 | bindings.LLVMInitializeWebAssemblyTargetMC(); |
| 12825 | llvm.LLVMInitializeWebAssemblyAsmPrinter(); | 4828 | bindings.LLVMInitializeWebAssemblyAsmPrinter(); |
| 12826 | llvm.LLVMInitializeWebAssemblyAsmParser(); | 4829 | bindings.LLVMInitializeWebAssemblyAsmParser(); |
| 12827 | }, | 4830 | }, |
| 12828 | .x86, .x86_64 => { | 4831 | .x86, .x86_64 => { |
| 12829 | llvm.LLVMInitializeX86Target(); | 4832 | bindings.LLVMInitializeX86Target(); |
| 12830 | llvm.LLVMInitializeX86TargetInfo(); | 4833 | bindings.LLVMInitializeX86TargetInfo(); |
| 12831 | llvm.LLVMInitializeX86TargetMC(); | 4834 | bindings.LLVMInitializeX86TargetMC(); |
| 12832 | llvm.LLVMInitializeX86AsmPrinter(); | 4835 | bindings.LLVMInitializeX86AsmPrinter(); |
| 12833 | llvm.LLVMInitializeX86AsmParser(); | 4836 | bindings.LLVMInitializeX86AsmParser(); |
| 12834 | }, | 4837 | }, |
| 12835 | .xtensa => { | 4838 | .xtensa => { |
| 12836 | if (build_options.llvm_has_xtensa) { | 4839 | if (build_options.llvm_has_xtensa) { |
| 12837 | llvm.LLVMInitializeXtensaTarget(); | 4840 | bindings.LLVMInitializeXtensaTarget(); |
| 12838 | llvm.LLVMInitializeXtensaTargetInfo(); | 4841 | bindings.LLVMInitializeXtensaTargetInfo(); |
| 12839 | llvm.LLVMInitializeXtensaTargetMC(); | 4842 | bindings.LLVMInitializeXtensaTargetMC(); |
| 12840 | // There is no LLVMInitializeXtensaAsmPrinter function. | 4843 | // There is no LLVMInitializeXtensaAsmPrinter function. |
| 12841 | llvm.LLVMInitializeXtensaAsmParser(); | 4844 | bindings.LLVMInitializeXtensaAsmParser(); |
| 12842 | } | 4845 | } |
| 12843 | }, | 4846 | }, |
| 12844 | .xcore => { | 4847 | .xcore => { |
| 12845 | llvm.LLVMInitializeXCoreTarget(); | 4848 | bindings.LLVMInitializeXCoreTarget(); |
| 12846 | llvm.LLVMInitializeXCoreTargetInfo(); | 4849 | bindings.LLVMInitializeXCoreTargetInfo(); |
| 12847 | llvm.LLVMInitializeXCoreTargetMC(); | 4850 | bindings.LLVMInitializeXCoreTargetMC(); |
| 12848 | llvm.LLVMInitializeXCoreAsmPrinter(); | 4851 | bindings.LLVMInitializeXCoreAsmPrinter(); |
| 12849 | // There is no LLVMInitializeXCoreAsmParser function. | 4852 | // There is no LLVMInitializeXCoreAsmParser function. |
| 12850 | }, | 4853 | }, |
| 12851 | .m68k => { | 4854 | .m68k => { |
| 12852 | if (build_options.llvm_has_m68k) { | 4855 | if (build_options.llvm_has_m68k) { |
| 12853 | llvm.LLVMInitializeM68kTarget(); | 4856 | bindings.LLVMInitializeM68kTarget(); |
| 12854 | llvm.LLVMInitializeM68kTargetInfo(); | 4857 | bindings.LLVMInitializeM68kTargetInfo(); |
| 12855 | llvm.LLVMInitializeM68kTargetMC(); | 4858 | bindings.LLVMInitializeM68kTargetMC(); |
| 12856 | llvm.LLVMInitializeM68kAsmPrinter(); | 4859 | bindings.LLVMInitializeM68kAsmPrinter(); |
| 12857 | llvm.LLVMInitializeM68kAsmParser(); | 4860 | bindings.LLVMInitializeM68kAsmParser(); |
| 12858 | } | 4861 | } |
| 12859 | }, | 4862 | }, |
| 12860 | .csky => { | 4863 | .csky => { |
| 12861 | if (build_options.llvm_has_csky) { | 4864 | if (build_options.llvm_has_csky) { |
| 12862 | llvm.LLVMInitializeCSKYTarget(); | 4865 | bindings.LLVMInitializeCSKYTarget(); |
| 12863 | llvm.LLVMInitializeCSKYTargetInfo(); | 4866 | bindings.LLVMInitializeCSKYTargetInfo(); |
| 12864 | llvm.LLVMInitializeCSKYTargetMC(); | 4867 | bindings.LLVMInitializeCSKYTargetMC(); |
| 12865 | // There is no LLVMInitializeCSKYAsmPrinter function. | 4868 | // There is no LLVMInitializeCSKYAsmPrinter function. |
| 12866 | llvm.LLVMInitializeCSKYAsmParser(); | 4869 | bindings.LLVMInitializeCSKYAsmParser(); |
| 12867 | } | 4870 | } |
| 12868 | }, | 4871 | }, |
| 12869 | .ve => { | 4872 | .ve => { |
| 12870 | llvm.LLVMInitializeVETarget(); | 4873 | bindings.LLVMInitializeVETarget(); |
| 12871 | llvm.LLVMInitializeVETargetInfo(); | 4874 | bindings.LLVMInitializeVETargetInfo(); |
| 12872 | llvm.LLVMInitializeVETargetMC(); | 4875 | bindings.LLVMInitializeVETargetMC(); |
| 12873 | llvm.LLVMInitializeVEAsmPrinter(); | 4876 | bindings.LLVMInitializeVEAsmPrinter(); |
| 12874 | llvm.LLVMInitializeVEAsmParser(); | 4877 | bindings.LLVMInitializeVEAsmParser(); |
| 12875 | }, | 4878 | }, |
| 12876 | .arc => { | 4879 | .arc => { |
| 12877 | if (build_options.llvm_has_arc) { | 4880 | if (build_options.llvm_has_arc) { |
| 12878 | llvm.LLVMInitializeARCTarget(); | 4881 | bindings.LLVMInitializeARCTarget(); |
| 12879 | llvm.LLVMInitializeARCTargetInfo(); | 4882 | bindings.LLVMInitializeARCTargetInfo(); |
| 12880 | llvm.LLVMInitializeARCTargetMC(); | 4883 | bindings.LLVMInitializeARCTargetMC(); |
| 12881 | llvm.LLVMInitializeARCAsmPrinter(); | 4884 | bindings.LLVMInitializeARCAsmPrinter(); |
| 12882 | // There is no LLVMInitializeARCAsmParser function. | 4885 | // There is no LLVMInitializeARCAsmParser function. |
| 12883 | } | 4886 | } |
| 12884 | }, | 4887 | }, |
| 12885 | .loongarch32, .loongarch64 => { | 4888 | .loongarch32, .loongarch64 => { |
| 12886 | llvm.LLVMInitializeLoongArchTarget(); | 4889 | bindings.LLVMInitializeLoongArchTarget(); |
| 12887 | llvm.LLVMInitializeLoongArchTargetInfo(); | 4890 | bindings.LLVMInitializeLoongArchTargetInfo(); |
| 12888 | llvm.LLVMInitializeLoongArchTargetMC(); | 4891 | bindings.LLVMInitializeLoongArchTargetMC(); |
| 12889 | llvm.LLVMInitializeLoongArchAsmPrinter(); | 4892 | bindings.LLVMInitializeLoongArchAsmPrinter(); |
| 12890 | llvm.LLVMInitializeLoongArchAsmParser(); | 4893 | bindings.LLVMInitializeLoongArchAsmParser(); |
| 12891 | }, | 4894 | }, |
| 12892 | .spirv32, | 4895 | .spirv32, |
| 12893 | .spirv64, | 4896 | .spirv64, |
| 12894 | => { | 4897 | => { |
| 12895 | llvm.LLVMInitializeSPIRVTarget(); | 4898 | bindings.LLVMInitializeSPIRVTarget(); |
| 12896 | llvm.LLVMInitializeSPIRVTargetInfo(); | 4899 | bindings.LLVMInitializeSPIRVTargetInfo(); |
| 12897 | llvm.LLVMInitializeSPIRVTargetMC(); | 4900 | bindings.LLVMInitializeSPIRVTargetMC(); |
| 12898 | llvm.LLVMInitializeSPIRVAsmPrinter(); | 4901 | bindings.LLVMInitializeSPIRVAsmPrinter(); |
| 12899 | }, | 4902 | }, |
| 12900 | | 4903 | |
| 12901 | // LLVM does does not have a backend for these. | 4904 | // LLVM does does not have a backend for these. |
| ... | @@ -12916,296 +4919,3 @@ pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void { | ... | @@ -12916,296 +4919,3 @@ pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void { |
| 12916 | => unreachable, | 4919 | => unreachable, |
| 12917 | } | 4920 | } |
| 12918 | } | 4921 | } |
| 12919 | | | |
| 12920 | fn minIntConst(b: *Builder, min_ty: Type, as_ty: Builder.Type, zcu: *const Zcu) Allocator.Error!Builder.Constant { | | |
| 12921 | const info = min_ty.intInfo(zcu); | | |
| 12922 | if (info.signedness == .unsigned or info.bits == 0) { | | |
| 12923 | return b.intConst(as_ty, 0); | | |
| 12924 | } | | |
| 12925 | if (std.math.cast(u6, info.bits - 1)) |shift| { | | |
| 12926 | const min_val: i64 = @as(i64, std.math.minInt(i64)) >> (63 - shift); | | |
| 12927 | return b.intConst(as_ty, min_val); | | |
| 12928 | } | | |
| 12929 | var res: std.math.big.int.Managed = try .init(zcu.gpa); | | |
| 12930 | defer res.deinit(); | | |
| 12931 | try res.setTwosCompIntLimit(.min, info.signedness, info.bits); | | |
| 12932 | return b.bigIntConst(as_ty, res.toConst()); | | |
| 12933 | } | | |
| 12934 | | | |
| 12935 | fn maxIntConst(b: *Builder, max_ty: Type, as_ty: Builder.Type, zcu: *const Zcu) Allocator.Error!Builder.Constant { | | |
| 12936 | const info = max_ty.intInfo(zcu); | | |
| 12937 | switch (info.bits) { | | |
| 12938 | 0 => return b.intConst(as_ty, 0), | | |
| 12939 | 1 => switch (info.signedness) { | | |
| 12940 | .signed => return b.intConst(as_ty, 0), | | |
| 12941 | .unsigned => return b.intConst(as_ty, 1), | | |
| 12942 | }, | | |
| 12943 | else => {}, | | |
| 12944 | } | | |
| 12945 | const unsigned_bits = switch (info.signedness) { | | |
| 12946 | .unsigned => info.bits, | | |
| 12947 | .signed => info.bits - 1, | | |
| 12948 | }; | | |
| 12949 | if (std.math.cast(u6, unsigned_bits)) |shift| { | | |
| 12950 | const max_val: u64 = (@as(u64, 1) << shift) - 1; | | |
| 12951 | return b.intConst(as_ty, max_val); | | |
| 12952 | } | | |
| 12953 | var res: std.math.big.int.Managed = try .init(zcu.gpa); | | |
| 12954 | defer res.deinit(); | | |
| 12955 | try res.setTwosCompIntLimit(.max, info.signedness, info.bits); | | |
| 12956 | return b.bigIntConst(as_ty, res.toConst()); | | |
| 12957 | } | | |
| 12958 | | | |
| 12959 | /// Appends zero or more LLVM constraints to `llvm_constraints`, returning how many were added. | | |
| 12960 | fn appendConstraints( | | |
| 12961 | gpa: Allocator, | | |
| 12962 | llvm_constraints: *std.ArrayList(u8), | | |
| 12963 | zig_name: []const u8, | | |
| 12964 | target: *const std.Target, | | |
| 12965 | ) error{OutOfMemory}!usize { | | |
| 12966 | switch (target.cpu.arch) { | | |
| 12967 | .mips, .mipsel, .mips64, .mips64el => if (mips_clobber_overrides.get(zig_name)) |llvm_tag| { | | |
| 12968 | const llvm_name = @tagName(llvm_tag); | | |
| 12969 | try llvm_constraints.ensureUnusedCapacity(gpa, llvm_name.len + 4); | | |
| 12970 | llvm_constraints.appendSliceAssumeCapacity("~{"); | | |
| 12971 | llvm_constraints.appendSliceAssumeCapacity(llvm_name); | | |
| 12972 | llvm_constraints.appendSliceAssumeCapacity("},"); | | |
| 12973 | return 1; | | |
| 12974 | }, | | |
| 12975 | else => {}, | | |
| 12976 | } | | |
| 12977 | | | |
| 12978 | try llvm_constraints.ensureUnusedCapacity(gpa, zig_name.len + 4); | | |
| 12979 | llvm_constraints.appendSliceAssumeCapacity("~{"); | | |
| 12980 | llvm_constraints.appendSliceAssumeCapacity(zig_name); | | |
| 12981 | llvm_constraints.appendSliceAssumeCapacity("},"); | | |
| 12982 | return 1; | | |
| 12983 | } | | |
| 12984 | | | |
| 12985 | const mips_clobber_overrides = std.StaticStringMap(enum { | | |
| 12986 | @"$msair", | | |
| 12987 | @"$msacsr", | | |
| 12988 | @"$msaaccess", | | |
| 12989 | @"$msasave", | | |
| 12990 | @"$msamodify", | | |
| 12991 | @"$msarequest", | | |
| 12992 | @"$msamap", | | |
| 12993 | @"$msaunmap", | | |
| 12994 | @"$f0", | | |
| 12995 | @"$f1", | | |
| 12996 | @"$f2", | | |
| 12997 | @"$f3", | | |
| 12998 | @"$f4", | | |
| 12999 | @"$f5", | | |
| 13000 | @"$f6", | | |
| 13001 | @"$f7", | | |
| 13002 | @"$f8", | | |
| 13003 | @"$f9", | | |
| 13004 | @"$f10", | | |
| 13005 | @"$f11", | | |
| 13006 | @"$f12", | | |
| 13007 | @"$f13", | | |
| 13008 | @"$f14", | | |
| 13009 | @"$f15", | | |
| 13010 | @"$f16", | | |
| 13011 | @"$f17", | | |
| 13012 | @"$f18", | | |
| 13013 | @"$f19", | | |
| 13014 | @"$f20", | | |
| 13015 | @"$f21", | | |
| 13016 | @"$f22", | | |
| 13017 | @"$f23", | | |
| 13018 | @"$f24", | | |
| 13019 | @"$f25", | | |
| 13020 | @"$f26", | | |
| 13021 | @"$f27", | | |
| 13022 | @"$f28", | | |
| 13023 | @"$f29", | | |
| 13024 | @"$f30", | | |
| 13025 | @"$f31", | | |
| 13026 | @"$fcc0", | | |
| 13027 | @"$fcc1", | | |
| 13028 | @"$fcc2", | | |
| 13029 | @"$fcc3", | | |
| 13030 | @"$fcc4", | | |
| 13031 | @"$fcc5", | | |
| 13032 | @"$fcc6", | | |
| 13033 | @"$fcc7", | | |
| 13034 | @"$w0", | | |
| 13035 | @"$w1", | | |
| 13036 | @"$w2", | | |
| 13037 | @"$w3", | | |
| 13038 | @"$w4", | | |
| 13039 | @"$w5", | | |
| 13040 | @"$w6", | | |
| 13041 | @"$w7", | | |
| 13042 | @"$w8", | | |
| 13043 | @"$w9", | | |
| 13044 | @"$w10", | | |
| 13045 | @"$w11", | | |
| 13046 | @"$w12", | | |
| 13047 | @"$w13", | | |
| 13048 | @"$w14", | | |
| 13049 | @"$w15", | | |
| 13050 | @"$w16", | | |
| 13051 | @"$w17", | | |
| 13052 | @"$w18", | | |
| 13053 | @"$w19", | | |
| 13054 | @"$w20", | | |
| 13055 | @"$w21", | | |
| 13056 | @"$w22", | | |
| 13057 | @"$w23", | | |
| 13058 | @"$w24", | | |
| 13059 | @"$w25", | | |
| 13060 | @"$w26", | | |
| 13061 | @"$w27", | | |
| 13062 | @"$w28", | | |
| 13063 | @"$w29", | | |
| 13064 | @"$w30", | | |
| 13065 | @"$w31", | | |
| 13066 | @"$0", | | |
| 13067 | @"$1", | | |
| 13068 | @"$2", | | |
| 13069 | @"$3", | | |
| 13070 | @"$4", | | |
| 13071 | @"$5", | | |
| 13072 | @"$6", | | |
| 13073 | @"$7", | | |
| 13074 | @"$8", | | |
| 13075 | @"$9", | | |
| 13076 | @"$10", | | |
| 13077 | @"$11", | | |
| 13078 | @"$12", | | |
| 13079 | @"$13", | | |
| 13080 | @"$14", | | |
| 13081 | @"$15", | | |
| 13082 | @"$16", | | |
| 13083 | @"$17", | | |
| 13084 | @"$18", | | |
| 13085 | @"$19", | | |
| 13086 | @"$20", | | |
| 13087 | @"$21", | | |
| 13088 | @"$22", | | |
| 13089 | @"$23", | | |
| 13090 | @"$24", | | |
| 13091 | @"$25", | | |
| 13092 | @"$26", | | |
| 13093 | @"$27", | | |
| 13094 | @"$28", | | |
| 13095 | @"$29", | | |
| 13096 | @"$30", | | |
| 13097 | @"$31", | | |
| 13098 | }).initComptime(.{ | | |
| 13099 | .{ "msa_ir", .@"$msair" }, | | |
| 13100 | .{ "msa_csr", .@"$msacsr" }, | | |
| 13101 | .{ "msa_access", .@"$msaaccess" }, | | |
| 13102 | .{ "msa_save", .@"$msasave" }, | | |
| 13103 | .{ "msa_modify", .@"$msamodify" }, | | |
| 13104 | .{ "msa_request", .@"$msarequest" }, | | |
| 13105 | .{ "msa_map", .@"$msamap" }, | | |
| 13106 | .{ "msa_unmap", .@"$msaunmap" }, | | |
| 13107 | .{ "f0", .@"$f0" }, | | |
| 13108 | .{ "f1", .@"$f1" }, | | |
| 13109 | .{ "f2", .@"$f2" }, | | |
| 13110 | .{ "f3", .@"$f3" }, | | |
| 13111 | .{ "f4", .@"$f4" }, | | |
| 13112 | .{ "f5", .@"$f5" }, | | |
| 13113 | .{ "f6", .@"$f6" }, | | |
| 13114 | .{ "f7", .@"$f7" }, | | |
| 13115 | .{ "f8", .@"$f8" }, | | |
| 13116 | .{ "f9", .@"$f9" }, | | |
| 13117 | .{ "f10", .@"$f10" }, | | |
| 13118 | .{ "f11", .@"$f11" }, | | |
| 13119 | .{ "f12", .@"$f12" }, | | |
| 13120 | .{ "f13", .@"$f13" }, | | |
| 13121 | .{ "f14", .@"$f14" }, | | |
| 13122 | .{ "f15", .@"$f15" }, | | |
| 13123 | .{ "f16", .@"$f16" }, | | |
| 13124 | .{ "f17", .@"$f17" }, | | |
| 13125 | .{ "f18", .@"$f18" }, | | |
| 13126 | .{ "f19", .@"$f19" }, | | |
| 13127 | .{ "f20", .@"$f20" }, | | |
| 13128 | .{ "f21", .@"$f21" }, | | |
| 13129 | .{ "f22", .@"$f22" }, | | |
| 13130 | .{ "f23", .@"$f23" }, | | |
| 13131 | .{ "f24", .@"$f24" }, | | |
| 13132 | .{ "f25", .@"$f25" }, | | |
| 13133 | .{ "f26", .@"$f26" }, | | |
| 13134 | .{ "f27", .@"$f27" }, | | |
| 13135 | .{ "f28", .@"$f28" }, | | |
| 13136 | .{ "f29", .@"$f29" }, | | |
| 13137 | .{ "f30", .@"$f30" }, | | |
| 13138 | .{ "f31", .@"$f31" }, | | |
| 13139 | .{ "fcc0", .@"$fcc0" }, | | |
| 13140 | .{ "fcc1", .@"$fcc1" }, | | |
| 13141 | .{ "fcc2", .@"$fcc2" }, | | |
| 13142 | .{ "fcc3", .@"$fcc3" }, | | |
| 13143 | .{ "fcc4", .@"$fcc4" }, | | |
| 13144 | .{ "fcc5", .@"$fcc5" }, | | |
| 13145 | .{ "fcc6", .@"$fcc6" }, | | |
| 13146 | .{ "fcc7", .@"$fcc7" }, | | |
| 13147 | .{ "w0", .@"$w0" }, | | |
| 13148 | .{ "w1", .@"$w1" }, | | |
| 13149 | .{ "w2", .@"$w2" }, | | |
| 13150 | .{ "w3", .@"$w3" }, | | |
| 13151 | .{ "w4", .@"$w4" }, | | |
| 13152 | .{ "w5", .@"$w5" }, | | |
| 13153 | .{ "w6", .@"$w6" }, | | |
| 13154 | .{ "w7", .@"$w7" }, | | |
| 13155 | .{ "w8", .@"$w8" }, | | |
| 13156 | .{ "w9", .@"$w9" }, | | |
| 13157 | .{ "w10", .@"$w10" }, | | |
| 13158 | .{ "w11", .@"$w11" }, | | |
| 13159 | .{ "w12", .@"$w12" }, | | |
| 13160 | .{ "w13", .@"$w13" }, | | |
| 13161 | .{ "w14", .@"$w14" }, | | |
| 13162 | .{ "w15", .@"$w15" }, | | |
| 13163 | .{ "w16", .@"$w16" }, | | |
| 13164 | .{ "w17", .@"$w17" }, | | |
| 13165 | .{ "w18", .@"$w18" }, | | |
| 13166 | .{ "w19", .@"$w19" }, | | |
| 13167 | .{ "w20", .@"$w20" }, | | |
| 13168 | .{ "w21", .@"$w21" }, | | |
| 13169 | .{ "w22", .@"$w22" }, | | |
| 13170 | .{ "w23", .@"$w23" }, | | |
| 13171 | .{ "w24", .@"$w24" }, | | |
| 13172 | .{ "w25", .@"$w25" }, | | |
| 13173 | .{ "w26", .@"$w26" }, | | |
| 13174 | .{ "w27", .@"$w27" }, | | |
| 13175 | .{ "w28", .@"$w28" }, | | |
| 13176 | .{ "w29", .@"$w29" }, | | |
| 13177 | .{ "w30", .@"$w30" }, | | |
| 13178 | .{ "w31", .@"$w31" }, | | |
| 13179 | .{ "r0", .@"$0" }, | | |
| 13180 | .{ "r1", .@"$1" }, | | |
| 13181 | .{ "r2", .@"$2" }, | | |
| 13182 | .{ "r3", .@"$3" }, | | |
| 13183 | .{ "r4", .@"$4" }, | | |
| 13184 | .{ "r5", .@"$5" }, | | |
| 13185 | .{ "r6", .@"$6" }, | | |
| 13186 | .{ "r7", .@"$7" }, | | |
| 13187 | .{ "r8", .@"$8" }, | | |
| 13188 | .{ "r9", .@"$9" }, | | |
| 13189 | .{ "r10", .@"$10" }, | | |
| 13190 | .{ "r11", .@"$11" }, | | |
| 13191 | .{ "r12", .@"$12" }, | | |
| 13192 | .{ "r13", .@"$13" }, | | |
| 13193 | .{ "r14", .@"$14" }, | | |
| 13194 | .{ "r15", .@"$15" }, | | |
| 13195 | .{ "r16", .@"$16" }, | | |
| 13196 | .{ "r17", .@"$17" }, | | |
| 13197 | .{ "r18", .@"$18" }, | | |
| 13198 | .{ "r19", .@"$19" }, | | |
| 13199 | .{ "r20", .@"$20" }, | | |
| 13200 | .{ "r21", .@"$21" }, | | |
| 13201 | .{ "r22", .@"$22" }, | | |
| 13202 | .{ "r23", .@"$23" }, | | |
| 13203 | .{ "r24", .@"$24" }, | | |
| 13204 | .{ "r25", .@"$25" }, | | |
| 13205 | .{ "r26", .@"$26" }, | | |
| 13206 | .{ "r27", .@"$27" }, | | |
| 13207 | .{ "r28", .@"$28" }, | | |
| 13208 | .{ "r29", .@"$29" }, | | |
| 13209 | .{ "r30", .@"$30" }, | | |
| 13210 | .{ "r31", .@"$31" }, | | |
| 13211 | }); | | |
Matthew Lugg <mlugg@mlugg.co.uk> 2026-03-29 13:41:10+02:00
lib