authorgravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2022-08-21 01:32:19+02:00
committergravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2022-10-12 20:34:42+02:00
log5f3b91437f5cd23bcae66227932555b7abe32669
tree3d1db6c0e791e7b206ea4eb88abdad252f3901bd
parent3392de87dcf516ed2e3fd1904372195bdc75c0eb
signaturelock-open Commit is signed but in an unrecognized format.

stage2: improve addrspace handling

This commit changes the way Zig is intended to deal with variable declaration for exotic targets. Where previously the idea was to enfore local/global variables to be placed into their respective address spaces, depending on the target, this is now fixed to the generic address space. To facilitate this for targets where local variables _must_ be generated into a specific address space (ex. amdgcn where locals must be generated into the private address space), the variable allocations (alloca) are generated into the right address space and then addrspace-casted back to the generic address space. While this could be less efficient in theory, LLVM will hopefull deal with figuring out the actual correct address space for a pointer for us. HIP seems to do the same thing in this regard. Global variables are handled in a similar way.

4 files changed, 203 insertions(+), 132 deletions(-)

src/codegen/llvm.zig+181-126
......@@ -929,8 +929,7 @@ pub const Object = struct {
929929 if (isByRef(param_ty)) {
930930 const alignment = param_ty.abiAlignment(target);
931931 const param_llvm_ty = param.typeOf();
932 const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty);
933 arg_ptr.setAlignment(alignment);
932 const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty, alignment, target);
934933 const store_inst = builder.buildStore(param, arg_ptr);
935934 store_inst.setAlignment(alignment);
936935 args.appendAssumeCapacity(arg_ptr);
......@@ -974,8 +973,7 @@ pub const Object = struct {
974973 param_ty.abiAlignment(target),
975974 dg.object.target_data.abiAlignmentOfType(int_llvm_ty),
976975 );
977 const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty);
978 arg_ptr.setAlignment(alignment);
976 const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty, alignment, target);
979977 const casted_ptr = builder.buildBitCast(arg_ptr, int_ptr_llvm_ty, "");
980978 const store_inst = builder.buildStore(param, casted_ptr);
981979 store_inst.setAlignment(alignment);
......@@ -1026,8 +1024,7 @@ pub const Object = struct {
10261024 const param_ty = fn_info.param_types[it.zig_index - 1];
10271025 const param_llvm_ty = try dg.lowerType(param_ty);
10281026 const param_alignment = param_ty.abiAlignment(target);
1029 const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty);
1030 arg_ptr.setAlignment(param_alignment);
1027 const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty, param_alignment, target);
10311028 var field_types_buf: [8]*llvm.Type = undefined;
10321029 const field_types = field_types_buf[0..llvm_ints.len];
10331030 for (llvm_ints) |int_bits, i| {
......@@ -1058,8 +1055,7 @@ pub const Object = struct {
10581055 const param_ty = fn_info.param_types[it.zig_index - 1];
10591056 const param_llvm_ty = try dg.lowerType(param_ty);
10601057 const param_alignment = param_ty.abiAlignment(target);
1061 const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty);
1062 arg_ptr.setAlignment(param_alignment);
1058 const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty, param_alignment, target);
10631059 var field_types_buf: [8]*llvm.Type = undefined;
10641060 const field_types = field_types_buf[0..llvm_floats.len];
10651061 for (llvm_floats) |float_bits, i| {
......@@ -1103,8 +1099,7 @@ pub const Object = struct {
11031099 llvm_arg_i += 1;
11041100
11051101 const alignment = param_ty.abiAlignment(target);
1106 const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty);
1107 arg_ptr.setAlignment(alignment);
1102 const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty, alignment, target);
11081103 const casted_ptr = builder.buildBitCast(arg_ptr, param.typeOf().pointerType(0), "");
11091104 _ = builder.buildStore(param, casted_ptr);
11101105
......@@ -2404,21 +2399,27 @@ pub const DeclGen = struct {
24042399 // mismatch, because we don't have the LLVM type until the *value* is created,
24052400 // whereas the global needs to be created based on the type alone, because
24062401 // lowering the value may reference the global as a pointer.
2402 const llvm_addrspace = toLlvmAddressSpace(decl.@"addrspace", target);
2403 const llvm_global_addrspace = toLlvmGlobalAddressSpace(llvm_addrspace, target);
24072404 const new_global = dg.object.llvm_module.addGlobalInAddressSpace(
24082405 llvm_init.typeOf(),
24092406 "",
2410 dg.llvmAddressSpace(decl.@"addrspace"),
2407 llvm_global_addrspace,
24112408 );
24122409 new_global.setLinkage(global.getLinkage());
24132410 new_global.setUnnamedAddr(global.getUnnamedAddress());
24142411 new_global.setAlignment(global.getAlignment());
24152412 if (decl.@"linksection") |section| new_global.setSection(section);
24162413 new_global.setInitializer(llvm_init);
2417 // replaceAllUsesWith requires the type to be unchanged. So we bitcast
2414 // replaceAllUsesWith requires the type to be unchanged. So we convert
24182415 // the new global to the old type and use that as the thing to replace
24192416 // old uses.
2420 const new_global_ptr = new_global.constBitCast(global.typeOf());
2421 global.replaceAllUsesWith(new_global_ptr);
2417 const new_global_ptr = if (llvm_addrspace != llvm_global_addrspace)
2418 new_global.constAddrSpaceCast(llvm_init.typeOf().pointerType(llvm_addrspace))
2419 else
2420 new_global;
2421 const new_global_casted_ptr = new_global_ptr.constBitCast(global.typeOf());
2422 global.replaceAllUsesWith(new_global_casted_ptr);
24222423 dg.object.decl_map.putAssumeCapacity(decl_index, new_global);
24232424 new_global.takeName(global);
24242425 global.deleteGlobal();
......@@ -2465,7 +2466,7 @@ pub const DeclGen = struct {
24652466 const fqn = try decl.getFullyQualifiedName(dg.module);
24662467 defer dg.gpa.free(fqn);
24672468
2468 const llvm_addrspace = dg.llvmAddressSpace(decl.@"addrspace");
2469 const llvm_addrspace = toLlvmAddressSpace(decl.@"addrspace", target);
24692470 const llvm_fn = dg.llvmModule().addFunctionInAddressSpace(fqn, fn_type, llvm_addrspace);
24702471 gop.value_ptr.* = llvm_fn;
24712472
......@@ -2613,9 +2614,15 @@ pub const DeclGen = struct {
26132614 const fqn = try decl.getFullyQualifiedName(dg.module);
26142615 defer dg.gpa.free(fqn);
26152616
2617 const target = dg.module.getTarget();
2618
26162619 const llvm_type = try dg.lowerType(decl.ty);
2617 const llvm_addrspace = dg.llvmAddressSpace(decl.@"addrspace");
2618 const llvm_global = dg.object.llvm_module.addGlobalInAddressSpace(llvm_type, fqn, llvm_addrspace);
2620 const llvm_addrspace = toLlvmAddressSpace(decl.@"addrspace", target);
2621 const llvm_global = dg.object.llvm_module.addGlobalInAddressSpace(
2622 llvm_type,
2623 fqn,
2624 toLlvmGlobalAddressSpace(llvm_addrspace, target),
2625 );
26192626 gop.value_ptr.* = llvm_global;
26202627
26212628 // This is needed for declarations created by `@extern`.
......@@ -2640,40 +2647,6 @@ pub const DeclGen = struct {
26402647 return llvm_global;
26412648 }
26422649
2643 fn llvmAddressSpace(self: DeclGen, address_space: std.builtin.AddressSpace) c_uint {
2644 const target = self.module.getTarget();
2645 return switch (target.cpu.arch) {
2646 .i386, .x86_64 => switch (address_space) {
2647 .generic => llvm.address_space.default,
2648 .gs => llvm.address_space.x86.gs,
2649 .fs => llvm.address_space.x86.fs,
2650 .ss => llvm.address_space.x86.ss,
2651 else => unreachable,
2652 },
2653 .nvptx, .nvptx64 => switch (address_space) {
2654 .generic => llvm.address_space.default,
2655 .global => llvm.address_space.nvptx.global,
2656 .constant => llvm.address_space.nvptx.constant,
2657 .param => llvm.address_space.nvptx.param,
2658 .shared => llvm.address_space.nvptx.shared,
2659 .local => llvm.address_space.nvptx.local,
2660 else => unreachable,
2661 },
2662 .amdgcn => switch (address_space) {
2663 .generic => llvm.address_space.flat,
2664 .global => llvm.address_space.amdgpu.global,
2665 .constant => llvm.address_space.amdgpu.constant,
2666 .shared => llvm.address_space.amdgpu.local,
2667 .local => llvm.address_space.amdgpu.private,
2668 else => unreachable,
2669 }.
2670 else => switch (address_space) {
2671 .generic => llvm.address_space.default,
2672 else => unreachable,
2673 },
2674 };
2675 }
2676
26772650 fn isUnnamedType(dg: *DeclGen, ty: Type, val: *llvm.Value) bool {
26782651 // Once `lowerType` succeeds, successive calls to it with the same Zig type
26792652 // are guaranteed to succeed. So if a call to `lowerType` fails here it means
......@@ -2739,7 +2712,7 @@ pub const DeclGen = struct {
27392712 return dg.context.structType(&fields, fields.len, .False);
27402713 }
27412714 const ptr_info = t.ptrInfo().data;
2742 const llvm_addrspace = dg.llvmAddressSpace(ptr_info.@"addrspace");
2715 const llvm_addrspace = toLlvmAddressSpace(ptr_info.@"addrspace", target);
27432716 if (ptr_info.host_size != 0) {
27442717 return dg.context.intType(ptr_info.host_size * 8).pointerType(llvm_addrspace);
27452718 }
......@@ -3268,11 +3241,18 @@ pub const DeclGen = struct {
32683241 const decl_index = tv.val.castTag(.variable).?.data.owner_decl;
32693242 const decl = dg.module.declPtr(decl_index);
32703243 dg.module.markDeclAlive(decl);
3271 const val = try dg.resolveGlobalDecl(decl_index);
32723244 const llvm_var_type = try dg.lowerType(tv.ty);
3273 const llvm_addrspace = dg.llvmAddressSpace(decl.@"addrspace");
3274 const llvm_type = llvm_var_type.pointerType(llvm_addrspace);
3275 return val.constBitCast(llvm_type);
3245 const llvm_var_addrspace = toLlvmAddressSpace(decl.@"addrspace", target);
3246 const llvm_global_addrspace = toLlvmGlobalAddressSpace(llvm_var_addrspace, target);
3247 const llvm_var_ptr_type = llvm_var_type.pointerType(llvm_global_addrspace);
3248
3249 const val = try dg.resolveGlobalDecl(decl_index);
3250 const val_ptr = val.constBitCast(llvm_var_ptr_type);
3251 if (llvm_global_addrspace != llvm_var_addrspace) {
3252 const llvm_ptr_type = llvm_var_type.pointerType(llvm_var_addrspace);
3253 return val_ptr.constAddrSpaceCast(llvm_ptr_type);
3254 }
3255 return val_ptr;
32763256 },
32773257 .slice => {
32783258 const slice = tv.val.castTag(.slice).?.data;
......@@ -4069,11 +4049,20 @@ pub const DeclGen = struct {
40694049
40704050 self.module.markDeclAlive(decl);
40714051
4072 const llvm_val = if (is_fn_body)
4052 const llvm_decl_val = if (is_fn_body)
40734053 try self.resolveLlvmFunction(decl_index)
40744054 else
40754055 try self.resolveGlobalDecl(decl_index);
40764056
4057 const target = self.module.getTarget();
4058 const llvm_addrspace = toLlvmAddressSpace(decl.@"addrspace", target);
4059 const llvm_global_addrspace = toLlvmGlobalAddressSpace(llvm_addrspace, target);
4060 const llvm_val = if (llvm_addrspace != llvm_global_addrspace) blk: {
4061 const llvm_decl_ty = try self.lowerType(decl.ty);
4062 const llvm_decl_ptr_ty = llvm_decl_ty.pointerType(llvm_addrspace);
4063 break :blk llvm_decl_val.constAddrSpaceCast(llvm_decl_ptr_ty);
4064 } else llvm_decl_val;
4065
40774066 const llvm_type = try self.lowerType(tv.ty);
40784067 if (tv.ty.zigTypeTag() == .Int) {
40794068 return llvm_val.constPtrToInt(llvm_type);
......@@ -4339,7 +4328,9 @@ pub const FuncGen = struct {
43394328 // We have an LLVM value but we need to create a global constant and
43404329 // set the value as its initializer, and then return a pointer to the global.
43414330 const target = self.dg.module.getTarget();
4342 const global = self.dg.object.llvm_module.addGlobal(llvm_val.typeOf(), "");
4331 const llvm_addrspace = toLlvmAddressSpace(.generic, target);
4332 const llvm_global_addrspace = toLlvmGlobalAddressSpace(llvm_addrspace, target);
4333 const global = self.dg.object.llvm_module.addGlobalInAddressSpace(llvm_val.typeOf(), "", llvm_global_addrspace);
43434334 global.setInitializer(llvm_val);
43444335 global.setLinkage(.Private);
43454336 global.setGlobalConstant(.True);
......@@ -4349,8 +4340,10 @@ pub const FuncGen = struct {
43494340 // the type of global constants might not match the type it is supposed to
43504341 // be, and so we must bitcast the pointer at the usage sites.
43514342 const wanted_llvm_ty = try self.dg.lowerType(ty);
4352 const wanted_llvm_ptr_ty = wanted_llvm_ty.pointerType(0);
4353 const casted_ptr = global.constBitCast(wanted_llvm_ptr_ty);
4343 const wanted_bitcasted_llvm_ptr_ty = wanted_llvm_ty.pointerType(llvm_global_addrspace);
4344 const bitcasted_ptr = global.constBitCast(wanted_bitcasted_llvm_ptr_ty);
4345 const wanted_llvm_ptr_ty = wanted_llvm_ty.pointerType(llvm_addrspace);
4346 const casted_ptr = bitcasted_ptr.constAddrSpaceCast(wanted_llvm_ptr_ty);
43544347 gop.value_ptr.* = casted_ptr;
43554348 return casted_ptr;
43564349 }
......@@ -4606,8 +4599,7 @@ pub const FuncGen = struct {
46064599
46074600 const ret_ptr = if (!sret) null else blk: {
46084601 const llvm_ret_ty = try self.dg.lowerType(return_type);
4609 const ret_ptr = self.buildAlloca(llvm_ret_ty);
4610 ret_ptr.setAlignment(return_type.abiAlignment(target));
4602 const ret_ptr = self.buildAlloca(llvm_ret_ty, return_type.abiAlignment(target));
46114603 try llvm_args.append(ret_ptr);
46124604 break :blk ret_ptr;
46134605 };
......@@ -4654,8 +4646,7 @@ pub const FuncGen = struct {
46544646 } else {
46554647 const alignment = param_ty.abiAlignment(target);
46564648 const param_llvm_ty = llvm_arg.typeOf();
4657 const arg_ptr = self.buildAlloca(param_llvm_ty);
4658 arg_ptr.setAlignment(alignment);
4649 const arg_ptr = self.buildAlloca(param_llvm_ty, alignment);
46594650 const store_inst = self.builder.buildStore(llvm_arg, arg_ptr);
46604651 store_inst.setAlignment(alignment);
46614652 try llvm_args.append(arg_ptr);
......@@ -4682,8 +4673,7 @@ pub const FuncGen = struct {
46824673 param_ty.abiAlignment(target),
46834674 self.dg.object.target_data.abiAlignmentOfType(int_llvm_ty),
46844675 );
4685 const int_ptr = self.buildAlloca(int_llvm_ty);
4686 int_ptr.setAlignment(alignment);
4676 const int_ptr = self.buildAlloca(int_llvm_ty, alignment);
46874677 const param_llvm_ty = try self.dg.lowerType(param_ty);
46884678 const casted_ptr = self.builder.buildBitCast(int_ptr, param_llvm_ty.pointerType(0), "");
46894679 const store_inst = self.builder.buildStore(llvm_arg, casted_ptr);
......@@ -4709,7 +4699,7 @@ pub const FuncGen = struct {
47094699 const llvm_arg = try self.resolveInst(arg);
47104700 const is_by_ref = isByRef(param_ty);
47114701 const arg_ptr = if (is_by_ref) llvm_arg else p: {
4712 const p = self.buildAlloca(llvm_arg.typeOf());
4702 const p = self.buildAlloca(llvm_arg.typeOf(), null);
47134703 const store_inst = self.builder.buildStore(llvm_arg, p);
47144704 store_inst.setAlignment(param_ty.abiAlignment(target));
47154705 break :p p;
......@@ -4738,7 +4728,7 @@ pub const FuncGen = struct {
47384728 const llvm_arg = try self.resolveInst(arg);
47394729 const is_by_ref = isByRef(param_ty);
47404730 const arg_ptr = if (is_by_ref) llvm_arg else p: {
4741 const p = self.buildAlloca(llvm_arg.typeOf());
4731 const p = self.buildAlloca(llvm_arg.typeOf(), null);
47424732 const store_inst = self.builder.buildStore(llvm_arg, p);
47434733 store_inst.setAlignment(param_ty.abiAlignment(target));
47444734 break :p p;
......@@ -4775,7 +4765,7 @@ pub const FuncGen = struct {
47754765 const arg_ty = self.air.typeOf(arg);
47764766 var llvm_arg = try self.resolveInst(arg);
47774767 if (!isByRef(arg_ty)) {
4778 const p = self.buildAlloca(llvm_arg.typeOf());
4768 const p = self.buildAlloca(llvm_arg.typeOf(), null);
47794769 const store_inst = self.builder.buildStore(llvm_arg, p);
47804770 store_inst.setAlignment(arg_ty.abiAlignment(target));
47814771 llvm_arg = store_inst;
......@@ -4832,9 +4822,8 @@ pub const FuncGen = struct {
48324822 // In this case the function return type is honoring the calling convention by having
48334823 // a different LLVM type than the usual one. We solve this here at the callsite
48344824 // by bitcasting a pointer to our canonical type, then loading it if necessary.
4835 const rp = self.buildAlloca(llvm_ret_ty);
48364825 const alignment = return_type.abiAlignment(target);
4837 rp.setAlignment(alignment);
4826 const rp = self.buildAlloca(llvm_ret_ty, alignment);
48384827 const ptr_abi_ty = abi_ret_ty.pointerType(0);
48394828 const casted_ptr = self.builder.buildBitCast(rp, ptr_abi_ty, "");
48404829 const store_inst = self.builder.buildStore(call, casted_ptr);
......@@ -4851,9 +4840,8 @@ pub const FuncGen = struct {
48514840 if (isByRef(return_type)) {
48524841 // our by-ref status disagrees with sret so we must allocate, store,
48534842 // and return the allocation pointer.
4854 const rp = self.buildAlloca(llvm_ret_ty);
48554843 const alignment = return_type.abiAlignment(target);
4856 rp.setAlignment(alignment);
4844 const rp = self.buildAlloca(llvm_ret_ty, alignment);
48574845 const store_inst = self.builder.buildStore(call, rp);
48584846 store_inst.setAlignment(alignment);
48594847 return rp;
......@@ -4912,8 +4900,7 @@ pub const FuncGen = struct {
49124900 return null;
49134901 }
49144902
4915 const rp = self.buildAlloca(llvm_ret_ty);
4916 rp.setAlignment(alignment);
4903 const rp = self.buildAlloca(llvm_ret_ty, alignment);
49174904 const store_inst = self.builder.buildStore(operand, rp);
49184905 store_inst.setAlignment(alignment);
49194906 const casted_ptr = self.builder.buildBitCast(rp, ptr_abi_ty, "");
......@@ -6031,8 +6018,7 @@ pub const FuncGen = struct {
60316018 llvm_param_types[llvm_param_i] = arg_llvm_value.typeOf();
60326019 } else {
60336020 const alignment = arg_ty.abiAlignment(target);
6034 const arg_ptr = self.buildAlloca(arg_llvm_value.typeOf());
6035 arg_ptr.setAlignment(alignment);
6021 const arg_ptr = self.buildAlloca(arg_llvm_value.typeOf(), alignment);
60366022 const store_inst = self.builder.buildStore(arg_llvm_value, arg_ptr);
60376023 store_inst.setAlignment(alignment);
60386024 llvm_param_values[llvm_param_i] = arg_ptr;
......@@ -6533,8 +6519,7 @@ pub const FuncGen = struct {
65336519 const llvm_optional_ty = try self.dg.lowerType(optional_ty);
65346520 if (isByRef(optional_ty)) {
65356521 const target = self.dg.module.getTarget();
6536 const optional_ptr = self.buildAlloca(llvm_optional_ty);
6537 optional_ptr.setAlignment(optional_ty.abiAlignment(target));
6522 const optional_ptr = self.buildAlloca(llvm_optional_ty, optional_ty.abiAlignment(target));
65386523 const payload_ptr = self.builder.buildStructGEP(llvm_optional_ty, optional_ptr, 0, "");
65396524 var ptr_ty_payload: Type.Payload.ElemType = .{
65406525 .base = .{ .tag = .single_mut_pointer },
......@@ -6567,8 +6552,7 @@ pub const FuncGen = struct {
65676552 const payload_offset = errUnionPayloadOffset(payload_ty, target);
65686553 const error_offset = errUnionErrorOffset(payload_ty, target);
65696554 if (isByRef(err_un_ty)) {
6570 const result_ptr = self.buildAlloca(err_un_llvm_ty);
6571 result_ptr.setAlignment(err_un_ty.abiAlignment(target));
6555 const result_ptr = self.buildAlloca(err_un_llvm_ty, err_un_ty.abiAlignment(target));
65726556 const err_ptr = self.builder.buildStructGEP(err_un_llvm_ty, result_ptr, error_offset, "");
65736557 const store_inst = self.builder.buildStore(ok_err_code, err_ptr);
65746558 store_inst.setAlignment(Type.anyerror.abiAlignment(target));
......@@ -6602,8 +6586,7 @@ pub const FuncGen = struct {
66026586 const payload_offset = errUnionPayloadOffset(payload_ty, target);
66036587 const error_offset = errUnionErrorOffset(payload_ty, target);
66046588 if (isByRef(err_un_ty)) {
6605 const result_ptr = self.buildAlloca(err_un_llvm_ty);
6606 result_ptr.setAlignment(err_un_ty.abiAlignment(target));
6589 const result_ptr = self.buildAlloca(err_un_llvm_ty, err_un_ty.abiAlignment(target));
66076590 const err_ptr = self.builder.buildStructGEP(err_un_llvm_ty, result_ptr, error_offset, "");
66086591 const store_inst = self.builder.buildStore(operand, err_ptr);
66096592 store_inst.setAlignment(Type.anyerror.abiAlignment(target));
......@@ -7021,9 +7004,8 @@ pub const FuncGen = struct {
70217004
70227005 if (isByRef(dest_ty)) {
70237006 const target = self.dg.module.getTarget();
7024 const alloca_inst = self.buildAlloca(llvm_dest_ty);
70257007 const result_alignment = dest_ty.abiAlignment(target);
7026 alloca_inst.setAlignment(result_alignment);
7008 const alloca_inst = self.buildAlloca(llvm_dest_ty, result_alignment);
70277009 {
70287010 const field_ptr = self.builder.buildStructGEP(llvm_dest_ty, alloca_inst, result_index, "");
70297011 const store_inst = self.builder.buildStore(result, field_ptr);
......@@ -7373,9 +7355,8 @@ pub const FuncGen = struct {
73737355
73747356 if (isByRef(dest_ty)) {
73757357 const target = self.dg.module.getTarget();
7376 const alloca_inst = self.buildAlloca(llvm_dest_ty);
73777358 const result_alignment = dest_ty.abiAlignment(target);
7378 alloca_inst.setAlignment(result_alignment);
7359 const alloca_inst = self.buildAlloca(llvm_dest_ty, result_alignment);
73797360 {
73807361 const field_ptr = self.builder.buildStructGEP(llvm_dest_ty, alloca_inst, result_index, "");
73817362 const store_inst = self.builder.buildStore(result, field_ptr);
......@@ -7653,7 +7634,7 @@ pub const FuncGen = struct {
76537634 if (!result_is_ref) {
76547635 return self.dg.todo("implement bitcast vector to non-ref array", .{});
76557636 }
7656 const array_ptr = self.buildAlloca(llvm_dest_ty);
7637 const array_ptr = self.buildAlloca(llvm_dest_ty, null);
76577638 const bitcast_ok = elem_ty.bitSize(target) == elem_ty.abiSize(target) * 8;
76587639 if (bitcast_ok) {
76597640 const llvm_vector_ty = try self.dg.lowerType(operand_ty);
......@@ -7729,8 +7710,7 @@ pub const FuncGen = struct {
77297710 if (result_is_ref) {
77307711 // Bitcast the result pointer, then store.
77317712 const alignment = @maximum(operand_ty.abiAlignment(target), inst_ty.abiAlignment(target));
7732 const result_ptr = self.buildAlloca(llvm_dest_ty);
7733 result_ptr.setAlignment(alignment);
7713 const result_ptr = self.buildAlloca(llvm_dest_ty, alignment);
77347714 const operand_llvm_ty = try self.dg.lowerType(operand_ty);
77357715 const casted_ptr = self.builder.buildBitCast(result_ptr, operand_llvm_ty.pointerType(0), "");
77367716 const store_inst = self.builder.buildStore(operand, casted_ptr);
......@@ -7743,8 +7723,7 @@ pub const FuncGen = struct {
77437723 // but LLVM won't let us bitcast struct values.
77447724 // Therefore, we store operand to bitcasted alloca, then load for result.
77457725 const alignment = @maximum(operand_ty.abiAlignment(target), inst_ty.abiAlignment(target));
7746 const result_ptr = self.buildAlloca(llvm_dest_ty);
7747 result_ptr.setAlignment(alignment);
7726 const result_ptr = self.buildAlloca(llvm_dest_ty, alignment);
77487727 const operand_llvm_ty = try self.dg.lowerType(operand_ty);
77497728 const casted_ptr = self.builder.buildBitCast(result_ptr, operand_llvm_ty.pointerType(0), "");
77507729 const store_inst = self.builder.buildStore(operand, casted_ptr);
......@@ -7820,11 +7799,9 @@ pub const FuncGen = struct {
78207799 if (!pointee_type.isFnOrHasRuntimeBitsIgnoreComptime()) return self.dg.lowerPtrToVoid(ptr_ty);
78217800
78227801 const pointee_llvm_ty = try self.dg.lowerType(pointee_type);
7823 const alloca_inst = self.buildAlloca(pointee_llvm_ty);
78247802 const target = self.dg.module.getTarget();
78257803 const alignment = ptr_ty.ptrAlignment(target);
7826 alloca_inst.setAlignment(alignment);
7827 return alloca_inst;
7804 return self.buildAlloca(pointee_llvm_ty, alignment);
78287805 }
78297806
78307807 fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {
......@@ -7835,15 +7812,13 @@ pub const FuncGen = struct {
78357812 if (self.ret_ptr) |ret_ptr| return ret_ptr;
78367813 const ret_llvm_ty = try self.dg.lowerType(ret_ty);
78377814 const target = self.dg.module.getTarget();
7838 const alloca_inst = self.buildAlloca(ret_llvm_ty);
7839 alloca_inst.setAlignment(ptr_ty.ptrAlignment(target));
7840 return alloca_inst;
7815 return self.buildAlloca(ret_llvm_ty, ptr_ty.ptrAlignment(target));
78417816 }
78427817
78437818 /// Use this instead of builder.buildAlloca, because this function makes sure to
78447819 /// put the alloca instruction at the top of the function!
7845 fn buildAlloca(self: *FuncGen, llvm_ty: *llvm.Type) *llvm.Value {
7846 return buildAllocaInner(self.builder, self.llvm_func, self.di_scope != null, llvm_ty);
7820 fn buildAlloca(self: *FuncGen, llvm_ty: *llvm.Type, alignment: ?c_uint) *llvm.Value {
7821 return buildAllocaInner(self.builder, self.llvm_func, self.di_scope != null, llvm_ty, alignment, self.dg.module.getTarget());
78477822 }
78487823
78497824 fn airStore(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {
......@@ -8801,10 +8776,9 @@ pub const FuncGen = struct {
88018776
88028777 if (isByRef(result_ty)) {
88038778 const llvm_u32 = self.context.intType(32);
8804 const alloca_inst = self.buildAlloca(llvm_result_ty);
88058779 // TODO in debug builds init to undef so that the padding will be 0xaa
88068780 // even if we fully populate the fields.
8807 alloca_inst.setAlignment(result_ty.abiAlignment(target));
8781 const alloca_inst = self.buildAlloca(llvm_result_ty, result_ty.abiAlignment(target));
88088782
88098783 var indices: [2]*llvm.Value = .{ llvm_u32.constNull(), undefined };
88108784 for (elements) |elem, i| {
......@@ -8842,8 +8816,7 @@ pub const FuncGen = struct {
88428816 assert(isByRef(result_ty));
88438817
88448818 const llvm_usize = try self.dg.lowerType(Type.usize);
8845 const alloca_inst = self.buildAlloca(llvm_result_ty);
8846 alloca_inst.setAlignment(result_ty.abiAlignment(target));
8819 const alloca_inst = self.buildAlloca(llvm_result_ty, result_ty.abiAlignment(target));
88478820
88488821 const array_info = result_ty.arrayInfo();
88498822 var elem_ptr_payload: Type.Payload.Pointer = .{
......@@ -8918,7 +8891,7 @@ pub const FuncGen = struct {
89188891 // necessarily match the format that we need, depending on which tag is active. We
89198892 // must construct the correct unnamed struct type here and bitcast, in order to
89208893 // then set the fields appropriately.
8921 const result_ptr = self.buildAlloca(union_llvm_ty);
8894 const result_ptr = self.buildAlloca(union_llvm_ty, null);
89228895 const llvm_payload = try self.resolveInst(extra.init);
89238896 assert(union_obj.haveFieldTypes());
89248897 const field = union_obj.fields.values()[extra.field_index];
......@@ -9234,9 +9207,8 @@ pub const FuncGen = struct {
92349207
92359208 if (isByRef(optional_ty)) {
92369209 const target = self.dg.module.getTarget();
9237 const alloca_inst = self.buildAlloca(optional_llvm_ty);
92389210 const payload_alignment = optional_ty.abiAlignment(target);
9239 alloca_inst.setAlignment(payload_alignment);
9211 const alloca_inst = self.buildAlloca(optional_llvm_ty, payload_alignment);
92409212
92419213 {
92429214 const field_ptr = self.builder.buildStructGEP(optional_llvm_ty, alloca_inst, 0, "");
......@@ -9360,8 +9332,7 @@ pub const FuncGen = struct {
93609332 if (isByRef(info.pointee_type)) {
93619333 const result_align = info.pointee_type.abiAlignment(target);
93629334 const max_align = @maximum(result_align, ptr_alignment);
9363 const result_ptr = self.buildAlloca(elem_llvm_ty);
9364 result_ptr.setAlignment(max_align);
9335 const result_ptr = self.buildAlloca(elem_llvm_ty, max_align);
93659336 const llvm_ptr_u8 = self.context.intType(8).pointerType(0);
93669337 const llvm_usize = self.context.intType(Type.usize.intInfo(target).bits);
93679338 const size_bytes = info.pointee_type.abiSize(target);
......@@ -9394,8 +9365,7 @@ pub const FuncGen = struct {
93949365
93959366 if (isByRef(info.pointee_type)) {
93969367 const result_align = info.pointee_type.abiAlignment(target);
9397 const result_ptr = self.buildAlloca(elem_llvm_ty);
9398 result_ptr.setAlignment(result_align);
9368 const result_ptr = self.buildAlloca(elem_llvm_ty, result_align);
93999369
94009370 const same_size_int = self.context.intType(elem_bits);
94019371 const truncated_int = self.builder.buildTrunc(shifted_value, same_size_int, "");
......@@ -9519,8 +9489,7 @@ pub const FuncGen = struct {
95199489 .x86_64 => {
95209490 const array_llvm_ty = usize_llvm_ty.arrayType(6);
95219491 const array_ptr = fg.valgrind_client_request_array orelse a: {
9522 const array_ptr = fg.buildAlloca(array_llvm_ty);
9523 array_ptr.setAlignment(usize_alignment);
9492 const array_ptr = fg.buildAlloca(array_llvm_ty, usize_alignment);
95249493 fg.valgrind_client_request_array = array_ptr;
95259494 break :a array_ptr;
95269495 };
......@@ -9822,6 +9791,74 @@ fn toLlvmCallConv(cc: std.builtin.CallingConvention, target: std.Target) llvm.Ca
98229791 };
98239792}
98249793
9794/// Convert a zig-address space to an llvm address space.
9795fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: std.Target) c_uint {
9796 return switch (target.cpu.arch) {
9797 .i386, .x86_64 => switch (address_space) {
9798 .generic => llvm.address_space.default,
9799 .gs => llvm.address_space.x86.gs,
9800 .fs => llvm.address_space.x86.fs,
9801 .ss => llvm.address_space.x86.ss,
9802 else => unreachable,
9803 },
9804 .nvptx, .nvptx64 => switch (address_space) {
9805 .generic => llvm.address_space.default,
9806 .global => llvm.address_space.nvptx.global,
9807 .constant => llvm.address_space.nvptx.constant,
9808 .param => llvm.address_space.nvptx.param,
9809 .shared => llvm.address_space.nvptx.shared,
9810 .local => llvm.address_space.nvptx.local,
9811 else => unreachable,
9812 },
9813 .amdgcn => switch (address_space) {
9814 .generic => llvm.address_space.amdgpu.flat,
9815 .global => llvm.address_space.amdgpu.global,
9816 .constant => llvm.address_space.amdgpu.constant,
9817 .shared => llvm.address_space.amdgpu.local,
9818 .local => llvm.address_space.amdgpu.private,
9819 else => unreachable,
9820 },
9821 else => switch (address_space) {
9822 .generic => llvm.address_space.default,
9823 else => unreachable,
9824 },
9825 };
9826}
9827
9828/// On some targets, local values that are in the generic address space must be generated into a
9829/// different address, space and then cast back to the generic address space.
9830/// For example, on GPUs local variable declarations must be generated into the local address space.
9831/// This function returns the address space local values should be generated into.
9832fn llvmAllocaAddressSpace(target: std.Target) c_uint {
9833 return switch (target.cpu.arch) {
9834 // On amdgcn, locals should be generated into the private address space.
9835 // To make Zig not impossible to use, these are then converted to addresses in the
9836 // generic address space and treates as regular pointers. This is the way that HIP also does it.
9837 .amdgcn => llvm.address_space.amdgpu.private,
9838 else => llvm.address_space.default,
9839 };
9840}
9841
9842/// On some targets, global values that are in the generic address space must be generated into a
9843/// different address space, and then cast back to the generic address space.
9844fn llvmDefaultGlobalAddressSpace(target: std.Target) c_uint {
9845 return switch (target.cpu.arch) {
9846 // On amdgcn, globals must be explicitly allocated and uploaded so that the program can access
9847 // them.
9848 .amdgcn => llvm.address_space.amdgpu.global,
9849 else => llvm.address_space.default,
9850 };
9851}
9852
9853/// If `llvm_addrspace` is generic, convert it to the actual address space that globals
9854/// should be stored in by default.
9855fn toLlvmGlobalAddressSpace(llvm_addrspace: c_uint, target: std.Target) c_uint {
9856 return if (llvm_addrspace == llvm.address_space.default)
9857 llvmDefaultGlobalAddressSpace(target)
9858 else
9859 llvm_addrspace;
9860}
9861
98259862/// Take into account 0 bit fields and padding. Returns null if an llvm
98269863/// field could not be found.
98279864/// This only happens if you want the field index of a zero sized field at
......@@ -10523,25 +10560,43 @@ fn buildAllocaInner(
1052310560 llvm_func: *llvm.Value,
1052410561 di_scope_non_null: bool,
1052510562 llvm_ty: *llvm.Type,
10563 maybe_alignment: ?c_uint,
10564 target: std.Target,
1052610565) *llvm.Value {
10527 const prev_block = builder.getInsertBlock();
10528 const prev_debug_location = builder.getCurrentDebugLocation2();
10529 defer {
10530 builder.positionBuilderAtEnd(prev_block);
10531 if (di_scope_non_null) {
10532 builder.setCurrentDebugLocation2(prev_debug_location);
10566 const address_space = llvmAllocaAddressSpace(target);
10567
10568 const alloca = blk: {
10569 const prev_block = builder.getInsertBlock();
10570 const prev_debug_location = builder.getCurrentDebugLocation2();
10571 defer {
10572 builder.positionBuilderAtEnd(prev_block);
10573 if (di_scope_non_null) {
10574 builder.setCurrentDebugLocation2(prev_debug_location);
10575 }
10576 }
10577
10578 const entry_block = llvm_func.getFirstBasicBlock().?;
10579 if (entry_block.getFirstInstruction()) |first_inst| {
10580 builder.positionBuilder(entry_block, first_inst);
10581 } else {
10582 builder.positionBuilderAtEnd(entry_block);
1053310583 }
10584 builder.clearCurrentDebugLocation();
10585
10586 break :blk builder.buildAllocaInAddressSpace(llvm_ty, address_space, "");
10587 };
10588
10589 if (maybe_alignment) |alignment| {
10590 alloca.setAlignment(alignment);
1053410591 }
1053510592
10536 const entry_block = llvm_func.getFirstBasicBlock().?;
10537 if (entry_block.getFirstInstruction()) |first_inst| {
10538 builder.positionBuilder(entry_block, first_inst);
10539 } else {
10540 builder.positionBuilderAtEnd(entry_block);
10593 // The pointer returned from this function should have the generic address space,
10594 // if this isn't the case then cast it to the generic address space.
10595 if (address_space != llvm.address_space.default) {
10596 return builder.buildAddrSpaceCast(alloca, llvm_ty.pointerType(llvm.address_space.default), "");
1054110597 }
10542 builder.clearCurrentDebugLocation();
1054310598
10544 return builder.buildAlloca(llvm_ty, "");
10599 return alloca;
1054510600}
1054610601
1054710602fn errUnionPayloadOffset(payload_ty: Type, target: std.Target) u1 {
src/codegen/llvm/bindings.zig+9
......@@ -171,6 +171,9 @@ pub const Value = opaque {
171171 pub const constAdd = LLVMConstAdd;
172172 extern fn LLVMConstAdd(LHSConstant: *Value, RHSConstant: *Value) *Value;
173173
174 pub const constAddrSpaceCast = LLVMConstAddrSpaceCast;
175 extern fn LLVMConstAddrSpaceCast(ConstantVal: *Value, ToType: *Type) *Value;
176
174177 pub const setWeak = LLVMSetWeak;
175178 extern fn LLVMSetWeak(CmpXchgInst: *Value, IsWeak: Bool) void;
176179
......@@ -956,6 +959,12 @@ pub const Builder = opaque {
956959
957960 pub const setFastMath = ZigLLVMSetFastMath;
958961 extern fn ZigLLVMSetFastMath(B: *Builder, on_state: bool) void;
962
963 pub const buildAddrSpaceCast = LLVMBuildAddrSpaceCast;
964 extern fn LLVMBuildAddrSpaceCast(B: *Builder, Val: *Value, DestTy: *Type, Name: [*:0]const u8) *Value;
965
966 pub const buildAllocaInAddressSpace = ZigLLVMBuildAllocaInAddressSpace;
967 extern fn ZigLLVMBuildAllocaInAddressSpace(B: *Builder, Ty: *Type, AddressSpace: c_uint, Name: [*:0]const u8) *Value;
959968};
960969
961970pub const MDString = opaque {
src/zig_llvm.cpp+11-6
......@@ -512,22 +512,22 @@ LLVMValueRef ZigLLVMBuildUSubSat(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRe
512512
513513LLVMValueRef ZigLLVMBuildSMulFixSat(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS, const char *name) {
514514 llvm::Type* types[1] = {
515 unwrap(LHS)->getType(),
515 unwrap(LHS)->getType(),
516516 };
517517 // pass scale = 0 as third argument
518518 llvm::Value* values[3] = {unwrap(LHS), unwrap(RHS), unwrap(B)->getInt32(0)};
519
519
520520 CallInst *call_inst = unwrap(B)->CreateIntrinsic(Intrinsic::smul_fix_sat, types, values, nullptr, name);
521521 return wrap(call_inst);
522522}
523523
524524LLVMValueRef ZigLLVMBuildUMulFixSat(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS, const char *name) {
525525 llvm::Type* types[1] = {
526 unwrap(LHS)->getType(),
526 unwrap(LHS)->getType(),
527527 };
528528 // pass scale = 0 as third argument
529529 llvm::Value* values[3] = {unwrap(LHS), unwrap(RHS), unwrap(B)->getInt32(0)};
530
530
531531 CallInst *call_inst = unwrap(B)->CreateIntrinsic(Intrinsic::umul_fix_sat, types, values, nullptr, name);
532532 return wrap(call_inst);
533533}
......@@ -808,7 +808,7 @@ void ZigLLVMSetCurrentDebugLocation2(LLVMBuilderRef builder, unsigned int line,
808808 unsigned int column, ZigLLVMDIScope *scope, ZigLLVMDILocation *inlined_at)
809809{
810810 DIScope* di_scope = reinterpret_cast<DIScope*>(scope);
811 DebugLoc debug_loc = DILocation::get(di_scope->getContext(), line, column, di_scope,
811 DebugLoc debug_loc = DILocation::get(di_scope->getContext(), line, column, di_scope,
812812 reinterpret_cast<DILocation *>(inlined_at), false);
813813 unwrap(builder)->SetCurrentDebugLocation(debug_loc);
814814}
......@@ -1177,9 +1177,14 @@ LLVMValueRef ZigLLVMBuildAShrExact(LLVMBuilderRef builder, LLVMValueRef LHS, LLV
11771177 return wrap(unwrap(builder)->CreateAShr(unwrap(LHS), unwrap(RHS), name, true));
11781178}
11791179
1180LLVMValueRef ZigLLVMBuildAllocaInAddressSpace(LLVMBuilderRef builder, LLVMTypeRef Ty,
1181 unsigned AddressSpace, const char *Name) {
1182 return wrap(unwrap(builder)->CreateAlloca(unwrap(Ty), AddressSpace, nullptr, Name));
1183}
1184
11801185void ZigLLVMSetTailCall(LLVMValueRef Call) {
11811186 unwrap<CallInst>(Call)->setTailCallKind(CallInst::TCK_MustTail);
1182}
1187}
11831188
11841189void ZigLLVMSetCallSret(LLVMValueRef Call, LLVMTypeRef return_type) {
11851190 CallInst *call_inst = unwrap<CallInst>(Call);
src/zig_llvm.h+2
......@@ -162,6 +162,8 @@ ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildLShrExact(LLVMBuilderRef builder, LLVMValu
162162ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildAShrExact(LLVMBuilderRef builder, LLVMValueRef LHS, LLVMValueRef RHS,
163163 const char *name);
164164
165ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildAllocaInAddressSpace(LLVMBuilderRef builder, LLVMTypeRef Ty, unsigned AddressSpace,
166 const char *Name);
165167
166168ZIG_EXTERN_C struct ZigLLVMDIType *ZigLLVMCreateDebugPointerType(struct ZigLLVMDIBuilder *dibuilder,
167169 struct ZigLLVMDIType *pointee_type, uint64_t size_in_bits, uint64_t align_in_bits, const char *name);