authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-12-15 20:34:26+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-12-15 21:06:35-05:00
log8a0a6b7387fcd0017db85de14793abfd6ec7f6e5
tree75382432d967dd5cff1fa5aaded940a627b8df7b
parent0d92fcf6a503780dcaadccef87e72824c7942a96

port packed vector elem ptr logic from stage1

Closes #12812 Closes #13925

18 files changed, 257 insertions(+), 60 deletions(-)

src/Air.zig+10
...@@ -737,6 +737,10 @@ pub const Inst = struct {...@@ -737,6 +737,10 @@ pub const Inst = struct {
737 /// Uses the `ty_pl` field.737 /// Uses the `ty_pl` field.
738 save_err_return_trace_index,738 save_err_return_trace_index,
739739
740 /// Store an element to a vector pointer at an index.
741 /// Uses the `vector_store_elem` field.
742 vector_store_elem,
743
740 pub fn fromCmpOp(op: std.math.CompareOperator, optimized: bool) Tag {744 pub fn fromCmpOp(op: std.math.CompareOperator, optimized: bool) Tag {
741 switch (op) {745 switch (op) {
742 .lt => return if (optimized) .cmp_lt_optimized else .cmp_lt,746 .lt => return if (optimized) .cmp_lt_optimized else .cmp_lt,
...@@ -814,6 +818,11 @@ pub const Inst = struct {...@@ -814,6 +818,11 @@ pub const Inst = struct {
814 operand: Ref,818 operand: Ref,
815 operation: std.builtin.ReduceOp,819 operation: std.builtin.ReduceOp,
816 },820 },
821 vector_store_elem: struct {
822 vector_ptr: Ref,
823 // Index into a different array.
824 payload: u32,
825 },
817826
818 // Make sure we don't accidentally add a field to make this union827 // Make sure we don't accidentally add a field to make this union
819 // bigger than expected. Note that in Debug builds, Zig is allowed828 // bigger than expected. Note that in Debug builds, Zig is allowed
...@@ -1177,6 +1186,7 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {...@@ -1177,6 +1186,7 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
1177 .set_union_tag,1186 .set_union_tag,
1178 .prefetch,1187 .prefetch,
1179 .set_err_return_trace,1188 .set_err_return_trace,
1189 .vector_store_elem,
1180 => return Type.void,1190 => return Type.void,
11811191
1182 .ptrtoint,1192 .ptrtoint,
src/Liveness.zig+15
...@@ -212,6 +212,15 @@ pub fn categorizeOperand(...@@ -212,6 +212,15 @@ pub fn categorizeOperand(
212 return .write;212 return .write;
213 },213 },
214214
215 .vector_store_elem => {
216 const o = air_datas[inst].vector_store_elem;
217 const extra = air.extraData(Air.Bin, o.payload).data;
218 if (o.vector_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
219 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
220 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
221 return .write;
222 },
223
215 .arg,224 .arg,
216 .alloc,225 .alloc,
217 .ret_ptr,226 .ret_ptr,
...@@ -824,6 +833,12 @@ fn analyzeInst(...@@ -824,6 +833,12 @@ fn analyzeInst(
824 return trackOperands(a, new_set, inst, main_tomb, .{ o.lhs, o.rhs, .none });833 return trackOperands(a, new_set, inst, main_tomb, .{ o.lhs, o.rhs, .none });
825 },834 },
826835
836 .vector_store_elem => {
837 const o = inst_datas[inst].vector_store_elem;
838 const extra = a.air.extraData(Air.Bin, o.payload).data;
839 return trackOperands(a, new_set, inst, main_tomb, .{ o.vector_ptr, extra.lhs, extra.rhs });
840 },
841
827 .arg,842 .arg,
828 .alloc,843 .alloc,
829 .ret_ptr,844 .ret_ptr,
src/Sema.zig+51-13
...@@ -25952,6 +25952,30 @@ fn storePtr2(...@@ -25952,6 +25952,30 @@ fn storePtr2(
2595225952
25953 try sema.requireRuntimeBlock(block, src, runtime_src);25953 try sema.requireRuntimeBlock(block, src, runtime_src);
25954 try sema.queueFullTypeResolution(elem_ty);25954 try sema.queueFullTypeResolution(elem_ty);
25955
25956 if (ptr_ty.ptrInfo().data.vector_index == .runtime) {
25957 const ptr_inst = Air.refToIndex(ptr).?;
25958 const air_tags = sema.air_instructions.items(.tag);
25959 if (air_tags[ptr_inst] == .ptr_elem_ptr) {
25960 const ty_pl = sema.air_instructions.items(.data)[ptr_inst].ty_pl;
25961 const bin_op = sema.getTmpAir().extraData(Air.Bin, ty_pl.payload).data;
25962 _ = try block.addInst(.{
25963 .tag = .vector_store_elem,
25964 .data = .{ .vector_store_elem = .{
25965 .vector_ptr = bin_op.lhs,
25966 .payload = try block.sema.addExtra(Air.Bin{
25967 .lhs = bin_op.rhs,
25968 .rhs = operand,
25969 }),
25970 } },
25971 });
25972 return;
25973 }
25974 return sema.fail(block, ptr_src, "unable to determine vector element index of type '{}'", .{
25975 ptr_ty.fmt(sema.mod),
25976 });
25977 }
25978
25955 if (is_ret) {25979 if (is_ret) {
25956 _ = try block.addBinOp(.store, ptr, operand);25980 _ = try block.addBinOp(.store, ptr, operand);
25957 } else {25981 } else {
...@@ -27827,6 +27851,19 @@ fn analyzeLoad(...@@ -27827,6 +27851,19 @@ fn analyzeLoad(
27827 }27851 }
27828 }27852 }
2782927853
27854 if (ptr_ty.ptrInfo().data.vector_index == .runtime) {
27855 const ptr_inst = Air.refToIndex(ptr).?;
27856 const air_tags = sema.air_instructions.items(.tag);
27857 if (air_tags[ptr_inst] == .ptr_elem_ptr) {
27858 const ty_pl = sema.air_instructions.items(.data)[ptr_inst].ty_pl;
27859 const bin_op = sema.getTmpAir().extraData(Air.Bin, ty_pl.payload).data;
27860 return block.addBinOp(.ptr_elem_val, bin_op.lhs, bin_op.rhs);
27861 }
27862 return sema.fail(block, ptr_src, "unable to determine vector element index of type '{}'", .{
27863 ptr_ty.fmt(sema.mod),
27864 });
27865 }
27866
27830 return block.addTyOp(.load, elem_ty, ptr);27867 return block.addTyOp(.load, elem_ty, ptr);
27831}27868}
2783227869
...@@ -32697,23 +32734,24 @@ fn elemPtrType(sema: *Sema, ptr_ty: Type, offset: ?usize) !Type {...@@ -32697,23 +32734,24 @@ fn elemPtrType(sema: *Sema, ptr_ty: Type, offset: ?usize) !Type {
32697 const target = sema.mod.getTarget();32734 const target = sema.mod.getTarget();
32698 const parent_ty = ptr_ty.childType();32735 const parent_ty = ptr_ty.childType();
3269932736
32737 const VI = Type.Payload.Pointer.Data.VectorIndex;
32738
32700 const vector_info: struct {32739 const vector_info: struct {
32701 host_size: u16,32740 host_size: u16 = 0,
32702 bit_offset: u16,32741 alignment: u32 = 0,
32703 alignment: u32,32742 vector_index: VI = .none,
32704 } = if (parent_ty.tag() == .vector) blk: {32743 } = if (parent_ty.tag() == .vector) blk: {
32705 const elem_bits = elem_ty.bitSize(target);32744 const elem_bits = elem_ty.bitSize(target);
32706 const is_packed = elem_bits != 0 and (elem_bits & (elem_bits - 1)) != 0;32745 if (elem_bits == 0) break :blk .{};
32707 // TODO: runtime-known index32746 const is_packed = elem_bits < 8 or !std.math.isPowerOfTwo(elem_bits);
32708 assert(!is_packed or offset != null);32747 if (!is_packed) break :blk .{};
32709 const is_packed_with_offset = is_packed and offset != null and offset.? != 0;32748
32710 const target_offset = if (is_packed_with_offset) (if (target.cpu.arch.endian() == .Big) (parent_ty.vectorLen() - 1 - offset.?) else offset.?) else 0;
32711 break :blk .{32749 break :blk .{
32712 .host_size = if (is_packed_with_offset) @intCast(u16, parent_ty.abiSize(target)) else 0,32750 .host_size = @intCast(u16, parent_ty.arrayLen()),
32713 .bit_offset = if (is_packed_with_offset) @intCast(u16, elem_bits * target_offset) else 0,32751 .alignment = @intCast(u16, parent_ty.abiAlignment(target)),
32714 .alignment = if (is_packed_with_offset) @intCast(u16, parent_ty.abiAlignment(target)) else 0,32752 .vector_index = if (offset) |some| @intToEnum(VI, some) else .runtime,
32715 };32753 };
32716 } else .{ .host_size = 0, .bit_offset = 0, .alignment = 0 };32754 } else .{};
3271732755
32718 const alignment: u32 = a: {32756 const alignment: u32 = a: {
32719 // Calculate the new pointer alignment.32757 // Calculate the new pointer alignment.
...@@ -32741,6 +32779,6 @@ fn elemPtrType(sema: *Sema, ptr_ty: Type, offset: ?usize) !Type {...@@ -32741,6 +32779,6 @@ fn elemPtrType(sema: *Sema, ptr_ty: Type, offset: ?usize) !Type {
32741 .@"volatile" = ptr_info.@"volatile",32779 .@"volatile" = ptr_info.@"volatile",
32742 .@"align" = alignment,32780 .@"align" = alignment,
32743 .host_size = vector_info.host_size,32781 .host_size = vector_info.host_size,
32744 .bit_offset = vector_info.bit_offset,32782 .vector_index = vector_info.vector_index,
32745 });32783 });
32746}32784}
src/arch/aarch64/CodeGen.zig+1
...@@ -873,6 +873,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -873,6 +873,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
873873
874 .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),874 .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),
875 .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),875 .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),
876 .vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),
876877
877 .wasm_memory_size => unreachable,878 .wasm_memory_size => unreachable,
878 .wasm_memory_grow => unreachable,879 .wasm_memory_grow => unreachable,
src/arch/arm/CodeGen.zig+1
...@@ -783,6 +783,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -783,6 +783,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
783783
784 .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),784 .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),
785 .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),785 .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),
786 .vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),
786787
787 .wasm_memory_size => unreachable,788 .wasm_memory_size => unreachable,
788 .wasm_memory_grow => unreachable,789 .wasm_memory_grow => unreachable,
src/arch/riscv64/CodeGen.zig+1
...@@ -697,6 +697,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -697,6 +697,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
697697
698 .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),698 .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),
699 .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),699 .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),
700 .vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),
700701
701 .wasm_memory_size => unreachable,702 .wasm_memory_size => unreachable,
702 .wasm_memory_grow => unreachable,703 .wasm_memory_grow => unreachable,
src/arch/sparc64/CodeGen.zig+1
...@@ -714,6 +714,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -714,6 +714,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
714714
715 .is_named_enum_value => @panic("TODO implement is_named_enum_value"),715 .is_named_enum_value => @panic("TODO implement is_named_enum_value"),
716 .error_set_has_value => @panic("TODO implement error_set_has_value"),716 .error_set_has_value => @panic("TODO implement error_set_has_value"),
717 .vector_store_elem => @panic("TODO implement vector_store_elem"),
717718
718 .wasm_memory_size => unreachable,719 .wasm_memory_size => unreachable,
719 .wasm_memory_grow => unreachable,720 .wasm_memory_grow => unreachable,
src/arch/wasm/CodeGen.zig+2
...@@ -1971,6 +1971,7 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1971,6 +1971,7 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1971 .is_named_enum_value,1971 .is_named_enum_value,
1972 .error_set_has_value,1972 .error_set_has_value,
1973 .addrspace_cast,1973 .addrspace_cast,
1974 .vector_store_elem,
1974 => |tag| return func.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),1975 => |tag| return func.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),
19751976
1976 .add_optimized,1977 .add_optimized,
...@@ -2213,6 +2214,7 @@ fn airStore(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2213,6 +2214,7 @@ fn airStore(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2213 const ptr_ty = func.air.typeOf(bin_op.lhs);2214 const ptr_ty = func.air.typeOf(bin_op.lhs);
2214 const ptr_info = ptr_ty.ptrInfo().data;2215 const ptr_info = ptr_ty.ptrInfo().data;
2215 const ty = ptr_ty.childType();2216 const ty = ptr_ty.childType();
2217
2216 if (ptr_info.host_size == 0) {2218 if (ptr_info.host_size == 0) {
2217 try func.store(lhs, rhs, ty, 0);2219 try func.store(lhs, rhs, ty, 0);
2218 } else {2220 } else {
src/arch/x86_64/CodeGen.zig+1
...@@ -785,6 +785,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -785,6 +785,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
785785
786 .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),786 .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),
787 .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),787 .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),
788 .vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),
788789
789 .wasm_memory_size => unreachable,790 .wasm_memory_size => unreachable,
790 .wasm_memory_grow => unreachable,791 .wasm_memory_grow => unreachable,
src/codegen/c.zig+1
...@@ -2908,6 +2908,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,...@@ -2908,6 +2908,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
29082908
2909 .is_named_enum_value => return f.fail("TODO: C backend: implement is_named_enum_value", .{}),2909 .is_named_enum_value => return f.fail("TODO: C backend: implement is_named_enum_value", .{}),
2910 .error_set_has_value => return f.fail("TODO: C backend: implement error_set_has_value", .{}),2910 .error_set_has_value => return f.fail("TODO: C backend: implement error_set_has_value", .{}),
2911 .vector_store_elem => return f.fail("TODO: C backend: implement vector_store_elem", .{}),
2911 // zig fmt: on2912 // zig fmt: on
2912 };2913 };
2913 if (result_value == .local) {2914 if (result_value == .local) {
src/codegen/llvm.zig+74-11
...@@ -1572,6 +1572,7 @@ pub const Object = struct {...@@ -1572,6 +1572,7 @@ pub const Object = struct {
1572 ptr_info.@"addrspace" != .generic or1572 ptr_info.@"addrspace" != .generic or
1573 ptr_info.bit_offset != 0 or1573 ptr_info.bit_offset != 0 or
1574 ptr_info.host_size != 0 or1574 ptr_info.host_size != 0 or
1575 ptr_info.vector_index != .none or
1575 ptr_info.@"allowzero" or1576 ptr_info.@"allowzero" or
1576 !ptr_info.mutable or1577 !ptr_info.mutable or
1577 ptr_info.@"volatile" or1578 ptr_info.@"volatile" or
...@@ -4660,6 +4661,8 @@ pub const FuncGen = struct {...@@ -4660,6 +4661,8 @@ pub const FuncGen = struct {
4660 .wasm_memory_size => try self.airWasmMemorySize(inst),4661 .wasm_memory_size => try self.airWasmMemorySize(inst),
4661 .wasm_memory_grow => try self.airWasmMemoryGrow(inst),4662 .wasm_memory_grow => try self.airWasmMemoryGrow(inst),
46624663
4664 .vector_store_elem => try self.airVectorStoreElem(inst),
4665
4663 .constant => unreachable,4666 .constant => unreachable,
4664 .const_ty => unreachable,4667 .const_ty => unreachable,
4665 .unreach => self.airUnreach(inst),4668 .unreach => self.airUnreach(inst),
...@@ -5022,7 +5025,7 @@ pub const FuncGen = struct {...@@ -5022,7 +5025,7 @@ pub const FuncGen = struct {
5022 .data = ret_ty,5025 .data = ret_ty,
5023 };5026 };
5024 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);5027 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
5025 self.store(ret_ptr, ptr_ty, operand, .NotAtomic);5028 try self.store(ret_ptr, ptr_ty, operand, .NotAtomic);
5026 _ = self.builder.buildRetVoid();5029 _ = self.builder.buildRetVoid();
5027 return null;5030 return null;
5028 }5031 }
...@@ -5779,6 +5782,10 @@ pub const FuncGen = struct {...@@ -5779,6 +5782,10 @@ pub const FuncGen = struct {
57795782
5780 const base_ptr = try self.resolveInst(bin_op.lhs);5783 const base_ptr = try self.resolveInst(bin_op.lhs);
5781 const rhs = try self.resolveInst(bin_op.rhs);5784 const rhs = try self.resolveInst(bin_op.rhs);
5785
5786 const elem_ptr = self.air.getRefType(ty_pl.ty);
5787 if (elem_ptr.ptrInfo().data.vector_index != .none) return base_ptr;
5788
5782 const llvm_elem_ty = try self.dg.lowerPtrElemTy(elem_ty);5789 const llvm_elem_ty = try self.dg.lowerPtrElemTy(elem_ty);
5783 if (ptr_ty.isSinglePointer()) {5790 if (ptr_ty.isSinglePointer()) {
5784 // If this is a single-item pointer to an array, we need another index in the GEP.5791 // If this is a single-item pointer to an array, we need another index in the GEP.
...@@ -6803,7 +6810,7 @@ pub const FuncGen = struct {...@@ -6803,7 +6810,7 @@ pub const FuncGen = struct {
6803 .data = payload_ty,6810 .data = payload_ty,
6804 };6811 };
6805 const payload_ptr_ty = Type.initPayload(&ptr_ty_payload.base);6812 const payload_ptr_ty = Type.initPayload(&ptr_ty_payload.base);
6806 self.store(payload_ptr, payload_ptr_ty, operand, .NotAtomic);6813 try self.store(payload_ptr, payload_ptr_ty, operand, .NotAtomic);
6807 const non_null_ptr = self.builder.buildStructGEP(llvm_optional_ty, optional_ptr, 1, "");6814 const non_null_ptr = self.builder.buildStructGEP(llvm_optional_ty, optional_ptr, 1, "");
6808 _ = self.builder.buildStore(non_null_bit, non_null_ptr);6815 _ = self.builder.buildStore(non_null_bit, non_null_ptr);
6809 return optional_ptr;6816 return optional_ptr;
...@@ -6839,7 +6846,7 @@ pub const FuncGen = struct {...@@ -6839,7 +6846,7 @@ pub const FuncGen = struct {
6839 .data = payload_ty,6846 .data = payload_ty,
6840 };6847 };
6841 const payload_ptr_ty = Type.initPayload(&ptr_ty_payload.base);6848 const payload_ptr_ty = Type.initPayload(&ptr_ty_payload.base);
6842 self.store(payload_ptr, payload_ptr_ty, operand, .NotAtomic);6849 try self.store(payload_ptr, payload_ptr_ty, operand, .NotAtomic);
6843 return result_ptr;6850 return result_ptr;
6844 }6851 }
68456852
...@@ -6908,6 +6915,28 @@ pub const FuncGen = struct {...@@ -6908,6 +6915,28 @@ pub const FuncGen = struct {
6908 return self.builder.buildCall(llvm_fn.globalGetValueType(), llvm_fn, &args, args.len, .Fast, .Auto, "");6915 return self.builder.buildCall(llvm_fn.globalGetValueType(), llvm_fn, &args, args.len, .Fast, .Auto, "");
6909 }6916 }
69106917
6918 fn airVectorStoreElem(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {
6919 const data = self.air.instructions.items(.data)[inst].vector_store_elem;
6920 const extra = self.air.extraData(Air.Bin, data.payload).data;
6921
6922 const vector_ptr = try self.resolveInst(data.vector_ptr);
6923 const vector_ptr_ty = self.air.typeOf(data.vector_ptr);
6924 const index = try self.resolveInst(extra.lhs);
6925 const operand = try self.resolveInst(extra.rhs);
6926
6927 const loaded_vector = blk: {
6928 const elem_llvm_ty = try self.dg.lowerType(vector_ptr_ty.elemType2());
6929 const load_inst = self.builder.buildLoad(elem_llvm_ty, vector_ptr, "");
6930 const target = self.dg.module.getTarget();
6931 load_inst.setAlignment(vector_ptr_ty.ptrAlignment(target));
6932 load_inst.setVolatile(llvm.Bool.fromBool(vector_ptr_ty.isVolatilePtr()));
6933 break :blk load_inst;
6934 };
6935 const modified_vector = self.builder.buildInsertElement(loaded_vector, operand, index, "");
6936 try self.store(vector_ptr, vector_ptr_ty, modified_vector, .NotAtomic);
6937 return null;
6938 }
6939
6911 fn airMin(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {6940 fn airMin(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {
6912 if (self.liveness.isUnused(inst)) return null;6941 if (self.liveness.isUnused(inst)) return null;
69136942
...@@ -8135,7 +8164,7 @@ pub const FuncGen = struct {...@@ -8135,7 +8164,7 @@ pub const FuncGen = struct {
8135 }8164 }
8136 } else {8165 } else {
8137 const src_operand = try self.resolveInst(bin_op.rhs);8166 const src_operand = try self.resolveInst(bin_op.rhs);
8138 self.store(dest_ptr, ptr_ty, src_operand, .NotAtomic);8167 try self.store(dest_ptr, ptr_ty, src_operand, .NotAtomic);
8139 }8168 }
8140 return null;8169 return null;
8141 }8170 }
...@@ -8385,7 +8414,7 @@ pub const FuncGen = struct {...@@ -8385,7 +8414,7 @@ pub const FuncGen = struct {
8385 element = self.builder.buildZExt(element, abi_ty, "");8414 element = self.builder.buildZExt(element, abi_ty, "");
8386 }8415 }
8387 }8416 }
8388 self.store(ptr, ptr_ty, element, ordering);8417 try self.store(ptr, ptr_ty, element, ordering);
8389 return null;8418 return null;
8390 }8419 }
83918420
...@@ -9178,7 +9207,7 @@ pub const FuncGen = struct {...@@ -9178,7 +9207,7 @@ pub const FuncGen = struct {
9178 },9207 },
9179 };9208 };
9180 const field_ptr_ty = Type.initPayload(&field_ptr_payload.base);9209 const field_ptr_ty = Type.initPayload(&field_ptr_payload.base);
9181 self.store(field_ptr, field_ptr_ty, llvm_elem, .NotAtomic);9210 try self.store(field_ptr, field_ptr_ty, llvm_elem, .NotAtomic);
9182 }9211 }
91839212
9184 return alloca_inst;9213 return alloca_inst;
...@@ -9216,7 +9245,7 @@ pub const FuncGen = struct {...@@ -9216,7 +9245,7 @@ pub const FuncGen = struct {
9216 };9245 };
9217 const elem_ptr = self.builder.buildInBoundsGEP(llvm_result_ty, alloca_inst, &indices, indices.len, "");9246 const elem_ptr = self.builder.buildInBoundsGEP(llvm_result_ty, alloca_inst, &indices, indices.len, "");
9218 const llvm_elem = try self.resolveInst(elem);9247 const llvm_elem = try self.resolveInst(elem);
9219 self.store(elem_ptr, elem_ptr_ty, llvm_elem, .NotAtomic);9248 try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .NotAtomic);
9220 }9249 }
9221 if (array_info.sentinel) |sent_val| {9250 if (array_info.sentinel) |sent_val| {
9222 const indices: [2]*llvm.Value = .{9251 const indices: [2]*llvm.Value = .{
...@@ -9229,7 +9258,7 @@ pub const FuncGen = struct {...@@ -9229,7 +9258,7 @@ pub const FuncGen = struct {
9229 .val = sent_val,9258 .val = sent_val,
9230 });9259 });
92319260
9232 self.store(elem_ptr, elem_ptr_ty, llvm_elem, .NotAtomic);9261 try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .NotAtomic);
9233 }9262 }
92349263
9235 return alloca_inst;9264 return alloca_inst;
...@@ -9352,7 +9381,7 @@ pub const FuncGen = struct {...@@ -9352,7 +9381,7 @@ pub const FuncGen = struct {
9352 };9381 };
9353 const len: c_uint = if (field_size == layout.payload_size) 2 else 3;9382 const len: c_uint = if (field_size == layout.payload_size) 2 else 3;
9354 const field_ptr = self.builder.buildInBoundsGEP(llvm_union_ty, casted_ptr, &indices, len, "");9383 const field_ptr = self.builder.buildInBoundsGEP(llvm_union_ty, casted_ptr, &indices, len, "");
9355 self.store(field_ptr, field_ptr_ty, llvm_payload, .NotAtomic);9384 try self.store(field_ptr, field_ptr_ty, llvm_payload, .NotAtomic);
9356 return result_ptr;9385 return result_ptr;
9357 }9386 }
93589387
...@@ -9364,7 +9393,7 @@ pub const FuncGen = struct {...@@ -9364,7 +9393,7 @@ pub const FuncGen = struct {
9364 };9393 };
9365 const len: c_uint = if (field_size == layout.payload_size) 2 else 3;9394 const len: c_uint = if (field_size == layout.payload_size) 2 else 3;
9366 const field_ptr = self.builder.buildInBoundsGEP(llvm_union_ty, casted_ptr, &indices, len, "");9395 const field_ptr = self.builder.buildInBoundsGEP(llvm_union_ty, casted_ptr, &indices, len, "");
9367 self.store(field_ptr, field_ptr_ty, llvm_payload, .NotAtomic);9396 try self.store(field_ptr, field_ptr_ty, llvm_payload, .NotAtomic);
9368 }9397 }
9369 {9398 {
9370 const indices: [2]*llvm.Value = .{9399 const indices: [2]*llvm.Value = .{
...@@ -9693,6 +9722,20 @@ pub const FuncGen = struct {...@@ -9693,6 +9722,20 @@ pub const FuncGen = struct {
9693 const target = self.dg.module.getTarget();9722 const target = self.dg.module.getTarget();
9694 const ptr_alignment = info.alignment(target);9723 const ptr_alignment = info.alignment(target);
9695 const ptr_volatile = llvm.Bool.fromBool(ptr_ty.isVolatilePtr());9724 const ptr_volatile = llvm.Bool.fromBool(ptr_ty.isVolatilePtr());
9725
9726 assert(info.vector_index != .runtime);
9727 if (info.vector_index != .none) {
9728 const index_u32 = self.dg.context.intType(32).constInt(@enumToInt(info.vector_index), .False);
9729 const vec_elem_ty = try self.dg.lowerType(info.pointee_type);
9730 const vec_ty = vec_elem_ty.vectorType(info.host_size);
9731
9732 const loaded_vector = self.builder.buildLoad(vec_ty, ptr, "");
9733 loaded_vector.setAlignment(ptr_alignment);
9734 loaded_vector.setVolatile(ptr_volatile);
9735
9736 return self.builder.buildExtractElement(loaded_vector, index_u32, "");
9737 }
9738
9696 if (info.host_size == 0) {9739 if (info.host_size == 0) {
9697 if (isByRef(info.pointee_type)) {9740 if (isByRef(info.pointee_type)) {
9698 return self.loadByRef(ptr, info.pointee_type, ptr_alignment, info.@"volatile");9741 return self.loadByRef(ptr, info.pointee_type, ptr_alignment, info.@"volatile");
...@@ -9748,7 +9791,7 @@ pub const FuncGen = struct {...@@ -9748,7 +9791,7 @@ pub const FuncGen = struct {
9748 ptr_ty: Type,9791 ptr_ty: Type,
9749 elem: *llvm.Value,9792 elem: *llvm.Value,
9750 ordering: llvm.AtomicOrdering,9793 ordering: llvm.AtomicOrdering,
9751 ) void {9794 ) !void {
9752 const info = ptr_ty.ptrInfo().data;9795 const info = ptr_ty.ptrInfo().data;
9753 const elem_ty = info.pointee_type;9796 const elem_ty = info.pointee_type;
9754 if (!elem_ty.isFnOrHasRuntimeBitsIgnoreComptime()) {9797 if (!elem_ty.isFnOrHasRuntimeBitsIgnoreComptime()) {
...@@ -9757,6 +9800,26 @@ pub const FuncGen = struct {...@@ -9757,6 +9800,26 @@ pub const FuncGen = struct {
9757 const target = self.dg.module.getTarget();9800 const target = self.dg.module.getTarget();
9758 const ptr_alignment = ptr_ty.ptrAlignment(target);9801 const ptr_alignment = ptr_ty.ptrAlignment(target);
9759 const ptr_volatile = llvm.Bool.fromBool(info.@"volatile");9802 const ptr_volatile = llvm.Bool.fromBool(info.@"volatile");
9803
9804 assert(info.vector_index != .runtime);
9805 if (info.vector_index != .none) {
9806 const index_u32 = self.dg.context.intType(32).constInt(@enumToInt(info.vector_index), .False);
9807 const vec_elem_ty = try self.dg.lowerType(elem_ty);
9808 const vec_ty = vec_elem_ty.vectorType(info.host_size);
9809
9810 const loaded_vector = self.builder.buildLoad(vec_ty, ptr, "");
9811 loaded_vector.setAlignment(ptr_alignment);
9812 loaded_vector.setVolatile(ptr_volatile);
9813
9814 const modified_vector = self.builder.buildInsertElement(loaded_vector, elem, index_u32, "");
9815
9816 const store_inst = self.builder.buildStore(modified_vector, ptr);
9817 assert(ordering == .NotAtomic);
9818 store_inst.setAlignment(ptr_alignment);
9819 store_inst.setVolatile(ptr_volatile);
9820 return;
9821 }
9822
9760 if (info.host_size != 0) {9823 if (info.host_size != 0) {
9761 const int_elem_ty = self.context.intType(info.host_size * 8);9824 const int_elem_ty = self.context.intType(info.host_size * 8);
9762 const int_ptr = self.builder.buildBitCast(ptr, int_elem_ty.pointerType(0), "");9825 const int_ptr = self.builder.buildBitCast(ptr, int_elem_ty.pointerType(0), "");
src/print_air.zig+12
...@@ -306,6 +306,7 @@ const Writer = struct {...@@ -306,6 +306,7 @@ const Writer = struct {
306 .shuffle => try w.writeShuffle(s, inst),306 .shuffle => try w.writeShuffle(s, inst),
307 .reduce, .reduce_optimized => try w.writeReduce(s, inst),307 .reduce, .reduce_optimized => try w.writeReduce(s, inst),
308 .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst),308 .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst),
309 .vector_store_elem => try w.writeVectorStoreElem(s, inst),
309310
310 .dbg_block_begin, .dbg_block_end => {},311 .dbg_block_begin, .dbg_block_end => {},
311 }312 }
...@@ -478,6 +479,17 @@ const Writer = struct {...@@ -478,6 +479,17 @@ const Writer = struct {
478 try w.writeOperand(s, inst, 1, extra.rhs);479 try w.writeOperand(s, inst, 1, extra.rhs);
479 }480 }
480481
482 fn writeVectorStoreElem(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
483 const data = w.air.instructions.items(.data)[inst].vector_store_elem;
484 const extra = w.air.extraData(Air.VectorCmp, data.payload).data;
485
486 try w.writeOperand(s, inst, 0, data.vector_ptr);
487 try s.writeAll(", ");
488 try w.writeOperand(s, inst, 1, extra.lhs);
489 try s.writeAll(", ");
490 try w.writeOperand(s, inst, 2, extra.rhs);
491 }
492
481 fn writeFence(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {493 fn writeFence(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
482 const atomic_order = w.air.instructions.items(.data)[inst].fence;494 const atomic_order = w.air.instructions.items(.data)[inst].fence;
483495
src/type.zig+27-3
...@@ -748,6 +748,8 @@ pub const Type = extern union {...@@ -748,6 +748,8 @@ pub const Type = extern union {
748 return false;748 return false;
749 if (info_a.host_size != info_b.host_size)749 if (info_a.host_size != info_b.host_size)
750 return false;750 return false;
751 if (info_a.vector_index != info_b.vector_index)
752 return false;
751 if (info_a.@"allowzero" != info_b.@"allowzero")753 if (info_a.@"allowzero" != info_b.@"allowzero")
752 return false;754 return false;
753 if (info_a.mutable != info_b.mutable)755 if (info_a.mutable != info_b.mutable)
...@@ -1126,6 +1128,7 @@ pub const Type = extern union {...@@ -1126,6 +1128,7 @@ pub const Type = extern union {
1126 std.hash.autoHash(hasher, info.@"addrspace");1128 std.hash.autoHash(hasher, info.@"addrspace");
1127 std.hash.autoHash(hasher, info.bit_offset);1129 std.hash.autoHash(hasher, info.bit_offset);
1128 std.hash.autoHash(hasher, info.host_size);1130 std.hash.autoHash(hasher, info.host_size);
1131 std.hash.autoHash(hasher, info.vector_index);
1129 std.hash.autoHash(hasher, info.@"allowzero");1132 std.hash.autoHash(hasher, info.@"allowzero");
1130 std.hash.autoHash(hasher, info.mutable);1133 std.hash.autoHash(hasher, info.mutable);
1131 std.hash.autoHash(hasher, info.@"volatile");1134 std.hash.autoHash(hasher, info.@"volatile");
...@@ -1467,6 +1470,7 @@ pub const Type = extern union {...@@ -1467,6 +1470,7 @@ pub const Type = extern union {
1467 .@"addrspace" = payload.@"addrspace",1470 .@"addrspace" = payload.@"addrspace",
1468 .bit_offset = payload.bit_offset,1471 .bit_offset = payload.bit_offset,
1469 .host_size = payload.host_size,1472 .host_size = payload.host_size,
1473 .vector_index = payload.vector_index,
1470 .@"allowzero" = payload.@"allowzero",1474 .@"allowzero" = payload.@"allowzero",
1471 .mutable = payload.mutable,1475 .mutable = payload.mutable,
1472 .@"volatile" = payload.@"volatile",1476 .@"volatile" = payload.@"volatile",
...@@ -1855,12 +1859,17 @@ pub const Type = extern union {...@@ -1855,12 +1859,17 @@ pub const Type = extern union {
1855 .C => try writer.writeAll("[*c]"),1859 .C => try writer.writeAll("[*c]"),
1856 .Slice => try writer.writeAll("[]"),1860 .Slice => try writer.writeAll("[]"),
1857 }1861 }
1858 if (payload.@"align" != 0 or payload.host_size != 0) {1862 if (payload.@"align" != 0 or payload.host_size != 0 or payload.vector_index != .none) {
1859 try writer.print("align({d}", .{payload.@"align"});1863 try writer.print("align({d}", .{payload.@"align"});
18601864
1861 if (payload.bit_offset != 0 or payload.host_size != 0) {1865 if (payload.bit_offset != 0 or payload.host_size != 0) {
1862 try writer.print(":{d}:{d}", .{ payload.bit_offset, payload.host_size });1866 try writer.print(":{d}:{d}", .{ payload.bit_offset, payload.host_size });
1863 }1867 }
1868 if (payload.vector_index == .runtime) {
1869 try writer.writeAll(":?");
1870 } else if (payload.vector_index != .none) {
1871 try writer.print(":{d}", .{@enumToInt(payload.vector_index)});
1872 }
1864 try writer.writeAll(") ");1873 try writer.writeAll(") ");
1865 }1874 }
1866 if (payload.@"addrspace" != .generic) {1875 if (payload.@"addrspace" != .generic) {
...@@ -2185,12 +2194,17 @@ pub const Type = extern union {...@@ -2185,12 +2194,17 @@ pub const Type = extern union {
2185 .C => try writer.writeAll("[*c]"),2194 .C => try writer.writeAll("[*c]"),
2186 .Slice => try writer.writeAll("[]"),2195 .Slice => try writer.writeAll("[]"),
2187 }2196 }
2188 if (info.@"align" != 0 or info.host_size != 0) {2197 if (info.@"align" != 0 or info.host_size != 0 or info.vector_index != .none) {
2189 try writer.print("align({d}", .{info.@"align"});2198 try writer.print("align({d}", .{info.@"align"});
21902199
2191 if (info.bit_offset != 0 or info.host_size != 0) {2200 if (info.bit_offset != 0 or info.host_size != 0) {
2192 try writer.print(":{d}:{d}", .{ info.bit_offset, info.host_size });2201 try writer.print(":{d}:{d}", .{ info.bit_offset, info.host_size });
2193 }2202 }
2203 if (info.vector_index == .runtime) {
2204 try writer.writeAll(":?");
2205 } else if (info.vector_index != .none) {
2206 try writer.print(":{d}", .{@enumToInt(info.vector_index)});
2207 }
2194 try writer.writeAll(") ");2208 try writer.writeAll(") ");
2195 }2209 }
2196 if (info.@"addrspace" != .generic) {2210 if (info.@"addrspace" != .generic) {
...@@ -3865,6 +3879,7 @@ pub const Type = extern union {...@@ -3865,6 +3879,7 @@ pub const Type = extern union {
3865 payload.@"addrspace" != .generic or3879 payload.@"addrspace" != .generic or
3866 payload.bit_offset != 0 or3880 payload.bit_offset != 0 or
3867 payload.host_size != 0 or3881 payload.host_size != 0 or
3882 payload.vector_index != .none or
3868 payload.@"allowzero" or3883 payload.@"allowzero" or
3869 payload.@"volatile")3884 payload.@"volatile")
3870 {3885 {
...@@ -3877,6 +3892,7 @@ pub const Type = extern union {...@@ -3877,6 +3892,7 @@ pub const Type = extern union {
3877 .@"addrspace" = payload.@"addrspace",3892 .@"addrspace" = payload.@"addrspace",
3878 .bit_offset = payload.bit_offset,3893 .bit_offset = payload.bit_offset,
3879 .host_size = payload.host_size,3894 .host_size = payload.host_size,
3895 .vector_index = payload.vector_index,
3880 .@"allowzero" = payload.@"allowzero",3896 .@"allowzero" = payload.@"allowzero",
3881 .mutable = payload.mutable,3897 .mutable = payload.mutable,
3882 .@"volatile" = payload.@"volatile",3898 .@"volatile" = payload.@"volatile",
...@@ -6365,11 +6381,18 @@ pub const Type = extern union {...@@ -6365,11 +6381,18 @@ pub const Type = extern union {
6365 /// When host_size=pointee_abi_size and bit_offset=0, this must be6381 /// When host_size=pointee_abi_size and bit_offset=0, this must be
6366 /// represented with host_size=0 instead.6382 /// represented with host_size=0 instead.
6367 host_size: u16 = 0,6383 host_size: u16 = 0,
6384 vector_index: VectorIndex = .none,
6368 @"allowzero": bool = false,6385 @"allowzero": bool = false,
6369 mutable: bool = true, // TODO rename this to const, not mutable6386 mutable: bool = true, // TODO rename this to const, not mutable
6370 @"volatile": bool = false,6387 @"volatile": bool = false,
6371 size: std.builtin.Type.Pointer.Size = .One,6388 size: std.builtin.Type.Pointer.Size = .One,
63726389
6390 pub const VectorIndex = enum(u32) {
6391 none = std.math.maxInt(u32),
6392 runtime = std.math.maxInt(u32) - 1,
6393 _,
6394 };
6395
6373 pub fn alignment(data: Data, target: Target) u32 {6396 pub fn alignment(data: Data, target: Target) u32 {
6374 if (data.@"align" != 0) return data.@"align";6397 if (data.@"align" != 0) return data.@"align";
6375 return abiAlignment(data.pointee_type, target);6398 return abiAlignment(data.pointee_type, target);
...@@ -6524,7 +6547,8 @@ pub const Type = extern union {...@@ -6524,7 +6547,8 @@ pub const Type = extern union {
6524 }6547 }
65256548
6526 if (d.@"align" == 0 and d.@"addrspace" == .generic and6549 if (d.@"align" == 0 and d.@"addrspace" == .generic and
6527 d.bit_offset == 0 and d.host_size == 0 and !d.@"allowzero" and !d.@"volatile")6550 d.bit_offset == 0 and d.host_size == 0 and d.vector_index == .none and
6551 !d.@"allowzero" and !d.@"volatile")
6528 {6552 {
6529 if (d.sentinel) |sent| {6553 if (d.sentinel) |sent| {
6530 if (!d.mutable and d.pointee_type.eql(Type.u8, mod)) {6554 if (!d.mutable and d.pointee_type.eql(Type.u8, mod)) {
test/behavior/vector.zig+27
...@@ -1234,3 +1234,30 @@ test "array operands to shuffle are coerced to vectors" {...@@ -1234,3 +1234,30 @@ test "array operands to shuffle are coerced to vectors" {
1234 var b = @shuffle(u32, a, @splat(5, @as(u24, 0)), mask);1234 var b = @shuffle(u32, a, @splat(5, @as(u24, 0)), mask);
1235 try expectEqual([_]u32{ 0, 3, 5, 7, 9 }, b);1235 try expectEqual([_]u32{ 0, 3, 5, 7, 9 }, b);
1236}1236}
1237
1238test "load packed vector element" {
1239 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1240 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1241 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
1242 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1243 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1244 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1245
1246 var x: @Vector(2, u15) = .{ 1, 4 };
1247 try expect((&x[0]).* == 1);
1248 try expect((&x[1]).* == 4);
1249}
1250
1251test "store packed vector element" {
1252 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1253 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1254 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
1255 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1256 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1257 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1258
1259 var v = @Vector(4, u1){ 1, 1, 1, 1 };
1260 try expectEqual(@Vector(4, u1){ 1, 1, 1, 1 }, v);
1261 v[0] = 0;
1262 try expectEqual(@Vector(4, u1){ 0, 1, 1, 1 }, v);
1263}
test/cases/compile_errors/load_vector_pointer_with_unknown_runtime_index.zig created+17
...@@ -0,0 +1,17 @@
1export fn entry() void {
2 var v: @Vector(4, i31) = [_]i31{ 1, 5, 3, undefined };
3
4 var i: u32 = 0;
5 var x = loadv(&v[i]);
6 _ = x;
7}
8
9fn loadv(ptr: anytype) i31 {
10 return ptr.*;
11}
12
13// error
14// backend=llvm
15// target=native
16//
17// :10:15: error: unable to determine vector element index of type '*align(16:0:4:?) i31'
test/cases/compile_errors/stage1/obj/load_vector_pointer_with_unknown_runtime_index.zig deleted-17
...@@ -1,17 +0,0 @@
1export fn entry() void {
2 var v: @import("std").meta.Vector(4, i32) = [_]i32{ 1, 5, 3, undefined };
3
4 var i: u32 = 0;
5 var x = loadv(&v[i]);
6 _ = x;
7}
8
9fn loadv(ptr: anytype) i32 {
10 return ptr.*;
11}
12
13// error
14// backend=stage1
15// target=native
16//
17// tmp.zig:10:12: error: unable to determine vector element index of type '*align(16:0:4:?) i32
test/cases/compile_errors/stage1/obj/store_vector_pointer_with_unknown_runtime_index.zig deleted-16
...@@ -1,16 +0,0 @@
1export fn entry() void {
2 var v: @import("std").meta.Vector(4, i32) = [_]i32{ 1, 5, 3, undefined };
3
4 var i: u32 = 0;
5 storev(&v[i], 42);
6}
7
8fn storev(ptr: anytype, val: i32) void {
9 ptr.* = val;
10}
11
12// error
13// backend=stage1
14// target=native
15//
16// tmp.zig:9:8: error: unable to determine vector element index of type '*align(16:0:4:?) i32
test/cases/compile_errors/store_vector_pointer_with_unknown_runtime_index.zig created+16
...@@ -0,0 +1,16 @@
1export fn entry() void {
2 var v: @Vector(4, i31) = [_]i31{ 1, 5, 3, undefined };
3
4 var i: u32 = 0;
5 storev(&v[i], 42);
6}
7
8fn storev(ptr: anytype, val: i31) void {
9 ptr.* = val;
10}
11
12// error
13// backend=llvm
14// target=native
15//
16// :9:8: error: unable to determine vector element index of type '*align(16:0:4:?) i31'