diff --git a/src/Air.zig b/src/Air.zig index 486656524cb87684419ddc10a5ff1022b319d4b8..3d6edeb5941bc82eae0ac30205b5dde64887a523 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -780,8 +780,8 @@ pub const Inst = struct { reduce, /// Same as `reduce` with optimized float mode. reduce_optimized, - /// Given an integer, bool, float, or pointer operand, return a vector with all elements - /// equal to the scalar value. + /// Given an operand, return a vector or array with all elements equal to the operand. + /// For a sentinel-terminated array, the sentinel is derived from the result type. /// Uses the `ty_op` field. splat, /// Constructs a vector by selecting elements from a single vector based on a mask. Each diff --git a/src/Air/Legalize.zig b/src/Air/Legalize.zig index 9c3f0417d36694202f31850e7389e7809d45c628..28fe9faf04c981155dd41c997784f7fba1baa972 100644 --- a/src/Air/Legalize.zig +++ b/src/Air/Legalize.zig @@ -200,6 +200,8 @@ pub const Feature = enum { expand_packed_agg_field_val, /// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bit_cast`, `int_cast`, and `bit_or`. expand_packed_aggregate_init, + /// Replace `splat` of an array with an `aggregate_init`. + expand_array_splat, /// Replace `array_to_vector` with an `array_elem_val` per element followed by an `aggregate_init`. expand_array_to_vector, @@ -891,15 +893,27 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { .soft_float => unreachable, // the operand is not a scalar } }, - .splat => if (l.features.has(.splat_one_elem_to_bit_cast)) { + .splat => { const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op; - switch (ty_op.ty.vectorLen(zcu)) { - 0 => unreachable, - 1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{ - .ty = ty_op.ty, - .operand = ty_op.operand, - } }), - else => {}, + switch (ty_op.ty.zigTypeTag(zcu)) { + .vector => switch (ty_op.ty.vectorLen(zcu)) { + 0 => unreachable, + 1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{ + .ty = ty_op.ty, + .operand = ty_op.operand, + } }), + else => {}, + }, + .array => if (l.features.has(.expand_array_splat)) { + const len: usize = @intCast(ty_op.ty.arrayLen(zcu)); + const elems_start: u32 = @intCast(l.air_extra.items.len); + try l.air_extra.appendNTimes(l.pt.zcu.gpa, @backingInt(ty_op.operand), len); + continue :inst l.replaceInst(inst, .aggregate_init, .{ .ty_pl = .{ + .ty = ty_op.ty, + .payload = elems_start, + } }); + }, + else => unreachable, } }, .shuffle_one => { diff --git a/src/Sema.zig b/src/Sema.zig index 3bef560c7a1bab9caf91761bae10526c178aaa49..33746cd9817b99ed88cf68090239901706533e8c 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -23271,13 +23271,7 @@ fn zirSplat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.I try sema.requireRuntimeBlock(block, src, scalar_src); switch (dest_ty.zigTypeTag(zcu)) { - .array => { - const elems = try sema.arena.alloc(Air.Inst.Ref, len + @intFromBool(maybe_sentinel != null)); - @memset(elems[0..len], scalar); - if (maybe_sentinel) |s| elems[len] = Air.internedToRef(s.toIntern()); - return block.addAggregateInit(dest_ty, elems); - }, - .vector => return block.addTyOp(.splat, dest_ty, scalar), + .vector, .array => return block.addTyOp(.splat, dest_ty, scalar), else => unreachable, } } diff --git a/src/codegen/aarch64.zig b/src/codegen/aarch64.zig index 3c9b683b9a2e908c21612d447308f98b34404690..63b07c3d7afa58d5005c86028f5d85645831c1cc 100644 --- a/src/codegen/aarch64.zig +++ b/src/codegen/aarch64.zig @@ -8,6 +8,7 @@ pub const Select = @import("aarch64/Select.zig"); pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { return comptime &.initMany(&.{ .expand_bit_cast_safe, + .expand_array_splat, .expand_array_to_vector, }); } diff --git a/src/codegen/c.zig b/src/codegen/c.zig index e040445c15c506adf466bfd29ead358ad79cbdb1..93598ba8ca644be3f899b9131260ab4e3ad028f2 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -39,6 +39,7 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features { .expand_packed_store = true, .expand_packed_agg_field_val = true, .expand_packed_aggregate_init = true, + .expand_array_splat = true, .expand_array_to_vector = true, .scalarize_bit_cast_array = true, diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig index d65a23146f081f5bbc1f7c40980db2faadb0db63..f213d7cf7bf3ec2d1d7d0e4764a1002d9b590dc3 100644 --- a/src/codegen/llvm/FuncGen.zig +++ b/src/codegen/llvm/FuncGen.zig @@ -5590,8 +5590,36 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error .slice => null, .many, .c => unreachable, }); + const len_bytes = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); - if (allow_byte_memset) if (bin_op.rhs.toInterned()) |elem_ip_index| { + try self.lowerMemset( + dest_ptr, + dest_ptr_align, + bin_op.rhs, + elem_ty, + len_bytes, + access_kind, + safety, + allow_byte_memset, + ); + return .none; +} + +fn lowerMemset( + self: *FuncGen, + dest_ptr: Builder.Value, + dest_ptr_align: InternPool.Alignment, + elem_ref: Air.Inst.Ref, + elem_ty: Type, + len_bytes: Builder.Value, + access_kind: Builder.MemoryAccessKind, + safety: bool, + allow_byte_memset: bool, +) Allocator.Error!void { + const o = self.object; + const zcu = o.zcu; + + if (allow_byte_memset) if (elem_ref.toInterned()) |elem_ip_index| { const elem_val: Value = .fromInterned(elem_ip_index); if (elem_val.isUndef(zcu)) { // Even if safety is disabled, we still emit a memset to undefined since it conveys @@ -5601,20 +5629,19 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8); - const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); _ = try self.wip.callMemSet( dest_ptr, dest_ptr_align.toLlvm(), fill_byte, - len, + len_bytes, access_kind, self.disable_intrinsics, ); const owner_mod = self.ownerModule(); if (safety and owner_mod.valgrind) { - try self.valgrindMarkUndef(dest_ptr, len); + try self.valgrindMarkUndef(dest_ptr, len_bytes); } - return .none; + return; } // Test if the element value is compile-time known to be a @@ -5623,20 +5650,19 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error // intrinsic can be used. if (try elem_val.hasRepeatedByteRepr(zcu)) |byte_val| { const fill_byte = try o.builder.intValue(.i8, byte_val); - const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); _ = try self.wip.callMemSet( dest_ptr, dest_ptr_align.toLlvm(), fill_byte, - len, + len_bytes, access_kind, self.disable_intrinsics, ); - return .none; + return; } }; - const value = try self.resolveInst(bin_op.rhs); + const value = try self.resolveInst(elem_ref); const elem_abi_size = elem_ty.abiSize(zcu); intrinsic: { @@ -5660,16 +5686,15 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error break :intrinsic; }; // Great, we can use the intrinsic! - const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); _ = try self.wip.callMemSet( dest_ptr, dest_ptr_align.toLlvm(), fill_byte, - len, + len_bytes, access_kind, self.disable_intrinsics, ); - return .none; + return; } // non-byte-sized element. lower with a loop. something like this: @@ -5693,15 +5718,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error const body_block = try self.wip.block(1, "InlineMemsetBody"); const end_block = try self.wip.block(1, "InlineMemsetEnd"); - const end_ptr = switch (ptr_ty.ptrSize(zcu)) { - .slice => try self.ptraddScaled( - dest_ptr, - try self.wip.extractValue(dest_slice, &.{1}, ""), - elem_abi_size, - ), - .one => try self.ptraddConst(dest_ptr, ptr_ty.childType(zcu).abiSize(zcu)), - .many, .c => unreachable, - }; + const end_ptr = try self.ptraddScaled(dest_ptr, len_bytes, 1); _ = try self.wip.br(loop_block); self.wip.cursor = .{ .block = loop_block }; @@ -5718,7 +5735,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error self.wip.cursor = .{ .block = end_block }; it_ptr.finish(&.{ next_ptr, dest_ptr }, &.{ body_block, entry_block }, &self.wip); - return .none; + return; } fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { @@ -5980,10 +5997,45 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va } fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { + const o = self.object; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op; - const scalar = try self.resolveInst(ty_op.operand); - const vector_ty = self.typeOfIndex(inst); - return self.wip.splatVector(try self.object.lowerType(vector_ty, .as_value), scalar, ""); + const result_ty = self.typeOfIndex(inst); + switch (result_ty.zigTypeTag(zcu)) { + .vector => { + const scalar = try self.resolveInst(ty_op.operand); + return self.wip.splatVector(try o.lowerType(result_ty, .as_value), scalar, ""); + }, + .array => { + assert(isByRef(result_ty, zcu)); + + const result_ptr = try self.buildZigAlloca(result_ty, .none); + const array_info = result_ty.arrayInfo(zcu); + const elem_size = array_info.elem_type.abiSize(zcu); + const len_bytes = array_info.len * elem_size; + const len_bytes_llvm = try o.builder.intValue(try o.lowerType(.usize, .as_value), len_bytes); + + try self.lowerMemset( + result_ptr, + result_ty.abiAlignment(zcu), + ty_op.operand, + array_info.elem_type, + len_bytes_llvm, + .normal, + false, + !self.needMemsetWorkaround(len_bytes), + ); + + if (array_info.sentinel) |sent_val| { + const sent_ptr = try self.ptraddConst(result_ptr, len_bytes); + const sent_elem = try self.resolveValue(sent_val); + try self.store(sent_ptr, .none, sent_elem.toValue(), array_info.elem_type, .normal); + } + + return result_ptr; + }, + else => unreachable, + } } fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { diff --git a/src/codegen/riscv64/CodeGen.zig b/src/codegen/riscv64/CodeGen.zig index 106640f519a47de8a43a420bc539a716f47f711f..9534e0dd8d5b441e3d36bba319b9c9dab15023dc 100644 --- a/src/codegen/riscv64/CodeGen.zig +++ b/src/codegen/riscv64/CodeGen.zig @@ -59,6 +59,7 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { .expand_sub_safe, .expand_mul_safe, + .expand_array_splat, .expand_array_to_vector, }); } diff --git a/src/codegen/sparc64/CodeGen.zig b/src/codegen/sparc64/CodeGen.zig index e354976c91628f88e6d4c71180ab8cf4351fc538..7ac53d48c2d02b2f2d70628c10b2bc8a2b91c780 100644 --- a/src/codegen/sparc64/CodeGen.zig +++ b/src/codegen/sparc64/CodeGen.zig @@ -42,6 +42,7 @@ const InnerError = codegen.Error || error{OutOfRegisters}; pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features { return comptime &.initMany(&.{ + .expand_array_splat, .expand_array_to_vector, }); } diff --git a/src/codegen/spirv/CodeGen.zig b/src/codegen/spirv/CodeGen.zig index ffaa71766afb2b5ae9f329b9d9ee14812231cf8b..37c78ef0339f4ee081420add0f1f775f856877be 100644 --- a/src/codegen/spirv/CodeGen.zig +++ b/src/codegen/spirv/CodeGen.zig @@ -129,6 +129,7 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { .expand_sub_safe, .expand_mul_safe, + .expand_array_splat, .expand_array_to_vector, }); } diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index 993cf26ad66ffae0467a097097c50398751caee9..343d089209acbb63da92b17dc8a90857a8f22e31 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -38,6 +38,7 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { .expand_packed_store, .expand_packed_agg_field_val, .expand_packed_aggregate_init, + .expand_array_splat, .expand_array_to_vector, .scalarize_add, diff --git a/src/codegen/x86_64/CodeGen.zig b/src/codegen/x86_64/CodeGen.zig index 0eccc1e5f7fd462a77e65b411cf88bbe6e7c68a8..410e2a878c767b55a85740a4576af8c3f4e9e8d8 100644 --- a/src/codegen/x86_64/CodeGen.zig +++ b/src/codegen/x86_64/CodeGen.zig @@ -78,6 +78,7 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features { .expand_packed_store, .expand_packed_agg_field_val, .expand_packed_aggregate_init, + .expand_array_splat, .expand_array_to_vector, }); } diff --git a/test/behavior/memset.zig b/test/behavior/memset.zig index ed9df298ac0b327506efdbe68871044974ba75dc..95005288f36f5eddb6001a7d8e543c29ddab905a 100644 --- a/test/behavior/memset.zig +++ b/test/behavior/memset.zig @@ -30,6 +30,28 @@ fn testMemsetArray() !void { } } +test "@memset preserves array sentinel" { + if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; + if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; + if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest; + + try testMemsetArraySentinel(); + try comptime testMemsetArraySentinel(); +} + +fn testMemsetArraySentinel() !void { + var value: u32 = 42; + _ = &value; + var array: [3:0]u32 = .{ 1, 2, 3 }; + + @memset(&array, value); + + try expect(array[0] == value); + try expect(array[1] == value); + try expect(array[2] == value); + try expect(array[3] == 0); +} + test "@memset on slices" { if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;