| ... | @@ -985,19 +985,19 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co | ... | @@ -985,19 +985,19 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co |
| 985 | return i + std.simd.firstTrue(matches).?; | 985 | return i + std.simd.firstTrue(matches).?; |
| 986 | } | 986 | } |
| 987 | | 987 | |
| 988 | i += std.mem.alignForward(usize, start_addr, block_len) - start_addr; | 988 | i += (std.mem.alignForward(usize, start_addr, @alignOf(Block)) - start_addr) / @sizeOf(T); |
| 989 | } else { | 989 | } else { |
| 990 | // Would read over a page boundary. Per-byte at a time until aligned or found. | 990 | // Would read over a page boundary. Per-byte at a time until aligned or found. |
| 991 | // 0.39% chance this branch is taken for 4K pages at 16b block length. | 991 | // 0.39% chance this branch is taken for 4K pages at 16b block length. |
| 992 | // | 992 | // |
| 993 | // An alternate strategy is to do read a full block (the last in the page) and | 993 | // An alternate strategy is to do read a full block (the last in the page) and |
| 994 | // mask the entries before the pointer. | 994 | // mask the entries before the pointer. |
| 995 | while ((@intFromPtr(&p[i]) & (block_len - 1)) != 0) : (i += 1) { | 995 | while ((@intFromPtr(&p[i]) & (@alignOf(Block) - 1)) != 0) : (i += 1) { |
| 996 | if (p[i] == sentinel) return i; | 996 | if (p[i] == sentinel) return i; |
| 997 | } | 997 | } |
| 998 | } | 998 | } |
| 999 | | 999 | |
| 1000 | std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), block_len)); | 1000 | std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), @alignOf(Block))); |
| 1001 | while (true) { | 1001 | while (true) { |
| 1002 | const block: *const Block = @ptrCast(@alignCast(p[i..][0..block_len])); | 1002 | const block: *const Block = @ptrCast(@alignCast(p[i..][0..block_len])); |
| 1003 | const matches = block.* == mask; | 1003 | const matches = block.* == mask; |
| ... | @@ -1017,6 +1017,41 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co | ... | @@ -1017,6 +1017,41 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co |
| 1017 | return i; | 1017 | return i; |
| 1018 | } | 1018 | } |
| 1019 | | 1019 | |
| | 1020 | test "indexOfSentinel vector paths" { |
| | 1021 | const Types = [_]type{ u8, u16, u32, u64 }; |
| | 1022 | const allocator = std.testing.allocator; |
| | 1023 | |
| | 1024 | inline for (Types) |T| { |
| | 1025 | const block_len = comptime std.simd.suggestVectorSize(T) orelse continue; |
| | 1026 | |
| | 1027 | // Allocate three pages so we guarantee a page-crossing address with a full page after |
| | 1028 | const memory = try allocator.alloc(T, 3 * std.mem.page_size / @sizeOf(T)); |
| | 1029 | defer allocator.free(memory); |
| | 1030 | @memset(memory, 0xaa); |
| | 1031 | |
| | 1032 | // Find starting page-alignment = 0 |
| | 1033 | var start: usize = 0; |
| | 1034 | const start_addr = @intFromPtr(&memory); |
| | 1035 | start += (std.mem.alignForward(usize, start_addr, std.mem.page_size) - start_addr) / @sizeOf(T); |
| | 1036 | try testing.expect(start < std.mem.page_size / @sizeOf(T)); |
| | 1037 | |
| | 1038 | // Validate all sub-block alignments |
| | 1039 | const search_len = std.mem.page_size / @sizeOf(T); |
| | 1040 | memory[start + search_len] = 0; |
| | 1041 | for (0..block_len) |offset| { |
| | 1042 | try testing.expectEqual(search_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start + offset]))); |
| | 1043 | } |
| | 1044 | memory[start + search_len] = 0xaa; |
| | 1045 | |
| | 1046 | // Validate page boundary crossing |
| | 1047 | const start_page_boundary = start + (std.mem.page_size / @sizeOf(T)); |
| | 1048 | memory[start_page_boundary + block_len] = 0; |
| | 1049 | for (0..block_len) |offset| { |
| | 1050 | try testing.expectEqual(2 * block_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start_page_boundary - block_len + offset]))); |
| | 1051 | } |
| | 1052 | } |
| | 1053 | } |
| | 1054 | |
| 1020 | /// Returns true if all elements in a slice are equal to the scalar value provided | 1055 | /// Returns true if all elements in a slice are equal to the scalar value provided |
| 1021 | pub fn allEqual(comptime T: type, slice: []const T, scalar: T) bool { | 1056 | pub fn allEqual(comptime T: type, slice: []const T, scalar: T) bool { |
| 1022 | for (slice) |item| { | 1057 | for (slice) |item| { |
| ... | @@ -1131,6 +1166,20 @@ pub fn indexOfScalarPos(comptime T: type, slice: []const T, start_index: usize, | ... | @@ -1131,6 +1166,20 @@ pub fn indexOfScalarPos(comptime T: type, slice: []const T, start_index: usize, |
| 1131 | return null; | 1166 | return null; |
| 1132 | } | 1167 | } |
| 1133 | | 1168 | |
| | 1169 | test "indexOfScalarPos" { |
| | 1170 | const Types = [_]type{ u8, u16, u32, u64 }; |
| | 1171 | |
| | 1172 | inline for (Types) |T| { |
| | 1173 | var memory: [64 / @sizeOf(T)]T = undefined; |
| | 1174 | @memset(&memory, 0xaa); |
| | 1175 | memory[memory.len - 1] = 0; |
| | 1176 | |
| | 1177 | for (0..memory.len) |i| { |
| | 1178 | try testing.expectEqual(memory.len - i - 1, indexOfScalarPos(T, memory[i..], 0, 0).?); |
| | 1179 | } |
| | 1180 | } |
| | 1181 | } |
| | 1182 | |
| 1134 | pub fn indexOfAny(comptime T: type, slice: []const T, values: []const T) ?usize { | 1183 | pub fn indexOfAny(comptime T: type, slice: []const T, values: []const T) ?usize { |
| 1135 | return indexOfAnyPos(T, slice, 0, values); | 1184 | return indexOfAnyPos(T, slice, 0, values); |
| 1136 | } | 1185 | } |
Marc Tiehuis <marc@tiehu.is> 2023-09-30 22:15:47+13:00
lib