| ... | ... | @@ -1050,15 +1050,16 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co |
| 1050 | 1050 | // as we don't read into a new page. This should be the case for most architectures |
| 1051 | 1051 | // which use paged memory, however should be confirmed before adding a new arch below. |
| 1052 | 1052 | .aarch64, .x86, .x86_64 => if (std.simd.suggestVectorLength(T)) |block_len| { |
| 1053 | const block_size = @sizeOf(T) * block_len; |
| 1053 | 1054 | const Block = @Vector(block_len, T); |
| 1054 | 1055 | const mask: Block = @splat(sentinel); |
| 1055 | 1056 | |
| 1056 | | comptime std.debug.assert(std.mem.page_size % @sizeOf(Block) == 0); |
| 1057 | comptime std.debug.assert(std.mem.page_size % block_size == 0); |
| 1057 | 1058 | |
| 1058 | 1059 | // First block may be unaligned |
| 1059 | 1060 | const start_addr = @intFromPtr(&p[i]); |
| 1060 | 1061 | const offset_in_page = start_addr & (std.mem.page_size - 1); |
| 1061 | | if (offset_in_page <= std.mem.page_size - @sizeOf(Block)) { |
| 1062 | if (offset_in_page <= std.mem.page_size - block_size) { |
| 1062 | 1063 | // Will not read past the end of a page, full block. |
| 1063 | 1064 | const block: Block = p[i..][0..block_len].*; |
| 1064 | 1065 | const matches = block == mask; |
| ... | ... | @@ -1066,19 +1067,19 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co |
| 1066 | 1067 | return i + std.simd.firstTrue(matches).?; |
| 1067 | 1068 | } |
| 1068 | 1069 | |
| 1069 | | i += (std.mem.alignForward(usize, start_addr, @alignOf(Block)) - start_addr) / @sizeOf(T); |
| 1070 | i += @divExact(std.mem.alignForward(usize, start_addr, block_size) - start_addr, @sizeOf(T)); |
| 1070 | 1071 | } else { |
| 1071 | 1072 | // Would read over a page boundary. Per-byte at a time until aligned or found. |
| 1072 | 1073 | // 0.39% chance this branch is taken for 4K pages at 16b block length. |
| 1073 | 1074 | // |
| 1074 | 1075 | // An alternate strategy is to do read a full block (the last in the page) and |
| 1075 | 1076 | // mask the entries before the pointer. |
| 1076 | | while ((@intFromPtr(&p[i]) & (@alignOf(Block) - 1)) != 0) : (i += 1) { |
| 1077 | while ((@intFromPtr(&p[i]) & (block_size - 1)) != 0) : (i += 1) { |
| 1077 | 1078 | if (p[i] == sentinel) return i; |
| 1078 | 1079 | } |
| 1079 | 1080 | } |
| 1080 | 1081 | |
| 1081 | | std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), @alignOf(Block))); |
| 1082 | std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), block_size)); |
| 1082 | 1083 | while (true) { |
| 1083 | 1084 | const block: *const Block = @ptrCast(@alignCast(p[i..][0..block_len])); |
| 1084 | 1085 | const matches = block.* == mask; |
Jacob Young <jacobly@ziglang.org> 2024-07-15 00:24:50-04:00
lib