| ... | ... | @@ -953,9 +953,57 @@ test "len" { |
| 953 | 953 | try testing.expect(len(c_ptr) == 2); |
| 954 | 954 | } |
| 955 | 955 | |
| 956 | | pub fn indexOfSentinel(comptime Elem: type, comptime sentinel: Elem, ptr: [*:sentinel]const Elem) usize { |
| 956 | pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]const T) usize { |
| 957 | 957 | var i: usize = 0; |
| 958 | | while (ptr[i] != sentinel) { |
| 958 | |
| 959 | if (!@inComptime() and (@typeInfo(T) == .Int or @typeInfo(T) == .Float) and std.math.isPowerOfTwo(@bitSizeOf(T))) { |
| 960 | switch (@import("builtin").cpu.arch) { |
| 961 | // The below branch assumes that reading past the end of the buffer is valid, as long |
| 962 | // as we don't read into a new page. This should be the case for most architectures |
| 963 | // which use paged memory, however should be confirmed before adding a new arch below. |
| 964 | .aarch64, .x86, .x86_64 => if (comptime std.simd.suggestVectorSize(T)) |block_len| { |
| 965 | comptime std.debug.assert(std.mem.page_size % block_len == 0); |
| 966 | const Block = @Vector(block_len, T); |
| 967 | const mask: Block = @splat(sentinel); |
| 968 | |
| 969 | // First block may be unaligned |
| 970 | const start_addr = @intFromPtr(&p[i]); |
| 971 | const offset_in_page = start_addr & (std.mem.page_size - 1); |
| 972 | if (offset_in_page < std.mem.page_size - block_len) { |
| 973 | // Will not read past the end of a page, full block. |
| 974 | const block: Block = p[i..][0..block_len].*; |
| 975 | const matches = block == mask; |
| 976 | if (@reduce(.Or, matches)) { |
| 977 | return i + std.simd.firstTrue(matches).?; |
| 978 | } |
| 979 | |
| 980 | i += std.mem.alignForward(usize, start_addr, block_len) - start_addr; |
| 981 | } else { |
| 982 | // Would read over a page boundary. Per-byte at a time until aligned or found. |
| 983 | // 0.39% chance this branch is taken for 4K pages at 16b block length. |
| 984 | // |
| 985 | // An alternate strategy is to do read a full block (the last in the page) and |
| 986 | // mask the entries before the pointer. |
| 987 | while ((@intFromPtr(&p[i]) & (block_len - 1)) != 0) : (i += 1) { |
| 988 | if (p[i] == sentinel) return i; |
| 989 | } |
| 990 | } |
| 991 | |
| 992 | std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), block_len)); |
| 993 | while (true) { |
| 994 | const block: *const Block = @ptrCast(@alignCast(p[i..][0..block_len])); |
| 995 | const matches = block.* == mask; |
| 996 | if (@reduce(.Or, matches)) { |
| 997 | return i + std.simd.firstTrue(matches).?; |
| 998 | } |
| 999 | i += block_len; |
| 1000 | } |
| 1001 | }, |
| 1002 | else => {}, |
| 1003 | } |
| 1004 | } |
| 1005 | |
| 1006 | while (p[i] != sentinel) { |
| 959 | 1007 | i += 1; |
| 960 | 1008 | } |
| 961 | 1009 | return i; |
| ... | ... | @@ -1016,8 +1064,58 @@ pub fn lastIndexOfScalar(comptime T: type, slice: []const T, value: T) ?usize { |
| 1016 | 1064 | |
| 1017 | 1065 | pub fn indexOfScalarPos(comptime T: type, slice: []const T, start_index: usize, value: T) ?usize { |
| 1018 | 1066 | if (start_index >= slice.len) return null; |
| 1019 | | for (slice[start_index..], start_index..) |c, i| { |
| 1020 | | if (c == value) return i; |
| 1067 | |
| 1068 | var i: usize = start_index; |
| 1069 | if (!@inComptime() and (@typeInfo(T) == .Int or @typeInfo(T) == .Float) and std.math.isPowerOfTwo(@bitSizeOf(T))) { |
| 1070 | if (comptime std.simd.suggestVectorSize(T)) |block_len| { |
| 1071 | // For Intel Nehalem (2009) and AMD Bulldozer (2012) or later, unaligned loads on aligned data result |
| 1072 | // in the same execution as aligned loads. We ignore older arch's here and don't bother pre-aligning. |
| 1073 | // |
| 1074 | // Use `comptime std.simd.suggestVectorSize(T)` to get the same alignment as used in this function |
| 1075 | // however this usually isn't necessary unless your arch has a performance penalty due to this. |
| 1076 | // |
| 1077 | // This may differ for other arch's. Arm for example costs a cycle when loading across a cache |
| 1078 | // line so explicit alignment prologues may be worth exploration. |
| 1079 | |
| 1080 | // Unrolling here is ~10% improvement. We can then do one bounds check every 2 blocks |
| 1081 | // instead of one which adds up. |
| 1082 | const Block = @Vector(block_len, T); |
| 1083 | if (i + 2 * block_len < slice.len) { |
| 1084 | const mask: Block = @splat(value); |
| 1085 | while (true) { |
| 1086 | inline for (0..2) |_| { |
| 1087 | const block: Block = slice[i..][0..block_len].*; |
| 1088 | const matches = block == mask; |
| 1089 | if (@reduce(.Or, matches)) { |
| 1090 | return i + std.simd.firstTrue(matches).?; |
| 1091 | } |
| 1092 | i += block_len; |
| 1093 | } |
| 1094 | if (i + 2 * block_len >= slice.len) break; |
| 1095 | } |
| 1096 | } |
| 1097 | |
| 1098 | // {block_len, block_len / 2} check |
| 1099 | inline for (0..2) |j| { |
| 1100 | const block_x_len = block_len / (1 << j); |
| 1101 | comptime if (block_x_len < 4) break; |
| 1102 | |
| 1103 | const BlockX = @Vector(block_x_len, T); |
| 1104 | if (i + block_x_len < slice.len) { |
| 1105 | const mask: BlockX = @splat(value); |
| 1106 | const block: BlockX = slice[i..][0..block_x_len].*; |
| 1107 | const matches = block == mask; |
| 1108 | if (@reduce(.Or, matches)) { |
| 1109 | return i + std.simd.firstTrue(matches).?; |
| 1110 | } |
| 1111 | i += block_x_len; |
| 1112 | } |
| 1113 | } |
| 1114 | } |
| 1115 | } |
| 1116 | |
| 1117 | for (slice[i..], i..) |c, j| { |
| 1118 | if (c == value) return j; |
| 1021 | 1119 | } |
| 1022 | 1120 | return null; |
| 1023 | 1121 | } |
Marc Tiehuis <marc@tiehu.is> 2023-09-14 19:54:36+12:00
lib