authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2023-09-14 19:54:36+12:00
committergravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2023-09-30 21:19:43+13:00
logcd766513febecd79fc62a0f46ee18867fabbbf7d
tree48be459f4fa7bb743867f22c683098cbc4353596
parent873c695c41dffd89ba7ef1b3ed6662e429bfa00d

std.mem: add vectorized indexOfScalarPos and indexOfSentinel

These are an order of magnitude quicker than the previous implementations: A relative comparison of each, measuring scanning a 1G file. Reading 1G (1.0000000009313226GiB) std.mem.sliceTo: 281.232ms vectorized.sliceTo: 24.769ms strlen: 24.291ms std.indexOfScalar: 229.016ms vectorized.indexOfScalar: 24.685ms memchr: 24.958ms

1 files changed, 102 insertions(+), 4 deletions(-)

lib/std/mem.zig+102-4
...@@ -953,9 +953,57 @@ test "len" {...@@ -953,9 +953,57 @@ test "len" {
953 try testing.expect(len(c_ptr) == 2);953 try testing.expect(len(c_ptr) == 2);
954}954}
955955
956pub fn indexOfSentinel(comptime Elem: type, comptime sentinel: Elem, ptr: [*:sentinel]const Elem) usize {956pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]const T) usize {
957 var i: usize = 0;957 var i: usize = 0;
958 while (ptr[i] != sentinel) {958
959 if (!@inComptime() and (@typeInfo(T) == .Int or @typeInfo(T) == .Float) and std.math.isPowerOfTwo(@bitSizeOf(T))) {
960 switch (@import("builtin").cpu.arch) {
961 // The below branch assumes that reading past the end of the buffer is valid, as long
962 // as we don't read into a new page. This should be the case for most architectures
963 // which use paged memory, however should be confirmed before adding a new arch below.
964 .aarch64, .x86, .x86_64 => if (comptime std.simd.suggestVectorSize(T)) |block_len| {
965 comptime std.debug.assert(std.mem.page_size % block_len == 0);
966 const Block = @Vector(block_len, T);
967 const mask: Block = @splat(sentinel);
968
969 // First block may be unaligned
970 const start_addr = @intFromPtr(&p[i]);
971 const offset_in_page = start_addr & (std.mem.page_size - 1);
972 if (offset_in_page < std.mem.page_size - block_len) {
973 // Will not read past the end of a page, full block.
974 const block: Block = p[i..][0..block_len].*;
975 const matches = block == mask;
976 if (@reduce(.Or, matches)) {
977 return i + std.simd.firstTrue(matches).?;
978 }
979
980 i += std.mem.alignForward(usize, start_addr, block_len) - start_addr;
981 } else {
982 // Would read over a page boundary. Per-byte at a time until aligned or found.
983 // 0.39% chance this branch is taken for 4K pages at 16b block length.
984 //
985 // An alternate strategy is to do read a full block (the last in the page) and
986 // mask the entries before the pointer.
987 while ((@intFromPtr(&p[i]) & (block_len - 1)) != 0) : (i += 1) {
988 if (p[i] == sentinel) return i;
989 }
990 }
991
992 std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), block_len));
993 while (true) {
994 const block: *const Block = @ptrCast(@alignCast(p[i..][0..block_len]));
995 const matches = block.* == mask;
996 if (@reduce(.Or, matches)) {
997 return i + std.simd.firstTrue(matches).?;
998 }
999 i += block_len;
1000 }
1001 },
1002 else => {},
1003 }
1004 }
1005
1006 while (p[i] != sentinel) {
959 i += 1;1007 i += 1;
960 }1008 }
961 return i;1009 return i;
...@@ -1016,8 +1064,58 @@ pub fn lastIndexOfScalar(comptime T: type, slice: []const T, value: T) ?usize {...@@ -1016,8 +1064,58 @@ pub fn lastIndexOfScalar(comptime T: type, slice: []const T, value: T) ?usize {
10161064
1017pub fn indexOfScalarPos(comptime T: type, slice: []const T, start_index: usize, value: T) ?usize {1065pub fn indexOfScalarPos(comptime T: type, slice: []const T, start_index: usize, value: T) ?usize {
1018 if (start_index >= slice.len) return null;1066 if (start_index >= slice.len) return null;
1019 for (slice[start_index..], start_index..) |c, i| {1067
1020 if (c == value) return i;1068 var i: usize = start_index;
1069 if (!@inComptime() and (@typeInfo(T) == .Int or @typeInfo(T) == .Float) and std.math.isPowerOfTwo(@bitSizeOf(T))) {
1070 if (comptime std.simd.suggestVectorSize(T)) |block_len| {
1071 // For Intel Nehalem (2009) and AMD Bulldozer (2012) or later, unaligned loads on aligned data result
1072 // in the same execution as aligned loads. We ignore older arch's here and don't bother pre-aligning.
1073 //
1074 // Use `comptime std.simd.suggestVectorSize(T)` to get the same alignment as used in this function
1075 // however this usually isn't necessary unless your arch has a performance penalty due to this.
1076 //
1077 // This may differ for other arch's. Arm for example costs a cycle when loading across a cache
1078 // line so explicit alignment prologues may be worth exploration.
1079
1080 // Unrolling here is ~10% improvement. We can then do one bounds check every 2 blocks
1081 // instead of one which adds up.
1082 const Block = @Vector(block_len, T);
1083 if (i + 2 * block_len < slice.len) {
1084 const mask: Block = @splat(value);
1085 while (true) {
1086 inline for (0..2) |_| {
1087 const block: Block = slice[i..][0..block_len].*;
1088 const matches = block == mask;
1089 if (@reduce(.Or, matches)) {
1090 return i + std.simd.firstTrue(matches).?;
1091 }
1092 i += block_len;
1093 }
1094 if (i + 2 * block_len >= slice.len) break;
1095 }
1096 }
1097
1098 // {block_len, block_len / 2} check
1099 inline for (0..2) |j| {
1100 const block_x_len = block_len / (1 << j);
1101 comptime if (block_x_len < 4) break;
1102
1103 const BlockX = @Vector(block_x_len, T);
1104 if (i + block_x_len < slice.len) {
1105 const mask: BlockX = @splat(value);
1106 const block: BlockX = slice[i..][0..block_x_len].*;
1107 const matches = block == mask;
1108 if (@reduce(.Or, matches)) {
1109 return i + std.simd.firstTrue(matches).?;
1110 }
1111 i += block_x_len;
1112 }
1113 }
1114 }
1115 }
1116
1117 for (slice[i..], i..) |c, j| {
1118 if (c == value) return j;
1021 }1119 }
1022 return null;1120 return null;
1023}1121}