| ... | ... | @@ -201,21 +201,18 @@ pub fn utf8CountCodepoints(s: []const u8) !usize { |
| 201 | 201 | pub fn utf8ValidateSlice(input: []const u8) bool { |
| 202 | 202 | var remaining = input; |
| 203 | 203 | |
| 204 | | const V_len = std.simd.suggestVectorSize(usize) orelse 1; |
| 205 | | const V = @Vector(V_len, usize); |
| 206 | | const u8s_in_vector = @sizeOf(usize) * V_len; |
| 204 | const chunk_len = std.simd.suggestVectorSize(u8) orelse 1; |
| 205 | const Chunk = @Vector(chunk_len, u8); |
| 207 | 206 | |
| 208 | 207 | // Fast path. Check for and skip ASCII characters at the start of the input. |
| 209 | | while (remaining.len >= u8s_in_vector) { |
| 210 | | const chunk: V = @bitCast(remaining[0..u8s_in_vector].*); |
| 211 | | const swapped = mem.littleToNative(V, chunk); |
| 212 | | const reduced = @reduce(.Or, swapped); |
| 213 | | const mask: usize = @bitCast([1]u8{0x80} ** @sizeOf(usize)); |
| 214 | | if (reduced & mask != 0) { |
| 215 | | // Found a non ASCII byte |
| 208 | while (remaining.len >= chunk_len) { |
| 209 | const chunk: Chunk = remaining[0..chunk_len].*; |
| 210 | const mask: Chunk = @splat(0x80); |
| 211 | if (@reduce(.Or, chunk & mask == mask)) { |
| 212 | // found a non ASCII byte |
| 216 | 213 | break; |
| 217 | 214 | } |
| 218 | | remaining = remaining[u8s_in_vector..]; |
| 215 | remaining = remaining[chunk_len..]; |
| 219 | 216 | } |
| 220 | 217 | |
| 221 | 218 | // default lowest and highest continuation byte |
Jacob Young <jacobly@ziglang.org> 2023-10-21 13:43:06-04:00
Andrew Kelley <andrew@ziglang.org> 2023-10-22 12:07:23-04:00
lib