authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2023-10-30 16:38:40-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2023-10-31 02:23:33-07:00
log03117c52905e2a18cbfb5a1aeb50e319a1ec4fba
tree3ca05bfc34c92e90516d2a4be6490f8e652cd656
parent91e117697ad90430d9266203415712b6cc59f669

std.unicode: Add ASCII fast path to UTF-8 -> UTF-16 conversion functions

1 files changed, 47 insertions(+), 4 deletions(-)

lib/std/unicode.zig+47-4
...@@ -2,6 +2,7 @@ const std = @import("./std.zig");...@@ -2,6 +2,7 @@ const std = @import("./std.zig");
2const assert = std.debug.assert;2const assert = std.debug.assert;
3const testing = std.testing;3const testing = std.testing;
4const mem = std.mem;4const mem = std.mem;
5const builtin = @import("builtin");
56
6/// Use this to replace an unknown, unrecognized, or unrepresentable character.7/// Use this to replace an unknown, unrecognized, or unrepresentable character.
7///8///
...@@ -863,7 +864,27 @@ pub fn utf8ToUtf16LeWithNull(allocator: mem.Allocator, utf8: []const u8) ![:0]u1...@@ -863,7 +864,27 @@ pub fn utf8ToUtf16LeWithNull(allocator: mem.Allocator, utf8: []const u8) ![:0]u1
863 var result = try std.ArrayList(u16).initCapacity(allocator, utf8.len + 1);864 var result = try std.ArrayList(u16).initCapacity(allocator, utf8.len + 1);
864 errdefer result.deinit();865 errdefer result.deinit();
865866
866 const view = try Utf8View.init(utf8);867 var remaining = utf8;
868 if (builtin.zig_backend != .stage2_x86_64) {
869 const chunk_len = std.simd.suggestVectorSize(u8) orelse 1;
870 const Chunk = @Vector(chunk_len, u8);
871
872 // Fast path. Check for and encode ASCII characters at the start of the input.
873 while (remaining.len >= chunk_len) {
874 const chunk: Chunk = remaining[0..chunk_len].*;
875 const mask: Chunk = @splat(0x80);
876 if (@reduce(.Or, chunk & mask == mask)) {
877 // found a non ASCII code unit
878 break;
879 }
880 const zeroes: Chunk = @splat(0);
881 const utf16_chunk: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes });
882 result.appendSliceAssumeCapacity(std.mem.bytesAsSlice(u16, &utf16_chunk));
883 remaining = remaining[chunk_len..];
884 }
885 }
886
887 const view = try Utf8View.init(remaining);
867 var it = view.iterator();888 var it = view.iterator();
868 while (it.nextCodepoint()) |codepoint| {889 while (it.nextCodepoint()) |codepoint| {
869 if (codepoint < 0x10000) {890 if (codepoint < 0x10000) {
...@@ -886,11 +907,33 @@ pub fn utf8ToUtf16LeWithNull(allocator: mem.Allocator, utf8: []const u8) ![:0]u1...@@ -886,11 +907,33 @@ pub fn utf8ToUtf16LeWithNull(allocator: mem.Allocator, utf8: []const u8) ![:0]u1
886/// Assumes there is enough space for the output.907/// Assumes there is enough space for the output.
887pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {908pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {
888 var dest_i: usize = 0;909 var dest_i: usize = 0;
910
911 var remaining = utf8;
912 if (builtin.zig_backend != .stage2_x86_64) {
913 const chunk_len = std.simd.suggestVectorSize(u8) orelse 1;
914 const Chunk = @Vector(chunk_len, u8);
915
916 // Fast path. Check for and encode ASCII characters at the start of the input.
917 while (remaining.len >= chunk_len) {
918 const chunk: Chunk = remaining[0..chunk_len].*;
919 const mask: Chunk = @splat(0x80);
920 if (@reduce(.Or, chunk & mask == mask)) {
921 // found a non ASCII code unit
922 break;
923 }
924 const zeroes: Chunk = @splat(0);
925 const utf16_bytes: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes });
926 @memcpy(utf16le[dest_i..][0..chunk_len], std.mem.bytesAsSlice(u16, &utf16_bytes));
927 dest_i += chunk_len;
928 remaining = remaining[chunk_len..];
929 }
930 }
931
889 var src_i: usize = 0;932 var src_i: usize = 0;
890 while (src_i < utf8.len) {933 while (src_i < remaining.len) {
891 const n = utf8ByteSequenceLength(utf8[src_i]) catch return error.InvalidUtf8;934 const n = utf8ByteSequenceLength(remaining[src_i]) catch return error.InvalidUtf8;
892 const next_src_i = src_i + n;935 const next_src_i = src_i + n;
893 const codepoint = utf8Decode(utf8[src_i..next_src_i]) catch return error.InvalidUtf8;936 const codepoint = utf8Decode(remaining[src_i..next_src_i]) catch return error.InvalidUtf8;
894 if (codepoint < 0x10000) {937 if (codepoint < 0x10000) {
895 const short = @as(u16, @intCast(codepoint));938 const short = @as(u16, @intCast(codepoint));
896 utf16le[dest_i] = mem.nativeToLittle(u16, short);939 utf16le[dest_i] = mem.nativeToLittle(u16, short);