authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2023-10-30 17:12:30-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2023-10-31 02:23:35-07:00
log13c8ec9db0365225e717140d282350be922a205d
tree3161fd148271a512443f5603033f9480d3dfc41f
parent03117c52905e2a18cbfb5a1aeb50e319a1ec4fba

std.unicode: Add ASCII fast path to UTF-16 -> UTF-8 conversion functions

1 files changed, 81 insertions(+), 5 deletions(-)

lib/std/unicode.zig+81-5
......@@ -757,8 +757,34 @@ pub fn utf16leToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) ![]u8
757757 // optimistically guess that it will all be ascii.
758758 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len);
759759 errdefer result.deinit();
760 var out_index: usize = 0;
761 var it = Utf16LeIterator.init(utf16le);
760
761 var remaining = utf16le;
762 if (builtin.zig_backend != .stage2_x86_64) {
763 const chunk_len = std.simd.suggestVectorSize(u16) orelse 1;
764 const Chunk = @Vector(chunk_len, u16);
765
766 // Fast path. Check for and encode ASCII characters at the start of the input.
767 while (remaining.len >= chunk_len) {
768 const chunk: Chunk = remaining[0..chunk_len].*;
769 const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F));
770 if (@reduce(.Or, chunk | mask != mask)) {
771 // found a non ASCII code unit
772 break;
773 }
774 const chunk_byte_len = chunk_len * 2;
775 const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*;
776 const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes);
777 const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0];
778 // We allocated enough space to encode every UTF-16 code unit
779 // as ASCII, so if the entire string is ASCII then we are
780 // guaranteed to have enough space allocated
781 result.appendSliceAssumeCapacity(&ascii_bytes);
782 remaining = remaining[chunk_len..];
783 }
784 }
785
786 var out_index: usize = result.items.len;
787 var it = Utf16LeIterator.init(remaining);
762788 while (try it.nextCodepoint()) |codepoint| {
763789 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
764790 try result.resize(result.items.len + utf8_len);
......@@ -774,8 +800,34 @@ pub fn utf16leToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) ![:0]
774800 // optimistically guess that it will all be ascii (and allocate space for the null terminator)
775801 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len + 1);
776802 errdefer result.deinit();
777 var out_index: usize = 0;
778 var it = Utf16LeIterator.init(utf16le);
803
804 var remaining = utf16le;
805 if (builtin.zig_backend != .stage2_x86_64) {
806 const chunk_len = std.simd.suggestVectorSize(u16) orelse 1;
807 const Chunk = @Vector(chunk_len, u16);
808
809 // Fast path. Check for and encode ASCII characters at the start of the input.
810 while (remaining.len >= chunk_len) {
811 const chunk: Chunk = remaining[0..chunk_len].*;
812 const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F));
813 if (@reduce(.Or, chunk | mask != mask)) {
814 // found a non ASCII code unit
815 break;
816 }
817 const chunk_byte_len = chunk_len * 2;
818 const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*;
819 const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes);
820 const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0];
821 // We allocated enough space to encode every UTF-16 code unit
822 // as ASCII, so if the entire string is ASCII then we are
823 // guaranteed to have enough space allocated
824 result.appendSliceAssumeCapacity(&ascii_bytes);
825 remaining = remaining[chunk_len..];
826 }
827 }
828
829 var out_index = result.items.len;
830 var it = Utf16LeIterator.init(remaining);
779831 while (try it.nextCodepoint()) |codepoint| {
780832 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
781833 try result.resize(result.items.len + utf8_len);
......@@ -789,7 +841,31 @@ pub fn utf16leToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) ![:0]
789841/// Returns end byte index into utf8.
790842pub fn utf16leToUtf8(utf8: []u8, utf16le: []const u16) !usize {
791843 var end_index: usize = 0;
792 var it = Utf16LeIterator.init(utf16le);
844
845 var remaining = utf16le;
846 if (builtin.zig_backend != .stage2_x86_64) {
847 const chunk_len = std.simd.suggestVectorSize(u16) orelse 1;
848 const Chunk = @Vector(chunk_len, u16);
849
850 // Fast path. Check for and encode ASCII characters at the start of the input.
851 while (remaining.len >= chunk_len) {
852 const chunk: Chunk = remaining[0..chunk_len].*;
853 const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F));
854 if (@reduce(.Or, chunk | mask != mask)) {
855 // found a non ASCII code unit
856 break;
857 }
858 const chunk_byte_len = chunk_len * 2;
859 const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*;
860 const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes);
861 const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0];
862 @memcpy(utf8[end_index .. end_index + chunk_len], &ascii_bytes);
863 end_index += chunk_len;
864 remaining = remaining[chunk_len..];
865 }
866 }
867
868 var it = Utf16LeIterator.init(remaining);
793869 while (try it.nextCodepoint()) |codepoint| {
794870 end_index += try utf8Encode(codepoint, utf8[end_index..]);
795871 }