| ... | @@ -757,8 +757,34 @@ pub fn utf16leToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) ![]u8 | ... | @@ -757,8 +757,34 @@ pub fn utf16leToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) ![]u8 |
| 757 | // optimistically guess that it will all be ascii. | 757 | // optimistically guess that it will all be ascii. |
| 758 | var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len); | 758 | var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len); |
| 759 | errdefer result.deinit(); | 759 | errdefer result.deinit(); |
| 760 | var out_index: usize = 0; | 760 | |
| 761 | var it = Utf16LeIterator.init(utf16le); | 761 | var remaining = utf16le; |
| | 762 | if (builtin.zig_backend != .stage2_x86_64) { |
| | 763 | const chunk_len = std.simd.suggestVectorSize(u16) orelse 1; |
| | 764 | const Chunk = @Vector(chunk_len, u16); |
| | 765 | |
| | 766 | // Fast path. Check for and encode ASCII characters at the start of the input. |
| | 767 | while (remaining.len >= chunk_len) { |
| | 768 | const chunk: Chunk = remaining[0..chunk_len].*; |
| | 769 | const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F)); |
| | 770 | if (@reduce(.Or, chunk | mask != mask)) { |
| | 771 | // found a non ASCII code unit |
| | 772 | break; |
| | 773 | } |
| | 774 | const chunk_byte_len = chunk_len * 2; |
| | 775 | const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*; |
| | 776 | const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes); |
| | 777 | const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0]; |
| | 778 | // We allocated enough space to encode every UTF-16 code unit |
| | 779 | // as ASCII, so if the entire string is ASCII then we are |
| | 780 | // guaranteed to have enough space allocated |
| | 781 | result.appendSliceAssumeCapacity(&ascii_bytes); |
| | 782 | remaining = remaining[chunk_len..]; |
| | 783 | } |
| | 784 | } |
| | 785 | |
| | 786 | var out_index: usize = result.items.len; |
| | 787 | var it = Utf16LeIterator.init(remaining); |
| 762 | while (try it.nextCodepoint()) |codepoint| { | 788 | while (try it.nextCodepoint()) |codepoint| { |
| 763 | const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable; | 789 | const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable; |
| 764 | try result.resize(result.items.len + utf8_len); | 790 | try result.resize(result.items.len + utf8_len); |
| ... | @@ -774,8 +800,34 @@ pub fn utf16leToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) ![:0] | ... | @@ -774,8 +800,34 @@ pub fn utf16leToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) ![:0] |
| 774 | // optimistically guess that it will all be ascii (and allocate space for the null terminator) | 800 | // optimistically guess that it will all be ascii (and allocate space for the null terminator) |
| 775 | var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len + 1); | 801 | var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len + 1); |
| 776 | errdefer result.deinit(); | 802 | errdefer result.deinit(); |
| 777 | var out_index: usize = 0; | 803 | |
| 778 | var it = Utf16LeIterator.init(utf16le); | 804 | var remaining = utf16le; |
| | 805 | if (builtin.zig_backend != .stage2_x86_64) { |
| | 806 | const chunk_len = std.simd.suggestVectorSize(u16) orelse 1; |
| | 807 | const Chunk = @Vector(chunk_len, u16); |
| | 808 | |
| | 809 | // Fast path. Check for and encode ASCII characters at the start of the input. |
| | 810 | while (remaining.len >= chunk_len) { |
| | 811 | const chunk: Chunk = remaining[0..chunk_len].*; |
| | 812 | const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F)); |
| | 813 | if (@reduce(.Or, chunk | mask != mask)) { |
| | 814 | // found a non ASCII code unit |
| | 815 | break; |
| | 816 | } |
| | 817 | const chunk_byte_len = chunk_len * 2; |
| | 818 | const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*; |
| | 819 | const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes); |
| | 820 | const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0]; |
| | 821 | // We allocated enough space to encode every UTF-16 code unit |
| | 822 | // as ASCII, so if the entire string is ASCII then we are |
| | 823 | // guaranteed to have enough space allocated |
| | 824 | result.appendSliceAssumeCapacity(&ascii_bytes); |
| | 825 | remaining = remaining[chunk_len..]; |
| | 826 | } |
| | 827 | } |
| | 828 | |
| | 829 | var out_index = result.items.len; |
| | 830 | var it = Utf16LeIterator.init(remaining); |
| 779 | while (try it.nextCodepoint()) |codepoint| { | 831 | while (try it.nextCodepoint()) |codepoint| { |
| 780 | const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable; | 832 | const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable; |
| 781 | try result.resize(result.items.len + utf8_len); | 833 | try result.resize(result.items.len + utf8_len); |
| ... | @@ -789,7 +841,31 @@ pub fn utf16leToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) ![:0] | ... | @@ -789,7 +841,31 @@ pub fn utf16leToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) ![:0] |
| 789 | /// Returns end byte index into utf8. | 841 | /// Returns end byte index into utf8. |
| 790 | pub fn utf16leToUtf8(utf8: []u8, utf16le: []const u16) !usize { | 842 | pub fn utf16leToUtf8(utf8: []u8, utf16le: []const u16) !usize { |
| 791 | var end_index: usize = 0; | 843 | var end_index: usize = 0; |
| 792 | var it = Utf16LeIterator.init(utf16le); | 844 | |
| | 845 | var remaining = utf16le; |
| | 846 | if (builtin.zig_backend != .stage2_x86_64) { |
| | 847 | const chunk_len = std.simd.suggestVectorSize(u16) orelse 1; |
| | 848 | const Chunk = @Vector(chunk_len, u16); |
| | 849 | |
| | 850 | // Fast path. Check for and encode ASCII characters at the start of the input. |
| | 851 | while (remaining.len >= chunk_len) { |
| | 852 | const chunk: Chunk = remaining[0..chunk_len].*; |
| | 853 | const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F)); |
| | 854 | if (@reduce(.Or, chunk | mask != mask)) { |
| | 855 | // found a non ASCII code unit |
| | 856 | break; |
| | 857 | } |
| | 858 | const chunk_byte_len = chunk_len * 2; |
| | 859 | const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*; |
| | 860 | const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes); |
| | 861 | const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0]; |
| | 862 | @memcpy(utf8[end_index .. end_index + chunk_len], &ascii_bytes); |
| | 863 | end_index += chunk_len; |
| | 864 | remaining = remaining[chunk_len..]; |
| | 865 | } |
| | 866 | } |
| | 867 | |
| | 868 | var it = Utf16LeIterator.init(remaining); |
| 793 | while (try it.nextCodepoint()) |codepoint| { | 869 | while (try it.nextCodepoint()) |codepoint| { |
| 794 | end_index += try utf8Encode(codepoint, utf8[end_index..]); | 870 | end_index += try utf8Encode(codepoint, utf8[end_index..]); |
| 795 | } | 871 | } |
Ryan Liptak <squeek502@hotmail.com> 2023-10-30 17:12:30-07:00
lib