| ... | ... | @@ -2,6 +2,7 @@ const std = @import("./std.zig"); |
| 2 | 2 | const assert = std.debug.assert; |
| 3 | 3 | const testing = std.testing; |
| 4 | 4 | const mem = std.mem; |
| 5 | const builtin = @import("builtin"); |
| 5 | 6 | |
| 6 | 7 | /// Use this to replace an unknown, unrecognized, or unrepresentable character. |
| 7 | 8 | /// |
| ... | ... | @@ -863,7 +864,27 @@ pub fn utf8ToUtf16LeWithNull(allocator: mem.Allocator, utf8: []const u8) ![:0]u1 |
| 863 | 864 | var result = try std.ArrayList(u16).initCapacity(allocator, utf8.len + 1); |
| 864 | 865 | errdefer result.deinit(); |
| 865 | 866 | |
| 866 | | const view = try Utf8View.init(utf8); |
| 867 | var remaining = utf8; |
| 868 | if (builtin.zig_backend != .stage2_x86_64) { |
| 869 | const chunk_len = std.simd.suggestVectorSize(u8) orelse 1; |
| 870 | const Chunk = @Vector(chunk_len, u8); |
| 871 | |
| 872 | // Fast path. Check for and encode ASCII characters at the start of the input. |
| 873 | while (remaining.len >= chunk_len) { |
| 874 | const chunk: Chunk = remaining[0..chunk_len].*; |
| 875 | const mask: Chunk = @splat(0x80); |
| 876 | if (@reduce(.Or, chunk & mask == mask)) { |
| 877 | // found a non ASCII code unit |
| 878 | break; |
| 879 | } |
| 880 | const zeroes: Chunk = @splat(0); |
| 881 | const utf16_chunk: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes }); |
| 882 | result.appendSliceAssumeCapacity(std.mem.bytesAsSlice(u16, &utf16_chunk)); |
| 883 | remaining = remaining[chunk_len..]; |
| 884 | } |
| 885 | } |
| 886 | |
| 887 | const view = try Utf8View.init(remaining); |
| 867 | 888 | var it = view.iterator(); |
| 868 | 889 | while (it.nextCodepoint()) |codepoint| { |
| 869 | 890 | if (codepoint < 0x10000) { |
| ... | ... | @@ -886,11 +907,33 @@ pub fn utf8ToUtf16LeWithNull(allocator: mem.Allocator, utf8: []const u8) ![:0]u1 |
| 886 | 907 | /// Assumes there is enough space for the output. |
| 887 | 908 | pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize { |
| 888 | 909 | var dest_i: usize = 0; |
| 910 | |
| 911 | var remaining = utf8; |
| 912 | if (builtin.zig_backend != .stage2_x86_64) { |
| 913 | const chunk_len = std.simd.suggestVectorSize(u8) orelse 1; |
| 914 | const Chunk = @Vector(chunk_len, u8); |
| 915 | |
| 916 | // Fast path. Check for and encode ASCII characters at the start of the input. |
| 917 | while (remaining.len >= chunk_len) { |
| 918 | const chunk: Chunk = remaining[0..chunk_len].*; |
| 919 | const mask: Chunk = @splat(0x80); |
| 920 | if (@reduce(.Or, chunk & mask == mask)) { |
| 921 | // found a non ASCII code unit |
| 922 | break; |
| 923 | } |
| 924 | const zeroes: Chunk = @splat(0); |
| 925 | const utf16_bytes: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes }); |
| 926 | @memcpy(utf16le[dest_i..][0..chunk_len], std.mem.bytesAsSlice(u16, &utf16_bytes)); |
| 927 | dest_i += chunk_len; |
| 928 | remaining = remaining[chunk_len..]; |
| 929 | } |
| 930 | } |
| 931 | |
| 889 | 932 | var src_i: usize = 0; |
| 890 | | while (src_i < utf8.len) { |
| 891 | | const n = utf8ByteSequenceLength(utf8[src_i]) catch return error.InvalidUtf8; |
| 933 | while (src_i < remaining.len) { |
| 934 | const n = utf8ByteSequenceLength(remaining[src_i]) catch return error.InvalidUtf8; |
| 892 | 935 | const next_src_i = src_i + n; |
| 893 | | const codepoint = utf8Decode(utf8[src_i..next_src_i]) catch return error.InvalidUtf8; |
| 936 | const codepoint = utf8Decode(remaining[src_i..next_src_i]) catch return error.InvalidUtf8; |
| 894 | 937 | if (codepoint < 0x10000) { |
| 895 | 938 | const short = @as(u16, @intCast(codepoint)); |
| 896 | 939 | utf16le[dest_i] = mem.nativeToLittle(u16, short); |
Ryan Liptak <squeek502@hotmail.com> 2023-10-30 16:38:40-07:00
lib