authorgravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-28 13:54:42+11:00
committergravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-28 14:39:38+11:00
loga81c0ba2e730bd69e7c7a672fa795c46ce6b001b
treeae692057f5de4ba63c4ad75f02d6097aba0f279e
parent25e71216c4640a3d88c8f63912ea574ad6fa004c
signature Commit is signed but in an unrecognized format.

std: fix unicode encoding of astral plane codepoints to utf16

1 files changed, 26 insertions(+), 13 deletions(-)

lib/std/unicode.zig+26-13
......@@ -555,9 +555,8 @@ pub fn utf8ToUtf16LeWithNull(allocator: *mem.Allocator, utf8: []const u8) ![]u16
555555 const short = @intCast(u16, codepoint);
556556 try result.append(mem.nativeToLittle(u16, short));
557557 } else {
558 const short = @intCast(u16, codepoint - 0x10000);
559 const high = (short >> 10) + 0xD800;
560 const low = (short & 0x3FF) + 0xDC00;
558 const high = @intCast(u16, (codepoint - 0x10000) >> 10) + 0xD800;
559 const low = @intCast(u16, codepoint & 0x3FF) + 0xDC00;
561560 var out: [2]u16 = undefined;
562561 out[0] = mem.nativeToLittle(u16, high);
563562 out[1] = mem.nativeToLittle(u16, low);
......@@ -592,9 +591,8 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {
592591 utf16le[dest_i] = mem.nativeToLittle(u16, short);
593592 dest_i += 1;
594593 } else {
595 const short = @intCast(u16, codepoint - 0x10000);
596 const high = (short >> 10) + 0xD800;
597 const low = (short & 0x3FF) + 0xDC00;
594 const high = @intCast(u16, (codepoint - 0x10000) >> 10) + 0xD800;
595 const low = @intCast(u16, codepoint & 0x3FF) + 0xDC00;
598596 utf16le[dest_i] = mem.nativeToLittle(u16, high);
599597 utf16le[dest_i + 1] = mem.nativeToLittle(u16, low);
600598 dest_i += 2;
......@@ -609,14 +607,29 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {
609607
610608test "utf8ToUtf16Le" {
611609 var utf16le: [2]u16 = [_]u16{0} ** 2;
612 const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");
613 testing.expect(@as(usize, 2) == length);
614 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", @sliceToBytes(utf16le[0..]));
610 {
611 const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");
612 testing.expectEqual(@as(usize, 2), length);
613 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", @sliceToBytes(utf16le[0..]));
614 }
615 {
616 const length = try utf8ToUtf16Le(utf16le[0..], "\u{10FFFF}");
617 testing.expectEqual(@as(usize, 2), length);
618 testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", @sliceToBytes(utf16le[0..]));
619 }
615620}
616621
617622test "utf8ToUtf16LeWithNull" {
618 var bytes: [128]u8 = undefined;
619 const allocator = &std.heap.FixedBufferAllocator.init(bytes[0..]).allocator;
620 const utf16 = try utf8ToUtf16LeWithNull(allocator, "𐐷");
621 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc\x00\x00", @sliceToBytes(utf16[0..]));
623 {
624 var bytes: [128]u8 = undefined;
625 const allocator = &std.heap.FixedBufferAllocator.init(bytes[0..]).allocator;
626 const utf16 = try utf8ToUtf16LeWithNull(allocator, "𐐷");
627 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc\x00\x00", @sliceToBytes(utf16[0..]));
628 }
629 {
630 var bytes: [128]u8 = undefined;
631 const allocator = &std.heap.FixedBufferAllocator.init(bytes[0..]).allocator;
632 const utf16 = try utf8ToUtf16LeWithNull(allocator, "\u{10FFFF}");
633 testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf\x00\x00", @sliceToBytes(utf16[0..]));
634 }
622635}