authorgravatar for data-man@users.noreply.github.comDmitry Atamanov <data-man@users.noreply.github.com> 2019-12-17 02:27:26+05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-12-16 16:27:26-05:00
log744133acb1358a9c3ed55001571c7cc7778563bd
treeabcf3bcd402fb530e9a92e9b484b9cb196558fc0
parentd4e56ae6ae15ed1b062b0d775893abb5579fc66d

Fixes utf8ToUtf16Le (#3923)

1 files changed, 38 insertions(+), 9 deletions(-)

lib/std/unicode.zig+38-9
......@@ -542,7 +542,6 @@ test "utf16leToUtf8" {
542542 }
543543}
544544
545/// TODO support codepoints bigger than 16 bits
546545/// TODO type for null terminated pointer
547546pub fn utf8ToUtf16LeWithNull(allocator: *mem.Allocator, utf8: []const u8) ![]u16 {
548547 var result = std.ArrayList(u16).init(allocator);
......@@ -552,7 +551,18 @@ pub fn utf8ToUtf16LeWithNull(allocator: *mem.Allocator, utf8: []const u8) ![]u16
552551 const view = try Utf8View.init(utf8);
553552 var it = view.iterator();
554553 while (it.nextCodepoint()) |codepoint| {
555 try result.append(@intCast(u16, codepoint)); // TODO surrogate pairs
554 if (codepoint < 0x10000) {
555 const short = @intCast(u16, codepoint);
556 try result.append(mem.nativeToLittle(u16, short));
557 } else {
558 const short = @intCast(u16, codepoint - 0x10000);
559 const high = (short >> 10) + 0xD800;
560 const low = (short & 0x3FF) + 0xDC00;
561 var out: [2]u16 = undefined;
562 out[0] = mem.nativeToLittle(u16, high);
563 out[1] = mem.nativeToLittle(u16, low);
564 try result.appendSlice(out[0..]);
565 }
556566 }
557567
558568 try result.append(0);
......@@ -561,7 +571,6 @@ pub fn utf8ToUtf16LeWithNull(allocator: *mem.Allocator, utf8: []const u8) ![]u16
561571
562572/// Returns index of next character. If exact fit, returned index equals output slice length.
563573/// Assumes there is enough space for the output.
564/// TODO support codepoints bigger than 16 bits
565574pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {
566575 var dest_i: usize = 0;
567576 var src_i: usize = 0;
......@@ -578,12 +587,18 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {
578587 2, 3, 4 => {
579588 const next_src_i = src_i + n;
580589 const codepoint = utf8Decode(utf8[src_i..next_src_i]) catch return error.InvalidUtf8;
581 const short = @intCast(u16, codepoint); // TODO surrogate pairs
582 utf16le[dest_i] = switch (builtin.endian) {
583 .Little => short,
584 .Big => @byteSwap(u16, short),
585 };
586 dest_i += 1;
590 if (codepoint < 0x10000) {
591 const short = @intCast(u16, codepoint);
592 utf16le[dest_i] = mem.nativeToLittle(u16, short);
593 dest_i += 1;
594 } else {
595 const short = @intCast(u16, codepoint - 0x10000);
596 const high = (short >> 10) + 0xD800;
597 const low = (short & 0x3FF) + 0xDC00;
598 utf16le[dest_i] = mem.nativeToLittle(u16, high);
599 utf16le[dest_i + 1] = mem.nativeToLittle(u16, low);
600 dest_i += 2;
601 }
587602 src_i = next_src_i;
588603 },
589604 else => return error.InvalidUtf8,
......@@ -591,3 +606,17 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {
591606 }
592607 return dest_i;
593608}
609
610test "utf8ToUtf16Le" {
611 var utf16le: [2]u16 = [_]u16{0} ** 2;
612 const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");
613 testing.expect(@as(usize, 2) == length);
614 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", @sliceToBytes(utf16le[0..]));
615}
616
617test "utf8ToUtf16LeWithNull" {
618 var bytes: [128]u8 = undefined;
619 const allocator = &std.heap.FixedBufferAllocator.init(bytes[0..]).allocator;
620 const utf16 = try utf8ToUtf16LeWithNull(allocator, "𐐷");
621 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc\x00\x00", @sliceToBytes(utf16[0..]));
622}