authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-12-28 18:13:50-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-12-28 18:13:50-05:00
logcb02125415b516b13824fe90aca19c6fc9203cda
tree3442197e12553da736a51d29bb08c368bf328525
parent48bf00bf340bb37155f9de85645e53f9304e7511
parentedb5deb39cc923d6fe2d3de507757d8ba9d74d68
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3987 from daurnimator/std.unicode-fixes

std.unicode fixes

2 files changed, 74 insertions(+), 66 deletions(-)

lib/std/unicode.zig+67-64
...@@ -6,7 +6,7 @@ const mem = std.mem;...@@ -6,7 +6,7 @@ const mem = std.mem;
66
7/// Returns how many bytes the UTF-8 representation would require7/// Returns how many bytes the UTF-8 representation would require
8/// for the given codepoint.8/// for the given codepoint.
9pub fn utf8CodepointSequenceLength(c: u32) !u3 {9pub fn utf8CodepointSequenceLength(c: u21) !u3 {
10 if (c < 0x80) return @as(u3, 1);10 if (c < 0x80) return @as(u3, 1);
11 if (c < 0x800) return @as(u3, 2);11 if (c < 0x800) return @as(u3, 2);
12 if (c < 0x10000) return @as(u3, 3);12 if (c < 0x10000) return @as(u3, 3);
...@@ -18,11 +18,13 @@ pub fn utf8CodepointSequenceLength(c: u32) !u3 {...@@ -18,11 +18,13 @@ pub fn utf8CodepointSequenceLength(c: u32) !u3 {
18/// returns a number 1-4 indicating the total length of the codepoint in bytes.18/// returns a number 1-4 indicating the total length of the codepoint in bytes.
19/// If this byte does not match the form of a UTF-8 start byte, returns Utf8InvalidStartByte.19/// If this byte does not match the form of a UTF-8 start byte, returns Utf8InvalidStartByte.
20pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {20pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
21 if (first_byte < 0b10000000) return @as(u3, 1);21 return switch (@clz(u8, ~first_byte)) {
22 if (first_byte & 0b11100000 == 0b11000000) return @as(u3, 2);22 0 => 1,
23 if (first_byte & 0b11110000 == 0b11100000) return @as(u3, 3);23 2 => 2,
24 if (first_byte & 0b11111000 == 0b11110000) return @as(u3, 4);24 3 => 3,
25 return error.Utf8InvalidStartByte;25 4 => 4,
26 else => error.Utf8InvalidStartByte,
27 };
26}28}
2729
28/// Encodes the given codepoint into a UTF-8 byte sequence.30/// Encodes the given codepoint into a UTF-8 byte sequence.
...@@ -30,7 +32,7 @@ pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {...@@ -30,7 +32,7 @@ pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
30/// out: the out buffer to write to. Must have a len >= utf8CodepointSequenceLength(c).32/// out: the out buffer to write to. Must have a len >= utf8CodepointSequenceLength(c).
31/// Errors: if c cannot be encoded in UTF-8.33/// Errors: if c cannot be encoded in UTF-8.
32/// Returns: the number of bytes written to out.34/// Returns: the number of bytes written to out.
33pub fn utf8Encode(c: u32, out: []u8) !u3 {35pub fn utf8Encode(c: u21, out: []u8) !u3 {
34 const length = try utf8CodepointSequenceLength(c);36 const length = try utf8CodepointSequenceLength(c);
35 assert(out.len >= length);37 assert(out.len >= length);
36 switch (length) {38 switch (length) {
...@@ -66,9 +68,9 @@ const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error...@@ -66,9 +68,9 @@ const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error
66/// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable.68/// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable.
67/// If you already know the length at comptime, you can call one of69/// If you already know the length at comptime, you can call one of
68/// utf8Decode2,utf8Decode3,utf8Decode4 directly instead of this function.70/// utf8Decode2,utf8Decode3,utf8Decode4 directly instead of this function.
69pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u32 {71pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u21 {
70 return switch (bytes.len) {72 return switch (bytes.len) {
71 1 => @as(u32, bytes[0]),73 1 => @as(u21, bytes[0]),
72 2 => utf8Decode2(bytes),74 2 => utf8Decode2(bytes),
73 3 => utf8Decode3(bytes),75 3 => utf8Decode3(bytes),
74 4 => utf8Decode4(bytes),76 4 => utf8Decode4(bytes),
...@@ -80,10 +82,10 @@ const Utf8Decode2Error = error{...@@ -80,10 +82,10 @@ const Utf8Decode2Error = error{
80 Utf8ExpectedContinuation,82 Utf8ExpectedContinuation,
81 Utf8OverlongEncoding,83 Utf8OverlongEncoding,
82};84};
83pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u32 {85pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {
84 assert(bytes.len == 2);86 assert(bytes.len == 2);
85 assert(bytes[0] & 0b11100000 == 0b11000000);87 assert(bytes[0] & 0b11100000 == 0b11000000);
86 var value: u32 = bytes[0] & 0b00011111;88 var value: u21 = bytes[0] & 0b00011111;
8789
88 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;90 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
89 value <<= 6;91 value <<= 6;
...@@ -99,10 +101,10 @@ const Utf8Decode3Error = error{...@@ -99,10 +101,10 @@ const Utf8Decode3Error = error{
99 Utf8OverlongEncoding,101 Utf8OverlongEncoding,
100 Utf8EncodesSurrogateHalf,102 Utf8EncodesSurrogateHalf,
101};103};
102pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u32 {104pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u21 {
103 assert(bytes.len == 3);105 assert(bytes.len == 3);
104 assert(bytes[0] & 0b11110000 == 0b11100000);106 assert(bytes[0] & 0b11110000 == 0b11100000);
105 var value: u32 = bytes[0] & 0b00001111;107 var value: u21 = bytes[0] & 0b00001111;
106108
107 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;109 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
108 value <<= 6;110 value <<= 6;
...@@ -123,10 +125,10 @@ const Utf8Decode4Error = error{...@@ -123,10 +125,10 @@ const Utf8Decode4Error = error{
123 Utf8OverlongEncoding,125 Utf8OverlongEncoding,
124 Utf8CodepointTooLarge,126 Utf8CodepointTooLarge,
125};127};
126pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u32 {128pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {
127 assert(bytes.len == 4);129 assert(bytes.len == 4);
128 assert(bytes[0] & 0b11111000 == 0b11110000);130 assert(bytes[0] & 0b11111000 == 0b11110000);
129 var value: u32 = bytes[0] & 0b00000111;131 var value: u21 = bytes[0] & 0b00000111;
130132
131 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;133 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
132 value <<= 6;134 value <<= 6;
...@@ -222,11 +224,11 @@ pub const Utf8Iterator = struct {...@@ -222,11 +224,11 @@ pub const Utf8Iterator = struct {
222 return it.bytes[it.i - cp_len .. it.i];224 return it.bytes[it.i - cp_len .. it.i];
223 }225 }
224226
225 pub fn nextCodepoint(it: *Utf8Iterator) ?u32 {227 pub fn nextCodepoint(it: *Utf8Iterator) ?u21 {
226 const slice = it.nextCodepointSlice() orelse return null;228 const slice = it.nextCodepointSlice() orelse return null;
227229
228 switch (slice.len) {230 switch (slice.len) {
229 1 => return @as(u32, slice[0]),231 1 => return @as(u21, slice[0]),
230 2 => return utf8Decode2(slice) catch unreachable,232 2 => return utf8Decode2(slice) catch unreachable,
231 3 => return utf8Decode3(slice) catch unreachable,233 3 => return utf8Decode3(slice) catch unreachable,
232 4 => return utf8Decode4(slice) catch unreachable,234 4 => return utf8Decode4(slice) catch unreachable,
...@@ -246,19 +248,19 @@ pub const Utf16LeIterator = struct {...@@ -246,19 +248,19 @@ pub const Utf16LeIterator = struct {
246 };248 };
247 }249 }
248250
249 pub fn nextCodepoint(it: *Utf16LeIterator) !?u32 {251 pub fn nextCodepoint(it: *Utf16LeIterator) !?u21 {
250 assert(it.i <= it.bytes.len);252 assert(it.i <= it.bytes.len);
251 if (it.i == it.bytes.len) return null;253 if (it.i == it.bytes.len) return null;
252 const c0: u32 = mem.readIntSliceLittle(u16, it.bytes[it.i .. it.i + 2]);254 const c0: u21 = mem.readIntSliceLittle(u16, it.bytes[it.i .. it.i + 2]);
253 if (c0 & ~@as(u32, 0x03ff) == 0xd800) {255 if (c0 & ~@as(u21, 0x03ff) == 0xd800) {
254 // surrogate pair256 // surrogate pair
255 it.i += 2;257 it.i += 2;
256 if (it.i >= it.bytes.len) return error.DanglingSurrogateHalf;258 if (it.i >= it.bytes.len) return error.DanglingSurrogateHalf;
257 const c1: u32 = mem.readIntSliceLittle(u16, it.bytes[it.i .. it.i + 2]);259 const c1: u21 = mem.readIntSliceLittle(u16, it.bytes[it.i .. it.i + 2]);
258 if (c1 & ~@as(u32, 0x03ff) != 0xdc00) return error.ExpectedSecondSurrogateHalf;260 if (c1 & ~@as(u21, 0x03ff) != 0xdc00) return error.ExpectedSecondSurrogateHalf;
259 it.i += 2;261 it.i += 2;
260 return 0x10000 + (((c0 & 0x03ff) << 10) | (c1 & 0x03ff));262 return 0x10000 + (((c0 & 0x03ff) << 10) | (c1 & 0x03ff));
261 } else if (c0 & ~@as(u32, 0x03ff) == 0xdc00) {263 } else if (c0 & ~@as(u21, 0x03ff) == 0xdc00) {
262 return error.UnexpectedSecondSurrogateHalf;264 return error.UnexpectedSecondSurrogateHalf;
263 } else {265 } else {
264 it.i += 2;266 it.i += 2;
...@@ -302,10 +304,10 @@ fn testUtf8EncodeError() void {...@@ -302,10 +304,10 @@ fn testUtf8EncodeError() void {
302 testErrorEncode(0xd800, array[0..], error.Utf8CannotEncodeSurrogateHalf);304 testErrorEncode(0xd800, array[0..], error.Utf8CannotEncodeSurrogateHalf);
303 testErrorEncode(0xdfff, array[0..], error.Utf8CannotEncodeSurrogateHalf);305 testErrorEncode(0xdfff, array[0..], error.Utf8CannotEncodeSurrogateHalf);
304 testErrorEncode(0x110000, array[0..], error.CodepointTooLarge);306 testErrorEncode(0x110000, array[0..], error.CodepointTooLarge);
305 testErrorEncode(0xffffffff, array[0..], error.CodepointTooLarge);307 testErrorEncode(0x1fffff, array[0..], error.CodepointTooLarge);
306}308}
307309
308fn testErrorEncode(codePoint: u32, array: []u8, expectedErr: anyerror) void {310fn testErrorEncode(codePoint: u21, array: []u8, expectedErr: anyerror) void {
309 testing.expectError(expectedErr, utf8Encode(codePoint, array));311 testing.expectError(expectedErr, utf8Encode(codePoint, array));
310}312}
311313
...@@ -453,11 +455,11 @@ fn testError(bytes: []const u8, expected_err: anyerror) void {...@@ -453,11 +455,11 @@ fn testError(bytes: []const u8, expected_err: anyerror) void {
453 testing.expectError(expected_err, testDecode(bytes));455 testing.expectError(expected_err, testDecode(bytes));
454}456}
455457
456fn testValid(bytes: []const u8, expected_codepoint: u32) void {458fn testValid(bytes: []const u8, expected_codepoint: u21) void {
457 testing.expect((testDecode(bytes) catch unreachable) == expected_codepoint);459 testing.expect((testDecode(bytes) catch unreachable) == expected_codepoint);
458}460}
459461
460fn testDecode(bytes: []const u8) !u32 {462fn testDecode(bytes: []const u8) !u21 {
461 const length = try utf8ByteSequenceLength(bytes[0]);463 const length = try utf8ByteSequenceLength(bytes[0]);
462 if (bytes.len < length) return error.UnexpectedEof;464 if (bytes.len < length) return error.UnexpectedEof;
463 testing.expect(bytes.len == length);465 testing.expect(bytes.len == length);
...@@ -555,9 +557,8 @@ pub fn utf8ToUtf16LeWithNull(allocator: *mem.Allocator, utf8: []const u8) ![]u16...@@ -555,9 +557,8 @@ pub fn utf8ToUtf16LeWithNull(allocator: *mem.Allocator, utf8: []const u8) ![]u16
555 const short = @intCast(u16, codepoint);557 const short = @intCast(u16, codepoint);
556 try result.append(mem.nativeToLittle(u16, short));558 try result.append(mem.nativeToLittle(u16, short));
557 } else {559 } else {
558 const short = @intCast(u16, codepoint - 0x10000);560 const high = @intCast(u16, (codepoint - 0x10000) >> 10) + 0xD800;
559 const high = (short >> 10) + 0xD800;561 const low = @intCast(u16, codepoint & 0x3FF) + 0xDC00;
560 const low = (short & 0x3FF) + 0xDC00;
561 var out: [2]u16 = undefined;562 var out: [2]u16 = undefined;
562 out[0] = mem.nativeToLittle(u16, high);563 out[0] = mem.nativeToLittle(u16, high);
563 out[1] = mem.nativeToLittle(u16, low);564 out[1] = mem.nativeToLittle(u16, low);
...@@ -575,48 +576,50 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {...@@ -575,48 +576,50 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {
575 var dest_i: usize = 0;576 var dest_i: usize = 0;
576 var src_i: usize = 0;577 var src_i: usize = 0;
577 while (src_i < utf8.len) {578 while (src_i < utf8.len) {
578 const byte = utf8[src_i];579 const n = utf8ByteSequenceLength(utf8[src_i]) catch return error.InvalidUtf8;
579 const n = @clz(u8, ~byte);580 const next_src_i = src_i + n;
580 switch (n) {581 const codepoint = utf8Decode(utf8[src_i..next_src_i]) catch return error.InvalidUtf8;
581 0 => {582 if (codepoint < 0x10000) {
582 utf16le[dest_i] = byte;583 const short = @intCast(u16, codepoint);
583 dest_i += 1;584 utf16le[dest_i] = mem.nativeToLittle(u16, short);
584 src_i += 1;585 dest_i += 1;
585 continue;586 } else {
586 },587 const high = @intCast(u16, (codepoint - 0x10000) >> 10) + 0xD800;
587 2, 3, 4 => {588 const low = @intCast(u16, codepoint & 0x3FF) + 0xDC00;
588 const next_src_i = src_i + n;589 utf16le[dest_i] = mem.nativeToLittle(u16, high);
589 const codepoint = utf8Decode(utf8[src_i..next_src_i]) catch return error.InvalidUtf8;590 utf16le[dest_i + 1] = mem.nativeToLittle(u16, low);
590 if (codepoint < 0x10000) {591 dest_i += 2;
591 const short = @intCast(u16, codepoint);
592 utf16le[dest_i] = mem.nativeToLittle(u16, short);
593 dest_i += 1;
594 } else {
595 const short = @intCast(u16, codepoint - 0x10000);
596 const high = (short >> 10) + 0xD800;
597 const low = (short & 0x3FF) + 0xDC00;
598 utf16le[dest_i] = mem.nativeToLittle(u16, high);
599 utf16le[dest_i + 1] = mem.nativeToLittle(u16, low);
600 dest_i += 2;
601 }
602 src_i = next_src_i;
603 },
604 else => return error.InvalidUtf8,
605 }592 }
593 src_i = next_src_i;
606 }594 }
607 return dest_i;595 return dest_i;
608}596}
609597
610test "utf8ToUtf16Le" {598test "utf8ToUtf16Le" {
611 var utf16le: [2]u16 = [_]u16{0} ** 2;599 var utf16le: [2]u16 = [_]u16{0} ** 2;
612 const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");600 {
613 testing.expect(@as(usize, 2) == length);601 const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");
614 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", @sliceToBytes(utf16le[0..]));602 testing.expectEqual(@as(usize, 2), length);
603 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", @sliceToBytes(utf16le[0..]));
604 }
605 {
606 const length = try utf8ToUtf16Le(utf16le[0..], "\u{10FFFF}");
607 testing.expectEqual(@as(usize, 2), length);
608 testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", @sliceToBytes(utf16le[0..]));
609 }
615}610}
616611
617test "utf8ToUtf16LeWithNull" {612test "utf8ToUtf16LeWithNull" {
618 var bytes: [128]u8 = undefined;613 {
619 const allocator = &std.heap.FixedBufferAllocator.init(bytes[0..]).allocator;614 var bytes: [128]u8 = undefined;
620 const utf16 = try utf8ToUtf16LeWithNull(allocator, "𐐷");615 const allocator = &std.heap.FixedBufferAllocator.init(bytes[0..]).allocator;
621 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc\x00\x00", @sliceToBytes(utf16[0..]));616 const utf16 = try utf8ToUtf16LeWithNull(allocator, "𐐷");
617 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc\x00\x00", @sliceToBytes(utf16[0..]));
618 }
619 {
620 var bytes: [128]u8 = undefined;
621 const allocator = &std.heap.FixedBufferAllocator.init(bytes[0..]).allocator;
622 const utf16 = try utf8ToUtf16LeWithNull(allocator, "\u{10FFFF}");
623 testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf\x00\x00", @sliceToBytes(utf16[0..]));
624 }
622}625}
lib/std/unicode/throughput_test.zig+7-2
...@@ -6,18 +6,23 @@ pub fn main() !void {...@@ -6,18 +6,23 @@ pub fn main() !void {
66
7 const args = try std.process.argsAlloc(std.heap.page_allocator);7 const args = try std.process.argsAlloc(std.heap.page_allocator);
88
9 // Warm up runs
10 var buffer0: [32767]u16 align(4096) = undefined;
11 _ = try std.unicode.utf8ToUtf16Le(&buffer0, args[1]);
12 _ = try std.unicode.utf8ToUtf16Le_better(&buffer0, args[1]);
13
9 @fence(.SeqCst);14 @fence(.SeqCst);
10 var timer = try std.time.Timer.start();15 var timer = try std.time.Timer.start();
11 @fence(.SeqCst);16 @fence(.SeqCst);
1217
13 var buffer1: [32767]u16 = undefined;18 var buffer1: [32767]u16 align(4096) = undefined;
14 _ = try std.unicode.utf8ToUtf16Le(&buffer1, args[1]);19 _ = try std.unicode.utf8ToUtf16Le(&buffer1, args[1]);
1520
16 @fence(.SeqCst);21 @fence(.SeqCst);
17 const elapsed_ns_orig = timer.lap();22 const elapsed_ns_orig = timer.lap();
18 @fence(.SeqCst);23 @fence(.SeqCst);
1924
20 var buffer2: [32767]u16 = undefined;25 var buffer2: [32767] u16 align(4096) = undefined;
21 _ = try std.unicode.utf8ToUtf16Le_better(&buffer2, args[1]);26 _ = try std.unicode.utf8ToUtf16Le_better(&buffer2, args[1]);
2227
23 @fence(.SeqCst);28 @fence(.SeqCst);