authorgravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-28 14:45:52+11:00
committergravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-28 16:49:59+11:00
logedb5deb39cc923d6fe2d3de507757d8ba9d74d68
tree4de2d4b045f3168182cff64f0150ac635476543b
parentab6065407d74fc8d63d398c60f9fe653374d9d6d
signaturelock-open Commit is signed but in an unrecognized format.

std: unicode codepoints are 21 bits

1 files changed, 22 insertions(+), 22 deletions(-)

lib/std/unicode.zig+22-22
...@@ -6,7 +6,7 @@ const mem = std.mem;...@@ -6,7 +6,7 @@ const mem = std.mem;
66
7/// Returns how many bytes the UTF-8 representation would require7/// Returns how many bytes the UTF-8 representation would require
8/// for the given codepoint.8/// for the given codepoint.
9pub fn utf8CodepointSequenceLength(c: u32) !u3 {9pub fn utf8CodepointSequenceLength(c: u21) !u3 {
10 if (c < 0x80) return @as(u3, 1);10 if (c < 0x80) return @as(u3, 1);
11 if (c < 0x800) return @as(u3, 2);11 if (c < 0x800) return @as(u3, 2);
12 if (c < 0x10000) return @as(u3, 3);12 if (c < 0x10000) return @as(u3, 3);
...@@ -32,7 +32,7 @@ pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {...@@ -32,7 +32,7 @@ pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
32/// out: the out buffer to write to. Must have a len >= utf8CodepointSequenceLength(c).32/// out: the out buffer to write to. Must have a len >= utf8CodepointSequenceLength(c).
33/// Errors: if c cannot be encoded in UTF-8.33/// Errors: if c cannot be encoded in UTF-8.
34/// Returns: the number of bytes written to out.34/// Returns: the number of bytes written to out.
35pub fn utf8Encode(c: u32, out: []u8) !u3 {35pub fn utf8Encode(c: u21, out: []u8) !u3 {
36 const length = try utf8CodepointSequenceLength(c);36 const length = try utf8CodepointSequenceLength(c);
37 assert(out.len >= length);37 assert(out.len >= length);
38 switch (length) {38 switch (length) {
...@@ -68,9 +68,9 @@ const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error...@@ -68,9 +68,9 @@ const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error
68/// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable.68/// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable.
69/// If you already know the length at comptime, you can call one of69/// If you already know the length at comptime, you can call one of
70/// utf8Decode2,utf8Decode3,utf8Decode4 directly instead of this function.70/// utf8Decode2,utf8Decode3,utf8Decode4 directly instead of this function.
71pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u32 {71pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u21 {
72 return switch (bytes.len) {72 return switch (bytes.len) {
73 1 => @as(u32, bytes[0]),73 1 => @as(u21, bytes[0]),
74 2 => utf8Decode2(bytes),74 2 => utf8Decode2(bytes),
75 3 => utf8Decode3(bytes),75 3 => utf8Decode3(bytes),
76 4 => utf8Decode4(bytes),76 4 => utf8Decode4(bytes),
...@@ -82,10 +82,10 @@ const Utf8Decode2Error = error{...@@ -82,10 +82,10 @@ const Utf8Decode2Error = error{
82 Utf8ExpectedContinuation,82 Utf8ExpectedContinuation,
83 Utf8OverlongEncoding,83 Utf8OverlongEncoding,
84};84};
85pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u32 {85pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {
86 assert(bytes.len == 2);86 assert(bytes.len == 2);
87 assert(bytes[0] & 0b11100000 == 0b11000000);87 assert(bytes[0] & 0b11100000 == 0b11000000);
88 var value: u32 = bytes[0] & 0b00011111;88 var value: u21 = bytes[0] & 0b00011111;
8989
90 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;90 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
91 value <<= 6;91 value <<= 6;
...@@ -101,10 +101,10 @@ const Utf8Decode3Error = error{...@@ -101,10 +101,10 @@ const Utf8Decode3Error = error{
101 Utf8OverlongEncoding,101 Utf8OverlongEncoding,
102 Utf8EncodesSurrogateHalf,102 Utf8EncodesSurrogateHalf,
103};103};
104pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u32 {104pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u21 {
105 assert(bytes.len == 3);105 assert(bytes.len == 3);
106 assert(bytes[0] & 0b11110000 == 0b11100000);106 assert(bytes[0] & 0b11110000 == 0b11100000);
107 var value: u32 = bytes[0] & 0b00001111;107 var value: u21 = bytes[0] & 0b00001111;
108108
109 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;109 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
110 value <<= 6;110 value <<= 6;
...@@ -125,10 +125,10 @@ const Utf8Decode4Error = error{...@@ -125,10 +125,10 @@ const Utf8Decode4Error = error{
125 Utf8OverlongEncoding,125 Utf8OverlongEncoding,
126 Utf8CodepointTooLarge,126 Utf8CodepointTooLarge,
127};127};
128pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u32 {128pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {
129 assert(bytes.len == 4);129 assert(bytes.len == 4);
130 assert(bytes[0] & 0b11111000 == 0b11110000);130 assert(bytes[0] & 0b11111000 == 0b11110000);
131 var value: u32 = bytes[0] & 0b00000111;131 var value: u21 = bytes[0] & 0b00000111;
132132
133 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;133 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
134 value <<= 6;134 value <<= 6;
...@@ -224,11 +224,11 @@ pub const Utf8Iterator = struct {...@@ -224,11 +224,11 @@ pub const Utf8Iterator = struct {
224 return it.bytes[it.i - cp_len .. it.i];224 return it.bytes[it.i - cp_len .. it.i];
225 }225 }
226226
227 pub fn nextCodepoint(it: *Utf8Iterator) ?u32 {227 pub fn nextCodepoint(it: *Utf8Iterator) ?u21 {
228 const slice = it.nextCodepointSlice() orelse return null;228 const slice = it.nextCodepointSlice() orelse return null;
229229
230 switch (slice.len) {230 switch (slice.len) {
231 1 => return @as(u32, slice[0]),231 1 => return @as(u21, slice[0]),
232 2 => return utf8Decode2(slice) catch unreachable,232 2 => return utf8Decode2(slice) catch unreachable,
233 3 => return utf8Decode3(slice) catch unreachable,233 3 => return utf8Decode3(slice) catch unreachable,
234 4 => return utf8Decode4(slice) catch unreachable,234 4 => return utf8Decode4(slice) catch unreachable,
...@@ -248,19 +248,19 @@ pub const Utf16LeIterator = struct {...@@ -248,19 +248,19 @@ pub const Utf16LeIterator = struct {
248 };248 };
249 }249 }
250250
251 pub fn nextCodepoint(it: *Utf16LeIterator) !?u32 {251 pub fn nextCodepoint(it: *Utf16LeIterator) !?u21 {
252 assert(it.i <= it.bytes.len);252 assert(it.i <= it.bytes.len);
253 if (it.i == it.bytes.len) return null;253 if (it.i == it.bytes.len) return null;
254 const c0: u32 = mem.readIntSliceLittle(u16, it.bytes[it.i .. it.i + 2]);254 const c0: u21 = mem.readIntSliceLittle(u16, it.bytes[it.i .. it.i + 2]);
255 if (c0 & ~@as(u32, 0x03ff) == 0xd800) {255 if (c0 & ~@as(u21, 0x03ff) == 0xd800) {
256 // surrogate pair256 // surrogate pair
257 it.i += 2;257 it.i += 2;
258 if (it.i >= it.bytes.len) return error.DanglingSurrogateHalf;258 if (it.i >= it.bytes.len) return error.DanglingSurrogateHalf;
259 const c1: u32 = mem.readIntSliceLittle(u16, it.bytes[it.i .. it.i + 2]);259 const c1: u21 = mem.readIntSliceLittle(u16, it.bytes[it.i .. it.i + 2]);
260 if (c1 & ~@as(u32, 0x03ff) != 0xdc00) return error.ExpectedSecondSurrogateHalf;260 if (c1 & ~@as(u21, 0x03ff) != 0xdc00) return error.ExpectedSecondSurrogateHalf;
261 it.i += 2;261 it.i += 2;
262 return 0x10000 + (((c0 & 0x03ff) << 10) | (c1 & 0x03ff));262 return 0x10000 + (((c0 & 0x03ff) << 10) | (c1 & 0x03ff));
263 } else if (c0 & ~@as(u32, 0x03ff) == 0xdc00) {263 } else if (c0 & ~@as(u21, 0x03ff) == 0xdc00) {
264 return error.UnexpectedSecondSurrogateHalf;264 return error.UnexpectedSecondSurrogateHalf;
265 } else {265 } else {
266 it.i += 2;266 it.i += 2;
...@@ -304,10 +304,10 @@ fn testUtf8EncodeError() void {...@@ -304,10 +304,10 @@ fn testUtf8EncodeError() void {
304 testErrorEncode(0xd800, array[0..], error.Utf8CannotEncodeSurrogateHalf);304 testErrorEncode(0xd800, array[0..], error.Utf8CannotEncodeSurrogateHalf);
305 testErrorEncode(0xdfff, array[0..], error.Utf8CannotEncodeSurrogateHalf);305 testErrorEncode(0xdfff, array[0..], error.Utf8CannotEncodeSurrogateHalf);
306 testErrorEncode(0x110000, array[0..], error.CodepointTooLarge);306 testErrorEncode(0x110000, array[0..], error.CodepointTooLarge);
307 testErrorEncode(0xffffffff, array[0..], error.CodepointTooLarge);307 testErrorEncode(0x1fffff, array[0..], error.CodepointTooLarge);
308}308}
309309
310fn testErrorEncode(codePoint: u32, array: []u8, expectedErr: anyerror) void {310fn testErrorEncode(codePoint: u21, array: []u8, expectedErr: anyerror) void {
311 testing.expectError(expectedErr, utf8Encode(codePoint, array));311 testing.expectError(expectedErr, utf8Encode(codePoint, array));
312}312}
313313
...@@ -455,11 +455,11 @@ fn testError(bytes: []const u8, expected_err: anyerror) void {...@@ -455,11 +455,11 @@ fn testError(bytes: []const u8, expected_err: anyerror) void {
455 testing.expectError(expected_err, testDecode(bytes));455 testing.expectError(expected_err, testDecode(bytes));
456}456}
457457
458fn testValid(bytes: []const u8, expected_codepoint: u32) void {458fn testValid(bytes: []const u8, expected_codepoint: u21) void {
459 testing.expect((testDecode(bytes) catch unreachable) == expected_codepoint);459 testing.expect((testDecode(bytes) catch unreachable) == expected_codepoint);
460}460}
461461
462fn testDecode(bytes: []const u8) !u32 {462fn testDecode(bytes: []const u8) !u21 {
463 const length = try utf8ByteSequenceLength(bytes[0]);463 const length = try utf8ByteSequenceLength(bytes[0]);
464 if (bytes.len < length) return error.UnexpectedEof;464 if (bytes.len < length) return error.UnexpectedEof;
465 testing.expect(bytes.len == length);465 testing.expect(bytes.len == length);