authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-09-21 15:19:14+02:00
committergravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-11-05 16:10:33+01:00
log6c4efab10611b9fe807f2517d2eec2ac60ae4f5c
tree0bd5f1a080a0e54210be0ed57fc57ddb9c6c92f9
parent2cce23062b95cf112ddbf4613c5a7e9ff60f0f88

std: Introduce std.unicode.utf8CountCodepoints

1 files changed, 29 insertions(+), 1 deletions(-)

lib/std/unicode.zig+29-1
......@@ -153,6 +153,23 @@ pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {
153153 return value;
154154}
155155
156/// Returns the length of a supplied UTF-8 string literal in terms of unicode
157/// codepoints.
158/// Asserts that the data is valid UTF-8.
159pub fn utf8CountCodepoints(s: []const u8) !usize {
160 var len: usize = 0;
161
162 var i: usize = 0;
163 while (i < s.len) : (len += 1) {
164 const n = try utf8ByteSequenceLength(s[i]);
165 if (i + n > s.len) return error.TruncatedInput;
166 _ = try utf8Decode(s[i .. i + n]);
167 i += n;
168 }
169
170 return len;
171}
172
156173pub fn utf8ValidateSlice(s: []const u8) bool {
157174 var i: usize = 0;
158175 while (i < s.len) {
......@@ -687,7 +704,6 @@ pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16Le
687704 }
688705}
689706
690/// Returns length of a supplied UTF-8 string literal. Asserts that the data is valid UTF-8.
691707fn calcUtf16LeLen(utf8: []const u8) usize {
692708 var src_i: usize = 0;
693709 var dest_len: usize = 0;
......@@ -757,3 +773,15 @@ test "utf8ToUtf16LeStringLiteral" {
757773 testing.expect(utf16[2] == 0);
758774 }
759775}
776
777fn testUtf8CountCodepoints() !void {
778 testing.expectEqual(@as(usize, 10), try utf8CountCodepoints("abcdefghij"));
779 testing.expectEqual(@as(usize, 10), try utf8CountCodepoints("äåéëþüúíóö"));
780 testing.expectEqual(@as(usize, 5), try utf8CountCodepoints("こんにちは"));
781 testing.expectError(error.Utf8EncodesSurrogateHalf, utf8CountCodepoints("\xED\xA0\x80"));
782}
783
784test "utf8 count codepoints" {
785 try testUtf8CountCodepoints();
786 comptime testUtf8CountCodepoints() catch unreachable;
787}