1const std = @import("./std.zig");
2const builtin = @import("builtin");
3const assert = std.debug.assert;
4const testing = std.testing;
5const mem = std.mem;
6const native_endian = builtin.cpu.arch.endian();
7const Allocator = std.mem.Allocator;
8
9/// Use this to replace an unknown, unrecognized, or unrepresentable character.
10///
11/// See also: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character
12pub const replacement_character: u21 = 0xFFFD;
13pub const replacement_character_utf8: [3]u8 = utf8EncodeComptime(replacement_character);
14
15/// Returns how many bytes the UTF-8 representation would require
16/// for the given codepoint.
17pub fn utf8CodepointSequenceLength(c: u21) !u3 {
18 if (c < 0x80) return @as(u3, 1);
19 if (c < 0x800) return @as(u3, 2);
20 if (c < 0x10000) return @as(u3, 3);
21 if (c < 0x110000) return @as(u3, 4);
22 return error.CodepointTooLarge;
23}
24
25/// Given the first byte of a UTF-8 codepoint,
26/// returns a number 1-4 indicating the total length of the codepoint in bytes.
27/// If this byte does not match the form of a UTF-8 start byte, returns Utf8InvalidStartByte.
28pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
29 // The switch is optimized much better than a "smart" approach using @clz
30 return switch (first_byte) {
31 0b0000_0000...0b0111_1111 => 1,
32 0b1100_0000...0b1101_1111 => 2,
33 0b1110_0000...0b1110_1111 => 3,
34 0b1111_0000...0b1111_0111 => 4,
35 else => error.Utf8InvalidStartByte,
36 };
37}
38
39/// Encodes the given codepoint into a UTF-8 byte sequence.
40/// c: the codepoint.
41/// out: the out buffer to write to. Must have a len >= utf8CodepointSequenceLength(c).
42/// Errors: if c cannot be encoded in UTF-8.
43/// Returns: the number of bytes written to out.
44pub fn utf8Encode(c: u21, out: []u8) error{ Utf8CannotEncodeSurrogateHalf, CodepointTooLarge }!u3 {
45 return utf8EncodeImpl(c, out, .cannot_encode_surrogate_half);
46}
47
48const Surrogates = enum {
49 cannot_encode_surrogate_half,
50 can_encode_surrogate_half,
51};
52
53fn utf8EncodeImpl(c: u21, out: []u8, comptime surrogates: Surrogates) !u3 {
54 const length = try utf8CodepointSequenceLength(c);
55 assert(out.len >= length);
56 switch (length) {
57 // The pattern for each is the same
58 // - Increasing the initial shift by 6 each time
59 // - Each time after the first shorten the shifted
60 // value to a max of 0b111111 (63)
61 1 => out[0] = @as(u8, @intCast(c)), // Can just do 0 + codepoint for initial range
62 2 => {
63 out[0] = @as(u8, @intCast(0b11000000 | (c >> 6)));
64 out[1] = @as(u8, @intCast(0b10000000 | (c & 0b111111)));
65 },
66 3 => {
67 if (surrogates == .cannot_encode_surrogate_half and isSurrogateCodepoint(c)) {
68 return error.Utf8CannotEncodeSurrogateHalf;
69 }
70 out[0] = @as(u8, @intCast(0b11100000 | (c >> 12)));
71 out[1] = @as(u8, @intCast(0b10000000 | ((c >> 6) & 0b111111)));
72 out[2] = @as(u8, @intCast(0b10000000 | (c & 0b111111)));
73 },
74 4 => {
75 out[0] = @as(u8, @intCast(0b11110000 | (c >> 18)));
76 out[1] = @as(u8, @intCast(0b10000000 | ((c >> 12) & 0b111111)));
77 out[2] = @as(u8, @intCast(0b10000000 | ((c >> 6) & 0b111111)));
78 out[3] = @as(u8, @intCast(0b10000000 | (c & 0b111111)));
79 },
80 else => unreachable,
81 }
82 return length;
83}
84
85pub inline fn utf8EncodeComptime(comptime c: u21) [
86 utf8CodepointSequenceLength(c) catch |err|
87 @compileError(@errorName(err))
88]u8 {
89 comptime var result: [
90 utf8CodepointSequenceLength(c) catch
91 unreachable
92 ]u8 = undefined;
93 comptime assert((utf8Encode(c, &result) catch |err|
94 @compileError(@errorName(err))) == result.len);
95 return result;
96}
97
98const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error;
99
100/// Deprecated. This function has an awkward API that is too easy to use incorrectly.
101pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u21 {
102 return switch (bytes.len) {
103 1 => bytes[0],
104 2 => utf8Decode2(bytes[0..2].*),
105 3 => utf8Decode3(bytes[0..3].*),
106 4 => utf8Decode4(bytes[0..4].*),
107 else => unreachable,
108 };
109}
110
111const Utf8Decode2Error = error{
112 Utf8ExpectedContinuation,
113 Utf8OverlongEncoding,
114};
115pub fn utf8Decode2(bytes: [2]u8) Utf8Decode2Error!u21 {
116 assert(bytes[0] & 0b11100000 == 0b11000000);
117 var value: u21 = bytes[0] & 0b00011111;
118
119 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
120 value <<= 6;
121 value |= bytes[1] & 0b00111111;
122
123 if (value < 0x80) return error.Utf8OverlongEncoding;
124
125 return value;
126}
127
128const Utf8Decode3Error = Utf8Decode3AllowSurrogateHalfError || error{
129 Utf8EncodesSurrogateHalf,
130};
131pub fn utf8Decode3(bytes: [3]u8) Utf8Decode3Error!u21 {
132 const value = try utf8Decode3AllowSurrogateHalf(bytes);
133
134 if (0xd800 <= value and value <= 0xdfff) return error.Utf8EncodesSurrogateHalf;
135
136 return value;
137}
138
139const Utf8Decode3AllowSurrogateHalfError = error{
140 Utf8ExpectedContinuation,
141 Utf8OverlongEncoding,
142};
143pub fn utf8Decode3AllowSurrogateHalf(bytes: [3]u8) Utf8Decode3AllowSurrogateHalfError!u21 {
144 assert(bytes[0] & 0b11110000 == 0b11100000);
145 var value: u21 = bytes[0] & 0b00001111;
146
147 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
148 value <<= 6;
149 value |= bytes[1] & 0b00111111;
150
151 if (bytes[2] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
152 value <<= 6;
153 value |= bytes[2] & 0b00111111;
154
155 if (value < 0x800) return error.Utf8OverlongEncoding;
156
157 return value;
158}
159
160const Utf8Decode4Error = error{
161 Utf8ExpectedContinuation,
162 Utf8OverlongEncoding,
163 Utf8CodepointTooLarge,
164};
165pub fn utf8Decode4(bytes: [4]u8) Utf8Decode4Error!u21 {
166 assert(bytes[0] & 0b11111000 == 0b11110000);
167 var value: u21 = bytes[0] & 0b00000111;
168
169 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
170 value <<= 6;
171 value |= bytes[1] & 0b00111111;
172
173 if (bytes[2] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
174 value <<= 6;
175 value |= bytes[2] & 0b00111111;
176
177 if (bytes[3] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
178 value <<= 6;
179 value |= bytes[3] & 0b00111111;
180
181 if (value < 0x10000) return error.Utf8OverlongEncoding;
182 if (value > 0x10FFFF) return error.Utf8CodepointTooLarge;
183
184 return value;
185}
186
187/// Returns true if the given unicode codepoint can be encoded in UTF-8.
188pub fn utf8ValidCodepoint(value: u21) bool {
189 return switch (value) {
190 0xD800...0xDFFF => false, // Surrogates range
191 0x110000...0x1FFFFF => false, // Above the maximum codepoint value
192 else => true,
193 };
194}
195
196/// Returns the length of a supplied UTF-8 string literal in terms of unicode
197/// codepoints.
198pub fn utf8CountCodepoints(s: []const u8) !usize {
199 var len: usize = 0;
200
201 const N = @sizeOf(usize);
202 const MASK = 0x80 * (std.math.maxInt(usize) / 0xff);
203
204 var i: usize = 0;
205 while (i < s.len) {
206 // Fast path for ASCII sequences
207 while (i + N <= s.len) : (i += N) {
208 const v = mem.readInt(usize, s[i..][0..N], native_endian);
209 if (v & MASK != 0) break;
210 len += N;
211 }
212
213 if (i < s.len) {
214 const n = try utf8ByteSequenceLength(s[i]);
215 if (i + n > s.len) return error.TruncatedInput;
216
217 switch (n) {
218 1 => {}, // ASCII, no validation needed
219 else => _ = try utf8Decode(s[i..][0..n]),
220 }
221
222 i += n;
223 len += 1;
224 }
225 }
226
227 return len;
228}
229
230/// Returns true if the input consists entirely of UTF-8 codepoints
231pub fn utf8ValidateSlice(input: []const u8) bool {
232 return utf8ValidateSliceImpl(input, .cannot_encode_surrogate_half);
233}
234
235fn utf8ValidateSliceImpl(input: []const u8, comptime surrogates: Surrogates) bool {
236 var remaining = input;
237
238 if (std.simd.suggestVectorLength(u8)) |chunk_len| {
239 const Chunk = @Vector(chunk_len, u8);
240
241 // Fast path. Check for and skip ASCII characters at the start of the input.
242 while (remaining.len >= chunk_len) {
243 const chunk: Chunk = remaining[0..chunk_len].*;
244 const mask: Chunk = @splat(0x80);
245 if (@reduce(.Or, chunk & mask == mask)) {
246 // found a non ASCII byte
247 break;
248 }
249 remaining = remaining[chunk_len..];
250 }
251 }
252
253 // default lowest and highest continuation byte
254 const lo_cb = 0b10000000;
255 const hi_cb = 0b10111111;
256
257 const min_non_ascii_codepoint = 0x80;
258
259 // The first nibble is used to identify the continuation byte range to
260 // accept. The second nibble is the size.
261 const xx = 0xF1; // invalid: size 1
262 const as = 0xF0; // ASCII: size 1
263 const s1 = 0x02; // accept 0, size 2
264 const s2 = switch (surrogates) {
265 .cannot_encode_surrogate_half => 0x13, // accept 1, size 3
266 .can_encode_surrogate_half => 0x03, // accept 0, size 3
267 };
268 const s3 = 0x03; // accept 0, size 3
269 const s4 = switch (surrogates) {
270 .cannot_encode_surrogate_half => 0x23, // accept 2, size 3
271 .can_encode_surrogate_half => 0x03, // accept 0, size 3
272 };
273 const s5 = 0x34; // accept 3, size 4
274 const s6 = 0x04; // accept 0, size 4
275 const s7 = 0x44; // accept 4, size 4
276
277 // Information about the first byte in a UTF-8 sequence.
278 const first = comptime first: {
279 const a: [128]u8 = @splat(as);
280 const b: [64]u8 = @splat(xx);
281 const c: [64]u8 = .{
282 xx, xx, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1,
283 s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1,
284 s2, s3, s3, s3, s3, s3, s3, s3, s3, s3, s3, s3, s3, s4, s3, s3,
285 s5, s6, s6, s6, s7, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx,
286 };
287 break :first a ++ b ++ c;
288 };
289
290 const n = remaining.len;
291 var i: usize = 0;
292 while (i < n) {
293 const first_byte = remaining[i];
294 if (first_byte < min_non_ascii_codepoint) {
295 i += 1;
296 continue;
297 }
298
299 const info = first[first_byte];
300 if (info == xx) {
301 return false; // Illegal starter byte.
302 }
303
304 const size = info & 7;
305 if (i + size > n) {
306 return false; // Short or invalid.
307 }
308
309 // Figure out the acceptable low and high continuation bytes, starting
310 // with our defaults.
311 var accept_lo: u8 = lo_cb;
312 var accept_hi: u8 = hi_cb;
313
314 switch (info >> 4) {
315 0 => {},
316 1 => accept_lo = 0xA0,
317 2 => accept_hi = 0x9F,
318 3 => accept_lo = 0x90,
319 4 => accept_hi = 0x8F,
320 else => unreachable,
321 }
322
323 const c1 = remaining[i + 1];
324 if (c1 < accept_lo or accept_hi < c1) {
325 return false;
326 }
327
328 switch (size) {
329 2 => i += 2,
330 3 => {
331 const c2 = remaining[i + 2];
332 if (c2 < lo_cb or hi_cb < c2) {
333 return false;
334 }
335 i += 3;
336 },
337 4 => {
338 const c2 = remaining[i + 2];
339 if (c2 < lo_cb or hi_cb < c2) {
340 return false;
341 }
342 const c3 = remaining[i + 3];
343 if (c3 < lo_cb or hi_cb < c3) {
344 return false;
345 }
346 i += 4;
347 },
348 else => unreachable,
349 }
350 }
351
352 return true;
353}
354
355/// Utf8View iterates the code points of a utf-8 encoded string.
356///
357/// ```
358/// var utf8 = (try std.unicode.Utf8View.init("hi there")).iterator();
359/// while (utf8.nextCodepointSlice()) |codepoint| {
360/// std.debug.print("got codepoint {s}\n", .{codepoint});
361/// }
362/// ```
363pub const Utf8View = struct {
364 bytes: []const u8,
365
366 pub fn init(s: []const u8) !Utf8View {
367 if (!utf8ValidateSlice(s)) {
368 return error.InvalidUtf8;
369 }
370
371 return initUnchecked(s);
372 }
373
374 pub fn initUnchecked(s: []const u8) Utf8View {
375 return Utf8View{ .bytes = s };
376 }
377
378 pub inline fn initComptime(comptime s: []const u8) Utf8View {
379 return comptime if (init(s)) |r| r else |err| switch (err) {
380 error.InvalidUtf8 => {
381 @compileError("invalid utf8");
382 },
383 };
384 }
385
386 pub fn iterator(s: Utf8View) Utf8Iterator {
387 return Utf8Iterator{
388 .bytes = s.bytes,
389 .i = 0,
390 };
391 }
392};
393
394pub const Utf8Iterator = struct {
395 bytes: []const u8,
396 i: usize,
397
398 pub fn nextCodepointSlice(it: *Utf8Iterator) ?[]const u8 {
399 if (it.i >= it.bytes.len) {
400 return null;
401 }
402
403 const cp_len = utf8ByteSequenceLength(it.bytes[it.i]) catch unreachable;
404 it.i += cp_len;
405 return it.bytes[it.i - cp_len .. it.i];
406 }
407
408 pub fn nextCodepoint(it: *Utf8Iterator) ?u21 {
409 const slice = it.nextCodepointSlice() orelse return null;
410 return utf8Decode(slice) catch unreachable;
411 }
412
413 /// Look ahead at the next n codepoints without advancing the iterator.
414 /// If fewer than n codepoints are available, then return the remainder of the string.
415 pub fn peek(it: *Utf8Iterator, n: usize) []const u8 {
416 const original_i = it.i;
417 defer it.i = original_i;
418
419 var end_ix = original_i;
420 var found: usize = 0;
421 while (found < n) : (found += 1) {
422 const next_codepoint = it.nextCodepointSlice() orelse return it.bytes[original_i..];
423 end_ix += next_codepoint.len;
424 }
425
426 return it.bytes[original_i..end_ix];
427 }
428
429 /// Look ahead at the next codepoint without advancing the iterator.
430 /// If no codepoints exist, then returns null.
431 pub fn peekCodepoint(it: *Utf8Iterator) ?u21 {
432 const original_i = it.i;
433 defer it.i = original_i;
434
435 return it.nextCodepoint();
436 }
437};
438
439pub fn utf16IsHighSurrogate(c: u16) bool {
440 return c & ~@as(u16, 0x03ff) == 0xd800;
441}
442
443pub fn utf16IsLowSurrogate(c: u16) bool {
444 return c & ~@as(u16, 0x03ff) == 0xdc00;
445}
446
447/// Returns how many code units the UTF-16 representation would require
448/// for the given codepoint.
449pub fn utf16CodepointSequenceLength(c: u21) !u2 {
450 if (c <= 0xFFFF) return 1;
451 if (c <= 0x10FFFF) return 2;
452 return error.CodepointTooLarge;
453}
454
455test utf16CodepointSequenceLength {
456 try testing.expectEqual(@as(u2, 1), try utf16CodepointSequenceLength('a'));
457 try testing.expectEqual(@as(u2, 1), try utf16CodepointSequenceLength(0xFFFF));
458 try testing.expectEqual(@as(u2, 2), try utf16CodepointSequenceLength(0x10000));
459 try testing.expectEqual(@as(u2, 2), try utf16CodepointSequenceLength(0x10FFFF));
460 try testing.expectError(error.CodepointTooLarge, utf16CodepointSequenceLength(0x110000));
461}
462
463/// Given the first code unit of a UTF-16 codepoint, returns a number 1-2
464/// indicating the total length of the codepoint in UTF-16 code units.
465/// If this code unit does not match the form of a UTF-16 start code unit, returns Utf16InvalidStartCodeUnit.
466pub fn utf16CodeUnitSequenceLength(first_code_unit: u16) !u2 {
467 if (utf16IsHighSurrogate(first_code_unit)) return 2;
468 if (utf16IsLowSurrogate(first_code_unit)) return error.Utf16InvalidStartCodeUnit;
469 return 1;
470}
471
472test utf16CodeUnitSequenceLength {
473 try testing.expectEqual(@as(u2, 1), try utf16CodeUnitSequenceLength('a'));
474 try testing.expectEqual(@as(u2, 1), try utf16CodeUnitSequenceLength(0xFFFF));
475 try testing.expectEqual(@as(u2, 2), try utf16CodeUnitSequenceLength(0xDBFF));
476 try testing.expectError(error.Utf16InvalidStartCodeUnit, utf16CodeUnitSequenceLength(0xDFFF));
477}
478
479/// Decodes the codepoint encoded in the given pair of UTF-16 code units.
480/// Asserts that `surrogate_pair.len >= 2` and that the first code unit is a high surrogate.
481/// If the second code unit is not a low surrogate, error.ExpectedSecondSurrogateHalf is returned.
482pub fn utf16DecodeSurrogatePair(surrogate_pair: []const u16) !u21 {
483 assert(surrogate_pair.len >= 2);
484 assert(utf16IsHighSurrogate(surrogate_pair[0]));
485 const high_half: u21 = surrogate_pair[0];
486 const low_half = surrogate_pair[1];
487 if (!utf16IsLowSurrogate(low_half)) return error.ExpectedSecondSurrogateHalf;
488 return 0x10000 + ((high_half & 0x03ff) << 10) | (low_half & 0x03ff);
489}
490
491pub const Utf16LeIterator = struct {
492 bytes: []const u8,
493 i: usize,
494
495 pub fn init(s: []const u16) Utf16LeIterator {
496 return Utf16LeIterator{
497 .bytes = mem.sliceAsBytes(s),
498 .i = 0,
499 };
500 }
501
502 pub const NextCodepointError = error{ DanglingSurrogateHalf, ExpectedSecondSurrogateHalf, UnexpectedSecondSurrogateHalf };
503
504 pub fn nextCodepoint(it: *Utf16LeIterator) NextCodepointError!?u21 {
505 assert(it.i <= it.bytes.len);
506 if (it.i == it.bytes.len) return null;
507 var code_units: [2]u16 = undefined;
508 code_units[0] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
509 it.i += 2;
510 if (utf16IsHighSurrogate(code_units[0])) {
511 // surrogate pair
512 if (it.i >= it.bytes.len) return error.DanglingSurrogateHalf;
513 code_units[1] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
514 const codepoint = try utf16DecodeSurrogatePair(&code_units);
515 it.i += 2;
516 return codepoint;
517 } else if (utf16IsLowSurrogate(code_units[0])) {
518 return error.UnexpectedSecondSurrogateHalf;
519 } else {
520 return code_units[0];
521 }
522 }
523};
524
525/// Returns the length of a supplied UTF-16 string literal in terms of unicode
526/// codepoints.
527pub fn utf16CountCodepoints(utf16le: []const u16) !usize {
528 var len: usize = 0;
529 var it = Utf16LeIterator.init(utf16le);
530 while (try it.nextCodepoint()) |_| len += 1;
531 return len;
532}
533
534fn testUtf16CountCodepoints() !void {
535 try testing.expectEqual(
536 @as(usize, 1),
537 try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("a")),
538 );
539 try testing.expectEqual(
540 @as(usize, 10),
541 try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("abcdefghij")),
542 );
543 try testing.expectEqual(
544 @as(usize, 10),
545 try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("äåéëþüúíóö")),
546 );
547 try testing.expectEqual(
548 @as(usize, 5),
549 try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("こんにちは")),
550 );
551}
552
553test "utf16 count codepoints" {
554 @setEvalBranchQuota(2000);
555 try testUtf16CountCodepoints();
556 try comptime testUtf16CountCodepoints();
557}
558
559test "utf8 encode" {
560 try comptime testUtf8Encode();
561 try testUtf8Encode();
562}
563fn testUtf8Encode() !void {
564 // A few taken from wikipedia a few taken elsewhere
565 var array: [4]u8 = undefined;
566 try testing.expect((try utf8Encode(try utf8Decode("€"), array[0..])) == 3);
567 try testing.expect(array[0] == 0b11100010);
568 try testing.expect(array[1] == 0b10000010);
569 try testing.expect(array[2] == 0b10101100);
570
571 try testing.expect((try utf8Encode(try utf8Decode("$"), array[0..])) == 1);
572 try testing.expect(array[0] == 0b00100100);
573
574 try testing.expect((try utf8Encode(try utf8Decode("¢"), array[0..])) == 2);
575 try testing.expect(array[0] == 0b11000010);
576 try testing.expect(array[1] == 0b10100010);
577
578 try testing.expect((try utf8Encode(try utf8Decode("𐍈"), array[0..])) == 4);
579 try testing.expect(array[0] == 0b11110000);
580 try testing.expect(array[1] == 0b10010000);
581 try testing.expect(array[2] == 0b10001101);
582 try testing.expect(array[3] == 0b10001000);
583}
584
585test "utf8 encode comptime" {
586 try testing.expectEqualSlices(u8, "€", &utf8EncodeComptime('€'));
587 try testing.expectEqualSlices(u8, "$", &utf8EncodeComptime('$'));
588 try testing.expectEqualSlices(u8, "¢", &utf8EncodeComptime('¢'));
589 try testing.expectEqualSlices(u8, "𐍈", &utf8EncodeComptime('𐍈'));
590}
591
592test "utf8 encode error" {
593 try comptime testUtf8EncodeError();
594 try testUtf8EncodeError();
595}
596fn testUtf8EncodeError() !void {
597 var array: [4]u8 = undefined;
598 try testErrorEncode(0xd800, array[0..], error.Utf8CannotEncodeSurrogateHalf);
599 try testErrorEncode(0xdfff, array[0..], error.Utf8CannotEncodeSurrogateHalf);
600 try testErrorEncode(0x110000, array[0..], error.CodepointTooLarge);
601 try testErrorEncode(0x1fffff, array[0..], error.CodepointTooLarge);
602}
603
604fn testErrorEncode(codePoint: u21, array: []u8, expectedErr: anyerror) !void {
605 try testing.expectError(expectedErr, utf8Encode(codePoint, array));
606}
607
608test "utf8 iterator on ascii" {
609 try comptime testUtf8IteratorOnAscii();
610 try testUtf8IteratorOnAscii();
611}
612fn testUtf8IteratorOnAscii() !void {
613 const s = Utf8View.initComptime("abc");
614
615 var it1 = s.iterator();
616 try testing.expect(mem.eql(u8, "a", it1.nextCodepointSlice().?));
617 try testing.expect(mem.eql(u8, "b", it1.nextCodepointSlice().?));
618 try testing.expect(mem.eql(u8, "c", it1.nextCodepointSlice().?));
619 try testing.expect(it1.nextCodepointSlice() == null);
620
621 var it2 = s.iterator();
622 try testing.expect(it2.nextCodepoint().? == 'a');
623 try testing.expect(it2.nextCodepoint().? == 'b');
624 try testing.expect(it2.nextCodepoint().? == 'c');
625 try testing.expect(it2.nextCodepoint() == null);
626}
627
628test "utf8 view bad" {
629 try comptime testUtf8ViewBad();
630 try testUtf8ViewBad();
631}
632fn testUtf8ViewBad() !void {
633 // Compile-time error.
634 // const s3 = Utf8View.initComptime("\xfe\xf2");
635 try testing.expectError(error.InvalidUtf8, Utf8View.init("hel\xadlo"));
636}
637
638test "utf8 view ok" {
639 try comptime testUtf8ViewOk();
640 try testUtf8ViewOk();
641}
642fn testUtf8ViewOk() !void {
643 const s = Utf8View.initComptime("東京市");
644
645 var it1 = s.iterator();
646 try testing.expect(mem.eql(u8, "東", it1.nextCodepointSlice().?));
647 try testing.expect(mem.eql(u8, "京", it1.nextCodepointSlice().?));
648 try testing.expect(mem.eql(u8, "市", it1.nextCodepointSlice().?));
649 try testing.expect(it1.nextCodepointSlice() == null);
650
651 var it2 = s.iterator();
652 try testing.expect(it2.nextCodepoint().? == 0x6771);
653 try testing.expect(it2.nextCodepoint().? == 0x4eac);
654 try testing.expect(it2.nextCodepoint().? == 0x5e02);
655 try testing.expect(it2.nextCodepoint() == null);
656}
657
658test "validate slice" {
659 try comptime testValidateSlice();
660 try testValidateSlice();
661
662 // We skip a variable (based on recommended vector size) chunks of
663 // ASCII characters. Let's make sure we're chunking correctly.
664 const str = @as([550]u8, @splat('a')) ++ "\xc0";
665 for (0..str.len - 3) |i| {
666 try testing.expect(!utf8ValidateSlice(str[i..]));
667 }
668}
669fn testValidateSlice() !void {
670 try testing.expect(utf8ValidateSlice("abc"));
671 try testing.expect(utf8ValidateSlice("abc\xdf\xbf"));
672 try testing.expect(utf8ValidateSlice(""));
673 try testing.expect(utf8ValidateSlice("a"));
674 try testing.expect(utf8ValidateSlice("abc"));
675 try testing.expect(utf8ValidateSlice("Ж"));
676 try testing.expect(utf8ValidateSlice("ЖЖ"));
677 try testing.expect(utf8ValidateSlice("брэд-ЛГТМ"));
678 try testing.expect(utf8ValidateSlice("☺☻☹"));
679 try testing.expect(utf8ValidateSlice("a\u{fffdb}"));
680 try testing.expect(utf8ValidateSlice("\xf4\x8f\xbf\xbf"));
681 try testing.expect(utf8ValidateSlice("abc\xdf\xbf"));
682
683 try testing.expect(!utf8ValidateSlice("abc\xc0"));
684 try testing.expect(!utf8ValidateSlice("abc\xc0abc"));
685 try testing.expect(!utf8ValidateSlice("aa\xe2"));
686 try testing.expect(!utf8ValidateSlice("\x42\xfa"));
687 try testing.expect(!utf8ValidateSlice("\x42\xfa\x43"));
688 try testing.expect(!utf8ValidateSlice("abc\xc0"));
689 try testing.expect(!utf8ValidateSlice("abc\xc0abc"));
690 try testing.expect(!utf8ValidateSlice("\xf4\x90\x80\x80"));
691 try testing.expect(!utf8ValidateSlice("\xf7\xbf\xbf\xbf"));
692 try testing.expect(!utf8ValidateSlice("\xfb\xbf\xbf\xbf\xbf"));
693 try testing.expect(!utf8ValidateSlice("\xc0\x80"));
694 try testing.expect(!utf8ValidateSlice("\xed\xa0\x80"));
695 try testing.expect(!utf8ValidateSlice("\xed\xbf\xbf"));
696}
697
698test "valid utf8" {
699 try comptime testValidUtf8();
700 try testValidUtf8();
701}
702fn testValidUtf8() !void {
703 try testValid("\x00", 0x0);
704 try testValid("\x20", 0x20);
705 try testValid("\x7f", 0x7f);
706 try testValid("\xc2\x80", 0x80);
707 try testValid("\xdf\xbf", 0x7ff);
708 try testValid("\xe0\xa0\x80", 0x800);
709 try testValid("\xe1\x80\x80", 0x1000);
710 try testValid("\xef\xbf\xbf", 0xffff);
711 try testValid("\xf0\x90\x80\x80", 0x10000);
712 try testValid("\xf1\x80\x80\x80", 0x40000);
713 try testValid("\xf3\xbf\xbf\xbf", 0xfffff);
714 try testValid("\xf4\x8f\xbf\xbf", 0x10ffff);
715}
716
717test "invalid utf8 continuation bytes" {
718 try comptime testInvalidUtf8ContinuationBytes();
719 try testInvalidUtf8ContinuationBytes();
720}
721fn testInvalidUtf8ContinuationBytes() !void {
722 // unexpected continuation
723 try testError("\x80", error.Utf8InvalidStartByte);
724 try testError("\xbf", error.Utf8InvalidStartByte);
725 // too many leading 1's
726 try testError("\xf8", error.Utf8InvalidStartByte);
727 try testError("\xff", error.Utf8InvalidStartByte);
728 // expected continuation for 2 byte sequences
729 try testError("\xc2", error.UnexpectedEof);
730 try testError("\xc2\x00", error.Utf8ExpectedContinuation);
731 try testError("\xc2\xc0", error.Utf8ExpectedContinuation);
732 // expected continuation for 3 byte sequences
733 try testError("\xe0", error.UnexpectedEof);
734 try testError("\xe0\x00", error.UnexpectedEof);
735 try testError("\xe0\xc0", error.UnexpectedEof);
736 try testError("\xe0\xa0", error.UnexpectedEof);
737 try testError("\xe0\xa0\x00", error.Utf8ExpectedContinuation);
738 try testError("\xe0\xa0\xc0", error.Utf8ExpectedContinuation);
739 // expected continuation for 4 byte sequences
740 try testError("\xf0", error.UnexpectedEof);
741 try testError("\xf0\x00", error.UnexpectedEof);
742 try testError("\xf0\xc0", error.UnexpectedEof);
743 try testError("\xf0\x90\x00", error.UnexpectedEof);
744 try testError("\xf0\x90\xc0", error.UnexpectedEof);
745 try testError("\xf0\x90\x80\x00", error.Utf8ExpectedContinuation);
746 try testError("\xf0\x90\x80\xc0", error.Utf8ExpectedContinuation);
747}
748
749test "overlong utf8 codepoint" {
750 try comptime testOverlongUtf8Codepoint();
751 try testOverlongUtf8Codepoint();
752}
753fn testOverlongUtf8Codepoint() !void {
754 try testError("\xc0\x80", error.Utf8OverlongEncoding);
755 try testError("\xc1\xbf", error.Utf8OverlongEncoding);
756 try testError("\xe0\x80\x80", error.Utf8OverlongEncoding);
757 try testError("\xe0\x9f\xbf", error.Utf8OverlongEncoding);
758 try testError("\xf0\x80\x80\x80", error.Utf8OverlongEncoding);
759 try testError("\xf0\x8f\xbf\xbf", error.Utf8OverlongEncoding);
760}
761
762test "misc invalid utf8" {
763 try comptime testMiscInvalidUtf8();
764 try testMiscInvalidUtf8();
765}
766fn testMiscInvalidUtf8() !void {
767 // codepoint out of bounds
768 try testError("\xf4\x90\x80\x80", error.Utf8CodepointTooLarge);
769 try testError("\xf7\xbf\xbf\xbf", error.Utf8CodepointTooLarge);
770 // surrogate halves
771 try testValid("\xed\x9f\xbf", 0xd7ff);
772 try testError("\xed\xa0\x80", error.Utf8EncodesSurrogateHalf);
773 try testError("\xed\xbf\xbf", error.Utf8EncodesSurrogateHalf);
774 try testValid("\xee\x80\x80", 0xe000);
775}
776
777test "utf8 iterator peeking" {
778 try comptime testUtf8Peeking();
779 try testUtf8Peeking();
780
781 comptime try testUtf8PeekCodepoint();
782 try testUtf8PeekCodepoint();
783}
784
785fn testUtf8Peeking() !void {
786 const s = Utf8View.initComptime("noël");
787 var it = s.iterator();
788
789 try testing.expect(mem.eql(u8, "n", it.nextCodepointSlice().?));
790
791 try testing.expect(mem.eql(u8, "o", it.peek(1)));
792 try testing.expect(mem.eql(u8, "oë", it.peek(2)));
793 try testing.expect(mem.eql(u8, "oël", it.peek(3)));
794 try testing.expect(mem.eql(u8, "oël", it.peek(4)));
795 try testing.expect(mem.eql(u8, "oël", it.peek(10)));
796
797 try testing.expect(mem.eql(u8, "o", it.nextCodepointSlice().?));
798 try testing.expect(mem.eql(u8, "ë", it.nextCodepointSlice().?));
799 try testing.expect(mem.eql(u8, "l", it.nextCodepointSlice().?));
800 try testing.expect(it.nextCodepointSlice() == null);
801
802 try testing.expect(mem.eql(u8, &[_]u8{}, it.peek(1)));
803}
804
805fn testUtf8PeekCodepoint() !void {
806 const s = Utf8View.initComptime("東京市");
807 var it = s.iterator();
808
809 try testing.expect(it.peekCodepoint().? == 0x6771);
810 try testing.expect(it.peekCodepoint().? == 0x6771);
811 _ = it.nextCodepoint();
812 try testing.expect(it.peekCodepoint().? == 0x4eac);
813 _ = it.nextCodepoint();
814 try testing.expect(it.peekCodepoint().? == 0x5e02);
815 _ = it.nextCodepoint();
816 try testing.expect(it.peekCodepoint() == null);
817}
818
819fn testError(bytes: []const u8, expected_err: anyerror) !void {
820 try testing.expectError(expected_err, testDecode(bytes));
821}
822
823fn testValid(bytes: []const u8, expected_codepoint: u21) !void {
824 try testing.expect((testDecode(bytes) catch unreachable) == expected_codepoint);
825}
826
827fn testDecode(bytes: []const u8) !u21 {
828 const length = try utf8ByteSequenceLength(bytes[0]);
829 if (bytes.len < length) return error.UnexpectedEof;
830 try testing.expect(bytes.len == length);
831 return utf8Decode(bytes);
832}
833
834/// Print the given `utf8` string, encoded as UTF-8 bytes.
835/// Ill-formed UTF-8 byte sequences are replaced by the replacement character (U+FFFD)
836/// according to "U+FFFD Substitution of Maximal Subparts" from Chapter 3 of
837/// the Unicode standard, and as specified by https://encoding.spec.whatwg.org/#utf-8-decoder
838fn formatUtf8(utf8: []const u8, writer: *std.Io.Writer) std.Io.Writer.Error!void {
839 var buf: [300]u8 = undefined; // just an arbitrary size
840 var u8len: usize = 0;
841
842 // This implementation is based on this specification:
843 // https://encoding.spec.whatwg.org/#utf-8-decoder
844 var codepoint: u21 = 0;
845 var cont_bytes_seen: u3 = 0;
846 var cont_bytes_needed: u3 = 0;
847 var lower_boundary: u8 = 0x80;
848 var upper_boundary: u8 = 0xBF;
849
850 var i: usize = 0;
851 while (i < utf8.len) {
852 const byte = utf8[i];
853 if (cont_bytes_needed == 0) {
854 switch (byte) {
855 0x00...0x7F => {
856 buf[u8len] = byte;
857 u8len += 1;
858 },
859 0xC2...0xDF => {
860 cont_bytes_needed = 1;
861 codepoint = byte & 0b00011111;
862 },
863 0xE0...0xEF => {
864 if (byte == 0xE0) lower_boundary = 0xA0;
865 if (byte == 0xED) upper_boundary = 0x9F;
866 cont_bytes_needed = 2;
867 codepoint = byte & 0b00001111;
868 },
869 0xF0...0xF4 => {
870 if (byte == 0xF0) lower_boundary = 0x90;
871 if (byte == 0xF4) upper_boundary = 0x8F;
872 cont_bytes_needed = 3;
873 codepoint = byte & 0b00000111;
874 },
875 else => {
876 u8len += utf8Encode(replacement_character, buf[u8len..]) catch unreachable;
877 },
878 }
879 // consume the byte
880 i += 1;
881 } else if (byte < lower_boundary or byte > upper_boundary) {
882 codepoint = 0;
883 cont_bytes_needed = 0;
884 cont_bytes_seen = 0;
885 lower_boundary = 0x80;
886 upper_boundary = 0xBF;
887 u8len += utf8Encode(replacement_character, buf[u8len..]) catch unreachable;
888 // do not consume the current byte, it should now be treated as a possible start byte
889 } else {
890 lower_boundary = 0x80;
891 upper_boundary = 0xBF;
892 codepoint <<= 6;
893 codepoint |= byte & 0b00111111;
894 cont_bytes_seen += 1;
895 // consume the byte
896 i += 1;
897
898 if (cont_bytes_seen == cont_bytes_needed) {
899 const codepoint_len = cont_bytes_seen + 1;
900 const codepoint_start_i = i - codepoint_len;
901 @memcpy(buf[u8len..][0..codepoint_len], utf8[codepoint_start_i..][0..codepoint_len]);
902 u8len += codepoint_len;
903
904 codepoint = 0;
905 cont_bytes_needed = 0;
906 cont_bytes_seen = 0;
907 }
908 }
909 // make sure there's always enough room for another maximum length UTF-8 codepoint
910 if (u8len + 4 > buf.len) {
911 try writer.writeAll(buf[0..u8len]);
912 u8len = 0;
913 }
914 }
915 if (cont_bytes_needed != 0) {
916 // we know there's enough room because we always flush
917 // if there's less than 4 bytes remaining in the buffer.
918 u8len += utf8Encode(replacement_character, buf[u8len..]) catch unreachable;
919 }
920 try writer.writeAll(buf[0..u8len]);
921}
922
923/// Return a Formatter for a (potentially ill-formed) UTF-8 string.
924/// Ill-formed UTF-8 byte sequences are replaced by the replacement character (U+FFFD)
925/// according to "U+FFFD Substitution of Maximal Subparts" from Chapter 3 of
926/// the Unicode standard, and as specified by https://encoding.spec.whatwg.org/#utf-8-decoder
927pub fn fmtUtf8(utf8: []const u8) std.fmt.Alt([]const u8, formatUtf8) {
928 return .{ .data = utf8 };
929}
930
931test fmtUtf8 {
932 const expectFmt = testing.expectFmt;
933 try expectFmt("", "{f}", .{fmtUtf8("")});
934 try expectFmt("foo", "{f}", .{fmtUtf8("foo")});
935 try expectFmt("𐐷", "{f}", .{fmtUtf8("𐐷")});
936
937 // Table 3-8. U+FFFD for Non-Shortest Form Sequences
938 try expectFmt("��������A", "{f}", .{fmtUtf8("\xC0\xAF\xE0\x80\xBF\xF0\x81\x82A")});
939
940 // Table 3-9. U+FFFD for Ill-Formed Sequences for Surrogates
941 try expectFmt("��������A", "{f}", .{fmtUtf8("\xED\xA0\x80\xED\xBF\xBF\xED\xAFA")});
942
943 // Table 3-10. U+FFFD for Other Ill-Formed Sequences
944 try expectFmt("�����A��B", "{f}", .{fmtUtf8("\xF4\x91\x92\x93\xFFA\x80\xBFB")});
945
946 // Table 3-11. U+FFFD for Truncated Sequences
947 try expectFmt("����A", "{f}", .{fmtUtf8("\xE1\x80\xE2\xF0\x91\x92\xF1\xBFA")});
948}
949
950fn utf16LeToUtf8ArrayListImpl(
951 result: *std.array_list.Managed(u8),
952 utf16le: []const u16,
953 comptime surrogates: Surrogates,
954) (switch (surrogates) {
955 .cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,
956 .can_encode_surrogate_half => Allocator.Error,
957})!void {
958 assert(result.unusedCapacitySlice().len >= utf16le.len);
959
960 var remaining = utf16le;
961 vectorized: {
962 const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
963 const Chunk = @Vector(chunk_len, u16);
964
965 // Fast path. Check for and encode ASCII characters at the start of the input.
966 while (remaining.len >= chunk_len) {
967 const chunk: Chunk = remaining[0..chunk_len].*;
968 const mask: Chunk = @splat(mem.nativeToLittle(u16, 0x7F));
969 if (@reduce(.Or, chunk | mask != mask)) {
970 // found a non ASCII code unit
971 break;
972 }
973 const ascii_chunk: @Vector(chunk_len, u8) = @truncate(mem.nativeToLittle(Chunk, chunk));
974 // We allocated enough space to encode every UTF-16 code unit
975 // as ASCII, so if the entire string is ASCII then we are
976 // guaranteed to have enough space allocated
977 result.addManyAsArrayAssumeCapacity(chunk_len).* = ascii_chunk;
978 remaining = remaining[chunk_len..];
979 }
980 }
981
982 switch (surrogates) {
983 .cannot_encode_surrogate_half => {
984 var it = Utf16LeIterator.init(remaining);
985 while (try it.nextCodepoint()) |codepoint| {
986 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
987 assert((utf8Encode(codepoint, try result.addManyAsSlice(utf8_len)) catch unreachable) == utf8_len);
988 }
989 },
990 .can_encode_surrogate_half => {
991 var it = Wtf16LeIterator.init(remaining);
992 while (it.nextCodepoint()) |codepoint| {
993 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
994 assert((wtf8Encode(codepoint, try result.addManyAsSlice(utf8_len)) catch unreachable) == utf8_len);
995 }
996 },
997 }
998}
999
1000pub const Utf16LeToUtf8AllocError = Allocator.Error || Utf16LeToUtf8Error;
1001
1002pub fn utf16LeToUtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {
1003 try result.ensureUnusedCapacity(utf16le.len);
1004 return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half);
1005}
1006
1007/// Caller owns returned memory.
1008pub fn utf16LeToUtf8Alloc(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![]u8 {
1009 // optimistically guess that it will all be ascii.
1010 var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len);
1011 errdefer result.deinit();
1012
1013 try utf16LeToUtf8ArrayListImpl(&result, utf16le, .cannot_encode_surrogate_half);
1014 return result.toOwnedSlice();
1015}
1016
1017/// Caller owns returned memory.
1018pub fn utf16LeToUtf8AllocZ(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![:0]u8 {
1019 // optimistically guess that it will all be ascii (and allocate space for the null terminator)
1020 var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len + 1);
1021 errdefer result.deinit();
1022
1023 try utf16LeToUtf8ArrayListImpl(&result, utf16le, .cannot_encode_surrogate_half);
1024 return result.toOwnedSliceSentinel(0);
1025}
1026
1027pub const Utf16LeToUtf8Error = Utf16LeIterator.NextCodepointError;
1028
1029/// Asserts that the output buffer is big enough.
1030/// Returns end byte index into utf8.
1031fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surrogates) (switch (surrogates) {
1032 .cannot_encode_surrogate_half => Utf16LeToUtf8Error,
1033 .can_encode_surrogate_half => error{},
1034})!usize {
1035 var dest_index: usize = 0;
1036
1037 var remaining = utf16le;
1038 vectorized: {
1039 const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
1040 const Chunk = @Vector(chunk_len, u16);
1041
1042 // Fast path. Check for and encode ASCII characters at the start of the input.
1043 while (remaining.len >= chunk_len) {
1044 const chunk: Chunk = remaining[0..chunk_len].*;
1045 const mask: Chunk = @splat(mem.nativeToLittle(u16, 0x7F));
1046 if (@reduce(.Or, chunk | mask != mask)) {
1047 // found a non ASCII code unit
1048 break;
1049 }
1050 const ascii_chunk: @Vector(chunk_len, u8) = @truncate(mem.nativeToLittle(Chunk, chunk));
1051 utf8[dest_index..][0..chunk_len].* = ascii_chunk;
1052 dest_index += chunk_len;
1053 remaining = remaining[chunk_len..];
1054 }
1055 }
1056
1057 switch (surrogates) {
1058 .cannot_encode_surrogate_half => {
1059 var it = Utf16LeIterator.init(remaining);
1060 while (try it.nextCodepoint()) |codepoint| {
1061 dest_index += utf8Encode(codepoint, utf8[dest_index..]) catch |err| switch (err) {
1062 // The maximum possible codepoint encoded by UTF-16 is U+10FFFF,
1063 // which is within the valid codepoint range.
1064 error.CodepointTooLarge => unreachable,
1065 // We know the codepoint was valid in UTF-16, meaning it is not
1066 // an unpaired surrogate codepoint.
1067 error.Utf8CannotEncodeSurrogateHalf => unreachable,
1068 };
1069 }
1070 },
1071 .can_encode_surrogate_half => {
1072 var it = Wtf16LeIterator.init(remaining);
1073 while (it.nextCodepoint()) |codepoint| {
1074 dest_index += wtf8Encode(codepoint, utf8[dest_index..]) catch |err| switch (err) {
1075 // The maximum possible codepoint encoded by UTF-16 is U+10FFFF,
1076 // which is within the valid codepoint range.
1077 error.CodepointTooLarge => unreachable,
1078 };
1079 }
1080 },
1081 }
1082 return dest_index;
1083}
1084
1085pub fn utf16LeToUtf8(utf8: []u8, utf16le: []const u16) Utf16LeToUtf8Error!usize {
1086 return utf16LeToUtf8Impl(utf8, utf16le, .cannot_encode_surrogate_half);
1087}
1088
1089test utf16LeToUtf8 {
1090 var utf16le: [2]u16 = undefined;
1091 const utf16le_as_bytes = mem.sliceAsBytes(utf16le[0..]);
1092
1093 {
1094 mem.writeInt(u16, utf16le_as_bytes[0..2], 'A', .little);
1095 mem.writeInt(u16, utf16le_as_bytes[2..4], 'a', .little);
1096 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
1097 defer testing.allocator.free(utf8);
1098 try testing.expect(mem.eql(u8, utf8, "Aa"));
1099 }
1100
1101 {
1102 mem.writeInt(u16, utf16le_as_bytes[0..2], 0x80, .little);
1103 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xffff, .little);
1104 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
1105 defer testing.allocator.free(utf8);
1106 try testing.expect(mem.eql(u8, utf8, "\xc2\x80" ++ "\xef\xbf\xbf"));
1107 }
1108
1109 {
1110 // the values just outside the surrogate half range
1111 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xd7ff, .little);
1112 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xe000, .little);
1113 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
1114 defer testing.allocator.free(utf8);
1115 try testing.expect(mem.eql(u8, utf8, "\xed\x9f\xbf" ++ "\xee\x80\x80"));
1116 }
1117
1118 {
1119 // smallest surrogate pair
1120 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xd800, .little);
1121 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdc00, .little);
1122 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
1123 defer testing.allocator.free(utf8);
1124 try testing.expect(mem.eql(u8, utf8, "\xf0\x90\x80\x80"));
1125 }
1126
1127 {
1128 // largest surrogate pair
1129 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdbff, .little);
1130 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdfff, .little);
1131 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
1132 defer testing.allocator.free(utf8);
1133 try testing.expect(mem.eql(u8, utf8, "\xf4\x8f\xbf\xbf"));
1134 }
1135
1136 {
1137 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdbff, .little);
1138 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdc00, .little);
1139 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
1140 defer testing.allocator.free(utf8);
1141 try testing.expect(mem.eql(u8, utf8, "\xf4\x8f\xb0\x80"));
1142 }
1143
1144 {
1145 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdcdc, .little);
1146 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdcdc, .little);
1147 const result = utf16LeToUtf8Alloc(testing.allocator, &utf16le);
1148 try testing.expectError(error.UnexpectedSecondSurrogateHalf, result);
1149 }
1150}
1151
1152fn utf8ToUtf16LeArrayListImpl(result: *std.array_list.Managed(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {
1153 assert(result.unusedCapacitySlice().len >= utf8.len);
1154
1155 var remaining = utf8;
1156 vectorized: {
1157 const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
1158 const Chunk = @Vector(chunk_len, u8);
1159
1160 // Fast path. Check for and encode ASCII characters at the start of the input.
1161 while (remaining.len >= chunk_len) {
1162 const chunk: Chunk = remaining[0..chunk_len].*;
1163 const mask: Chunk = @splat(0x80);
1164 if (@reduce(.Or, chunk & mask == mask)) {
1165 // found a non ASCII code unit
1166 break;
1167 }
1168 const utf16_chunk = mem.nativeToLittle(@Vector(chunk_len, u16), chunk);
1169 result.addManyAsArrayAssumeCapacity(chunk_len).* = utf16_chunk;
1170 remaining = remaining[chunk_len..];
1171 }
1172 }
1173
1174 const view = switch (surrogates) {
1175 .cannot_encode_surrogate_half => try Utf8View.init(remaining),
1176 .can_encode_surrogate_half => try Wtf8View.init(remaining),
1177 };
1178 var it = view.iterator();
1179 while (it.nextCodepoint()) |codepoint| {
1180 if (codepoint < 0x10000) {
1181 try result.append(mem.nativeToLittle(u16, @intCast(codepoint)));
1182 } else {
1183 const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
1184 const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
1185 try result.appendSlice(&.{ mem.nativeToLittle(u16, high), mem.nativeToLittle(u16, low) });
1186 }
1187 }
1188}
1189
1190pub fn utf8ToUtf16LeArrayList(result: *std.array_list.Managed(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void {
1191 try result.ensureUnusedCapacity(utf8.len);
1192 return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half);
1193}
1194
1195pub fn utf8ToUtf16LeAlloc(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 {
1196 // optimistically guess that it will not require surrogate pairs
1197 var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len);
1198 errdefer result.deinit();
1199
1200 try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
1201 return result.toOwnedSlice();
1202}
1203
1204pub fn utf8ToUtf16LeAllocZ(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![:0]u16 {
1205 // optimistically guess that it will not require surrogate pairs
1206 var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len + 1);
1207 errdefer result.deinit();
1208
1209 try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
1210 return result.toOwnedSliceSentinel(0);
1211}
1212
1213/// Returns index of next character. If exact fit, returned index equals output slice length.
1214/// Assumes there is enough space for the output.
1215pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) error{InvalidUtf8}!usize {
1216 return utf8ToUtf16LeImpl(utf16le, utf8, .cannot_encode_surrogate_half);
1217}
1218
1219pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates: Surrogates) !usize {
1220 var dest_index: usize = 0;
1221
1222 var remaining = utf8;
1223 vectorized: {
1224 const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
1225 const Chunk = @Vector(chunk_len, u8);
1226
1227 // Fast path. Check for and encode ASCII characters at the start of the input.
1228 while (remaining.len >= chunk_len) {
1229 const chunk: Chunk = remaining[0..chunk_len].*;
1230 const mask: Chunk = @splat(0x80);
1231 if (@reduce(.Or, chunk & mask == mask)) {
1232 // found a non ASCII code unit
1233 break;
1234 }
1235 const utf16_chunk = mem.nativeToLittle(@Vector(chunk_len, u16), chunk);
1236 utf16le[dest_index..][0..chunk_len].* = utf16_chunk;
1237 dest_index += chunk_len;
1238 remaining = remaining[chunk_len..];
1239 }
1240 }
1241
1242 const view = switch (surrogates) {
1243 .cannot_encode_surrogate_half => try Utf8View.init(remaining),
1244 .can_encode_surrogate_half => try Wtf8View.init(remaining),
1245 };
1246 var it = view.iterator();
1247 while (it.nextCodepoint()) |codepoint| {
1248 if (codepoint < 0x10000) {
1249 utf16le[dest_index] = mem.nativeToLittle(u16, @intCast(codepoint));
1250 dest_index += 1;
1251 } else {
1252 const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
1253 const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
1254 utf16le[dest_index..][0..2].* = .{ mem.nativeToLittle(u16, high), mem.nativeToLittle(u16, low) };
1255 dest_index += 2;
1256 }
1257 }
1258 return dest_index;
1259}
1260
1261test utf8ToUtf16Le {
1262 var utf16le: [128]u16 = undefined;
1263 {
1264 const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");
1265 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16le[0..length]));
1266 }
1267 {
1268 const length = try utf8ToUtf16Le(utf16le[0..], "\u{10FFFF}");
1269 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16le[0..length]));
1270 }
1271 {
1272 const result = utf8ToUtf16Le(utf16le[0..], "\xf4\x90\x80\x80");
1273 try testing.expectError(error.InvalidUtf8, result);
1274 }
1275 {
1276 const length = try utf8ToUtf16Le(utf16le[0..], "This string has been designed to test the vectorized implementat" ++
1277 "ion by beginning with one hundred twenty-seven ASCII characters¡");
1278 try testing.expectEqualSlices(u8, &.{
1279 'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ', 0,
1280 'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o', 0,
1281 ' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r', 0,
1282 'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't', 0,
1283 'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g', 0,
1284 ' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e', 0,
1285 'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A', 0,
1286 'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0,
1287 '¡',
1288 0,
1289 }, mem.sliceAsBytes(utf16le[0..length]));
1290 }
1291}
1292
1293test utf8ToUtf16LeArrayList {
1294 {
1295 var list = std.array_list.Managed(u16).init(testing.allocator);
1296 defer list.deinit();
1297 try utf8ToUtf16LeArrayList(&list, "𐐷");
1298 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(list.items));
1299 }
1300 {
1301 var list = std.array_list.Managed(u16).init(testing.allocator);
1302 defer list.deinit();
1303 try utf8ToUtf16LeArrayList(&list, "\u{10FFFF}");
1304 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(list.items));
1305 }
1306 {
1307 var list = std.array_list.Managed(u16).init(testing.allocator);
1308 defer list.deinit();
1309 const result = utf8ToUtf16LeArrayList(&list, "\xf4\x90\x80\x80");
1310 try testing.expectError(error.InvalidUtf8, result);
1311 }
1312}
1313
1314test utf8ToUtf16LeAlloc {
1315 {
1316 const utf16 = try utf8ToUtf16LeAlloc(testing.allocator, "𐐷");
1317 defer testing.allocator.free(utf16);
1318 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16[0..]));
1319 }
1320 {
1321 const utf16 = try utf8ToUtf16LeAlloc(testing.allocator, "\u{10FFFF}");
1322 defer testing.allocator.free(utf16);
1323 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16[0..]));
1324 }
1325 {
1326 const result = utf8ToUtf16LeAlloc(testing.allocator, "\xf4\x90\x80\x80");
1327 try testing.expectError(error.InvalidUtf8, result);
1328 }
1329}
1330
1331test utf8ToUtf16LeAllocZ {
1332 {
1333 const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "𐐷");
1334 defer testing.allocator.free(utf16);
1335 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16));
1336 try testing.expect(utf16[2] == 0);
1337 }
1338 {
1339 const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "\u{10FFFF}");
1340 defer testing.allocator.free(utf16);
1341 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16));
1342 try testing.expect(utf16[2] == 0);
1343 }
1344 {
1345 const result = utf8ToUtf16LeAllocZ(testing.allocator, "\xf4\x90\x80\x80");
1346 try testing.expectError(error.InvalidUtf8, result);
1347 }
1348 {
1349 const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "This string has been designed to test the vectorized implementat" ++
1350 "ion by beginning with one hundred twenty-seven ASCII characters¡");
1351 defer testing.allocator.free(utf16);
1352 try testing.expectEqualSlices(u8, &.{
1353 'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ', 0,
1354 'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o', 0,
1355 ' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r', 0,
1356 'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't', 0,
1357 'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g', 0,
1358 ' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e', 0,
1359 'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A', 0,
1360 'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0,
1361 '¡',
1362 0,
1363 }, mem.sliceAsBytes(utf16));
1364 }
1365}
1366
1367test "ArrayList functions on a re-used list" {
1368 // utf8ToUtf16LeArrayList
1369 {
1370 var list = std.array_list.Managed(u16).init(testing.allocator);
1371 defer list.deinit();
1372
1373 const init_slice = utf8ToUtf16LeStringLiteral("abcdefg");
1374 try list.ensureTotalCapacityPrecise(init_slice.len);
1375 list.appendSliceAssumeCapacity(init_slice);
1376
1377 try utf8ToUtf16LeArrayList(&list, "hijklmnopqrstuvwyxz");
1378
1379 try testing.expectEqualSlices(u16, utf8ToUtf16LeStringLiteral("abcdefghijklmnopqrstuvwyxz"), list.items);
1380 }
1381
1382 // utf16LeToUtf8ArrayList
1383 {
1384 var list = std.array_list.Managed(u8).init(testing.allocator);
1385 defer list.deinit();
1386
1387 const init_slice = "abcdefg";
1388 try list.ensureTotalCapacityPrecise(init_slice.len);
1389 list.appendSliceAssumeCapacity(init_slice);
1390
1391 try utf16LeToUtf8ArrayList(&list, utf8ToUtf16LeStringLiteral("hijklmnopqrstuvwyxz"));
1392
1393 try testing.expectEqualStrings("abcdefghijklmnopqrstuvwyxz", list.items);
1394 }
1395
1396 // wtf8ToWtf16LeArrayList
1397 {
1398 var list = std.array_list.Managed(u16).init(testing.allocator);
1399 defer list.deinit();
1400
1401 const init_slice = utf8ToUtf16LeStringLiteral("abcdefg");
1402 try list.ensureTotalCapacityPrecise(init_slice.len);
1403 list.appendSliceAssumeCapacity(init_slice);
1404
1405 try wtf8ToWtf16LeArrayList(&list, "hijklmnopqrstuvwyxz");
1406
1407 try testing.expectEqualSlices(u16, utf8ToUtf16LeStringLiteral("abcdefghijklmnopqrstuvwyxz"), list.items);
1408 }
1409
1410 // wtf16LeToWtf8ArrayList
1411 {
1412 var list = std.array_list.Managed(u8).init(testing.allocator);
1413 defer list.deinit();
1414
1415 const init_slice = "abcdefg";
1416 try list.ensureTotalCapacityPrecise(init_slice.len);
1417 list.appendSliceAssumeCapacity(init_slice);
1418
1419 try wtf16LeToWtf8ArrayList(&list, utf8ToUtf16LeStringLiteral("hijklmnopqrstuvwyxz"));
1420
1421 try testing.expectEqualStrings("abcdefghijklmnopqrstuvwyxz", list.items);
1422 }
1423}
1424
1425fn utf8ToUtf16LeStringLiteralImpl(comptime utf8: []const u8, comptime surrogates: Surrogates) *const [calcUtf16LeLenImpl(utf8, surrogates) catch |err| @compileError(err):0]u16 {
1426 return comptime blk: {
1427 const len: usize = calcUtf16LeLenImpl(utf8, surrogates) catch unreachable;
1428 var utf16le: [len:0]u16 = @splat(0);
1429 const utf16le_len = utf8ToUtf16LeImpl(&utf16le, utf8[0..], surrogates) catch |err| @compileError(err);
1430 assert(len == utf16le_len);
1431 const final = utf16le;
1432 break :blk &final;
1433 };
1434}
1435
1436/// Converts a UTF-8 string literal into a UTF-16LE string literal.
1437pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch |err| @compileError(err):0]u16 {
1438 return utf8ToUtf16LeStringLiteralImpl(utf8, .cannot_encode_surrogate_half);
1439}
1440
1441/// Converts a WTF-8 string literal into a WTF-16LE string literal.
1442pub fn wtf8ToWtf16LeStringLiteral(comptime wtf8: []const u8) *const [calcWtf16LeLen(wtf8) catch |err| @compileError(err):0]u16 {
1443 return utf8ToUtf16LeStringLiteralImpl(wtf8, .can_encode_surrogate_half);
1444}
1445
1446pub fn calcUtf16LeLenImpl(utf8: []const u8, comptime surrogates: Surrogates) !usize {
1447 const utf8DecodeImpl = switch (surrogates) {
1448 .cannot_encode_surrogate_half => utf8Decode,
1449 .can_encode_surrogate_half => wtf8Decode,
1450 };
1451 var src_i: usize = 0;
1452 var dest_len: usize = 0;
1453 while (src_i < utf8.len) {
1454 const n = try utf8ByteSequenceLength(utf8[src_i]);
1455 const next_src_i = src_i + n;
1456 const codepoint = try utf8DecodeImpl(utf8[src_i..next_src_i]);
1457 if (codepoint < 0x10000) {
1458 dest_len += 1;
1459 } else {
1460 dest_len += 2;
1461 }
1462 src_i = next_src_i;
1463 }
1464 return dest_len;
1465}
1466
1467const CalcUtf16LeLenError = Utf8DecodeError || error{Utf8InvalidStartByte};
1468
1469/// Returns length in UTF-16LE of UTF-8 slice as length of []u16.
1470/// Length in []u8 is 2*len16.
1471pub fn calcUtf16LeLen(utf8: []const u8) CalcUtf16LeLenError!usize {
1472 return calcUtf16LeLenImpl(utf8, .cannot_encode_surrogate_half);
1473}
1474
1475const CalcWtf16LeLenError = Wtf8DecodeError || error{Utf8InvalidStartByte};
1476
1477/// Returns length in WTF-16LE of WTF-8 slice as length of []u16.
1478/// Length in []u8 is 2*len16.
1479pub fn calcWtf16LeLen(wtf8: []const u8) CalcWtf16LeLenError!usize {
1480 return calcUtf16LeLenImpl(wtf8, .can_encode_surrogate_half);
1481}
1482
1483fn testCalcUtf16LeLenImpl(calcUtf16LeLenImpl_: anytype) !void {
1484 try testing.expectEqual(@as(usize, 1), try calcUtf16LeLenImpl_("a"));
1485 try testing.expectEqual(@as(usize, 10), try calcUtf16LeLenImpl_("abcdefghij"));
1486 try testing.expectEqual(@as(usize, 10), try calcUtf16LeLenImpl_("äåéëþüúíóö"));
1487 try testing.expectEqual(@as(usize, 5), try calcUtf16LeLenImpl_("こんにちは"));
1488}
1489
1490test calcUtf16LeLen {
1491 try testCalcUtf16LeLenImpl(calcUtf16LeLen);
1492 try comptime testCalcUtf16LeLenImpl(calcUtf16LeLen);
1493}
1494
1495test calcWtf16LeLen {
1496 try testCalcUtf16LeLenImpl(calcWtf16LeLen);
1497 try comptime testCalcUtf16LeLenImpl(calcWtf16LeLen);
1498}
1499
1500/// Print the given `utf16le` string, encoded as UTF-8 bytes.
1501/// Unpaired surrogates are replaced by the replacement character (U+FFFD).
1502fn formatUtf16Le(utf16le: []const u16, writer: *std.Io.Writer) std.Io.Writer.Error!void {
1503 var buf: [300]u8 = undefined; // just an arbitrary size
1504 var it = Utf16LeIterator.init(utf16le);
1505 var u8len: usize = 0;
1506 while (it.nextCodepoint() catch replacement_character) |codepoint| {
1507 u8len += utf8Encode(codepoint, buf[u8len..]) catch
1508 utf8Encode(replacement_character, buf[u8len..]) catch unreachable;
1509 // make sure there's always enough room for another maximum length UTF-8 codepoint
1510 if (u8len + 4 > buf.len) {
1511 try writer.writeAll(buf[0..u8len]);
1512 u8len = 0;
1513 }
1514 }
1515 try writer.writeAll(buf[0..u8len]);
1516}
1517
1518/// Return a Formatter for a (potentially ill-formed) UTF-16 LE string,
1519/// which will be converted to UTF-8 during formatting.
1520/// Unpaired surrogates are replaced by the replacement character (U+FFFD).
1521pub fn fmtUtf16Le(utf16le: []const u16) std.fmt.Alt([]const u16, formatUtf16Le) {
1522 return .{ .data = utf16le };
1523}
1524
1525test fmtUtf16Le {
1526 const expectFmt = testing.expectFmt;
1527 try expectFmt("", "{f}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral(""))});
1528 try expectFmt("", "{f}", .{fmtUtf16Le(wtf8ToWtf16LeStringLiteral(""))});
1529 try expectFmt("foo", "{f}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("foo"))});
1530 try expectFmt("foo", "{f}", .{fmtUtf16Le(wtf8ToWtf16LeStringLiteral("foo"))});
1531 try expectFmt("𐐷", "{f}", .{fmtUtf16Le(wtf8ToWtf16LeStringLiteral("𐐷"))});
1532 try expectFmt("퟿", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xd7", native_endian)})});
1533 try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xd8", native_endian)})});
1534 try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdb", native_endian)})});
1535 try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xdc", native_endian)})});
1536 try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdf", native_endian)})});
1537 try expectFmt("", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xe0", native_endian)})});
1538}
1539
1540fn testUtf8ToUtf16LeStringLiteral(utf8ToUtf16LeStringLiteral_: anytype) !void {
1541 {
1542 const bytes = [_:0]u16{
1543 mem.nativeToLittle(u16, 0x41),
1544 };
1545 const utf16 = utf8ToUtf16LeStringLiteral_("A");
1546 try testing.expectEqualSlices(u16, &bytes, utf16);
1547 try testing.expect(utf16[1] == 0);
1548 }
1549 {
1550 const bytes = [_:0]u16{
1551 mem.nativeToLittle(u16, 0xD801),
1552 mem.nativeToLittle(u16, 0xDC37),
1553 };
1554 const utf16 = utf8ToUtf16LeStringLiteral_("𐐷");
1555 try testing.expectEqualSlices(u16, &bytes, utf16);
1556 try testing.expect(utf16[2] == 0);
1557 }
1558 {
1559 const bytes = [_:0]u16{
1560 mem.nativeToLittle(u16, 0x02FF),
1561 };
1562 const utf16 = utf8ToUtf16LeStringLiteral_("\u{02FF}");
1563 try testing.expectEqualSlices(u16, &bytes, utf16);
1564 try testing.expect(utf16[1] == 0);
1565 }
1566 {
1567 const bytes = [_:0]u16{
1568 mem.nativeToLittle(u16, 0x7FF),
1569 };
1570 const utf16 = utf8ToUtf16LeStringLiteral_("\u{7FF}");
1571 try testing.expectEqualSlices(u16, &bytes, utf16);
1572 try testing.expect(utf16[1] == 0);
1573 }
1574 {
1575 const bytes = [_:0]u16{
1576 mem.nativeToLittle(u16, 0x801),
1577 };
1578 const utf16 = utf8ToUtf16LeStringLiteral_("\u{801}");
1579 try testing.expectEqualSlices(u16, &bytes, utf16);
1580 try testing.expect(utf16[1] == 0);
1581 }
1582 {
1583 const bytes = [_:0]u16{
1584 mem.nativeToLittle(u16, 0xDBFF),
1585 mem.nativeToLittle(u16, 0xDFFF),
1586 };
1587 const utf16 = utf8ToUtf16LeStringLiteral_("\u{10FFFF}");
1588 try testing.expectEqualSlices(u16, &bytes, utf16);
1589 try testing.expect(utf16[2] == 0);
1590 }
1591}
1592
1593test utf8ToUtf16LeStringLiteral {
1594 try testUtf8ToUtf16LeStringLiteral(utf8ToUtf16LeStringLiteral);
1595}
1596
1597test wtf8ToWtf16LeStringLiteral {
1598 try testUtf8ToUtf16LeStringLiteral(wtf8ToWtf16LeStringLiteral);
1599}
1600
1601fn testUtf8CountCodepoints() !void {
1602 try testing.expectEqual(@as(usize, 10), try utf8CountCodepoints("abcdefghij"));
1603 try testing.expectEqual(@as(usize, 10), try utf8CountCodepoints("äåéëþüúíóö"));
1604 try testing.expectEqual(@as(usize, 5), try utf8CountCodepoints("こんにちは"));
1605 // testing.expectError(error.Utf8EncodesSurrogateHalf, utf8CountCodepoints("\xED\xA0\x80"));
1606}
1607
1608test "utf8 count codepoints" {
1609 try testUtf8CountCodepoints();
1610 try comptime testUtf8CountCodepoints();
1611}
1612
1613fn testUtf8ValidCodepoint() !void {
1614 try testing.expect(utf8ValidCodepoint('e'));
1615 try testing.expect(utf8ValidCodepoint('ë'));
1616 try testing.expect(utf8ValidCodepoint('は'));
1617 try testing.expect(utf8ValidCodepoint(0xe000));
1618 try testing.expect(utf8ValidCodepoint(0x10ffff));
1619 try testing.expect(!utf8ValidCodepoint(0xd800));
1620 try testing.expect(!utf8ValidCodepoint(0xdfff));
1621 try testing.expect(!utf8ValidCodepoint(0x110000));
1622}
1623
1624test "utf8 valid codepoint" {
1625 try testUtf8ValidCodepoint();
1626 try comptime testUtf8ValidCodepoint();
1627}
1628
1629/// Returns true if the codepoint is a surrogate (U+DC00 to U+DFFF)
1630pub fn isSurrogateCodepoint(c: u21) bool {
1631 return switch (c) {
1632 0xD800...0xDFFF => true,
1633 else => false,
1634 };
1635}
1636
1637/// Encodes the given codepoint into a WTF-8 byte sequence.
1638/// c: the codepoint.
1639/// out: the out buffer to write to. Must have a len >= utf8CodepointSequenceLength(c).
1640/// Errors: if c cannot be encoded in WTF-8.
1641/// Returns: the number of bytes written to out.
1642pub fn wtf8Encode(c: u21, out: []u8) error{CodepointTooLarge}!u3 {
1643 return utf8EncodeImpl(c, out, .can_encode_surrogate_half);
1644}
1645
1646const Wtf8DecodeError = Utf8Decode2Error || Utf8Decode3AllowSurrogateHalfError || Utf8Decode4Error;
1647
1648/// Deprecated. This function has an awkward API that is too easy to use incorrectly.
1649pub fn wtf8Decode(bytes: []const u8) Wtf8DecodeError!u21 {
1650 return switch (bytes.len) {
1651 1 => bytes[0],
1652 2 => utf8Decode2(bytes[0..2].*),
1653 3 => utf8Decode3AllowSurrogateHalf(bytes[0..3].*),
1654 4 => utf8Decode4(bytes[0..4].*),
1655 else => unreachable,
1656 };
1657}
1658
1659/// Returns true if the input consists entirely of WTF-8 codepoints
1660/// (all the same restrictions as UTF-8, but allows surrogate codepoints
1661/// U+D800 to U+DFFF).
1662/// Does not check for well-formed WTF-8, meaning that this function
1663/// does not check that all surrogate halves are unpaired.
1664pub fn wtf8ValidateSlice(input: []const u8) bool {
1665 return utf8ValidateSliceImpl(input, .can_encode_surrogate_half);
1666}
1667
1668test "validate WTF-8 slice" {
1669 try testValidateWtf8Slice();
1670 try comptime testValidateWtf8Slice();
1671
1672 // We skip a variable (based on recommended vector size) chunks of
1673 // ASCII characters. Let's make sure we're chunking correctly.
1674 const str = @as([550]u8, @splat('a')) ++ "\xc0";
1675 for (0..str.len - 3) |i| {
1676 try testing.expect(!wtf8ValidateSlice(str[i..]));
1677 }
1678}
1679fn testValidateWtf8Slice() !void {
1680 // These are valid/invalid under both UTF-8 and WTF-8 rules.
1681 try testing.expect(wtf8ValidateSlice("abc"));
1682 try testing.expect(wtf8ValidateSlice("abc\xdf\xbf"));
1683 try testing.expect(wtf8ValidateSlice(""));
1684 try testing.expect(wtf8ValidateSlice("a"));
1685 try testing.expect(wtf8ValidateSlice("abc"));
1686 try testing.expect(wtf8ValidateSlice("Ж"));
1687 try testing.expect(wtf8ValidateSlice("ЖЖ"));
1688 try testing.expect(wtf8ValidateSlice("брэд-ЛГТМ"));
1689 try testing.expect(wtf8ValidateSlice("☺☻☹"));
1690 try testing.expect(wtf8ValidateSlice("a\u{fffdb}"));
1691 try testing.expect(wtf8ValidateSlice("\xf4\x8f\xbf\xbf"));
1692 try testing.expect(wtf8ValidateSlice("abc\xdf\xbf"));
1693
1694 try testing.expect(!wtf8ValidateSlice("abc\xc0"));
1695 try testing.expect(!wtf8ValidateSlice("abc\xc0abc"));
1696 try testing.expect(!wtf8ValidateSlice("aa\xe2"));
1697 try testing.expect(!wtf8ValidateSlice("\x42\xfa"));
1698 try testing.expect(!wtf8ValidateSlice("\x42\xfa\x43"));
1699 try testing.expect(!wtf8ValidateSlice("abc\xc0"));
1700 try testing.expect(!wtf8ValidateSlice("abc\xc0abc"));
1701 try testing.expect(!wtf8ValidateSlice("\xf4\x90\x80\x80"));
1702 try testing.expect(!wtf8ValidateSlice("\xf7\xbf\xbf\xbf"));
1703 try testing.expect(!wtf8ValidateSlice("\xfb\xbf\xbf\xbf\xbf"));
1704 try testing.expect(!wtf8ValidateSlice("\xc0\x80"));
1705
1706 // But surrogate codepoints are only valid in WTF-8.
1707 try testing.expect(wtf8ValidateSlice("\xed\xa0\x80"));
1708 try testing.expect(wtf8ValidateSlice("\xed\xbf\xbf"));
1709}
1710
1711/// Wtf8View iterates the code points of a WTF-8 encoded string,
1712/// including surrogate halves.
1713///
1714/// ```
1715/// var wtf8 = (try std.unicode.Wtf8View.init("hi there")).iterator();
1716/// while (wtf8.nextCodepointSlice()) |codepoint| {
1717/// // note: codepoint could be a surrogate half which is invalid
1718/// // UTF-8, avoid printing or otherwise sending/emitting this directly
1719/// }
1720/// ```
1721pub const Wtf8View = struct {
1722 bytes: []const u8,
1723
1724 pub fn init(s: []const u8) error{InvalidWtf8}!Wtf8View {
1725 if (!wtf8ValidateSlice(s)) {
1726 return error.InvalidWtf8;
1727 }
1728
1729 return initUnchecked(s);
1730 }
1731
1732 pub fn initUnchecked(s: []const u8) Wtf8View {
1733 return Wtf8View{ .bytes = s };
1734 }
1735
1736 pub inline fn initComptime(comptime s: []const u8) Wtf8View {
1737 return comptime if (init(s)) |r| r else |err| switch (err) {
1738 error.InvalidWtf8 => {
1739 @compileError("invalid wtf8");
1740 },
1741 };
1742 }
1743
1744 pub fn iterator(s: Wtf8View) Wtf8Iterator {
1745 return Wtf8Iterator{
1746 .bytes = s.bytes,
1747 .i = 0,
1748 };
1749 }
1750};
1751
1752/// Asserts that `bytes` is valid WTF-8
1753pub const Wtf8Iterator = struct {
1754 bytes: []const u8,
1755 i: usize,
1756
1757 pub fn nextCodepointSlice(it: *Wtf8Iterator) ?[]const u8 {
1758 if (it.i >= it.bytes.len) {
1759 return null;
1760 }
1761
1762 const cp_len = utf8ByteSequenceLength(it.bytes[it.i]) catch unreachable;
1763 it.i += cp_len;
1764 return it.bytes[it.i - cp_len .. it.i];
1765 }
1766
1767 pub fn nextCodepoint(it: *Wtf8Iterator) ?u21 {
1768 const slice = it.nextCodepointSlice() orelse return null;
1769 return wtf8Decode(slice) catch unreachable;
1770 }
1771
1772 /// Look ahead at the next n codepoints without advancing the iterator.
1773 /// If fewer than n codepoints are available, then return the remainder of the string.
1774 pub fn peek(it: *Wtf8Iterator, n: usize) []const u8 {
1775 const original_i = it.i;
1776 defer it.i = original_i;
1777
1778 var end_ix = original_i;
1779 var found: usize = 0;
1780 while (found < n) : (found += 1) {
1781 const next_codepoint = it.nextCodepointSlice() orelse return it.bytes[original_i..];
1782 end_ix += next_codepoint.len;
1783 }
1784
1785 return it.bytes[original_i..end_ix];
1786 }
1787
1788 /// Look ahead at the next codepoint without advancing the iterator.
1789 /// If no codepoints exist, then returns null.
1790 pub fn peekCodepoint(it: *Wtf8Iterator) ?u21 {
1791 const original_i = it.i;
1792 defer it.i = original_i;
1793
1794 return it.nextCodepoint();
1795 }
1796};
1797
1798pub fn wtf16LeToWtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Allocator.Error!void {
1799 try result.ensureUnusedCapacity(utf16le.len);
1800 return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half);
1801}
1802
1803/// Caller must free returned memory.
1804pub fn wtf16LeToWtf8Alloc(allocator: Allocator, wtf16le: []const u16) Allocator.Error![]u8 {
1805 // optimistically guess that it will all be ascii.
1806 var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len);
1807 errdefer result.deinit();
1808
1809 try utf16LeToUtf8ArrayListImpl(&result, wtf16le, .can_encode_surrogate_half);
1810 return result.toOwnedSlice();
1811}
1812
1813/// Caller must free returned memory.
1814pub fn wtf16LeToWtf8AllocZ(allocator: Allocator, wtf16le: []const u16) Allocator.Error![:0]u8 {
1815 // optimistically guess that it will all be ascii (and allocate space for the null terminator)
1816 var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len + 1);
1817 errdefer result.deinit();
1818
1819 try utf16LeToUtf8ArrayListImpl(&result, wtf16le, .can_encode_surrogate_half);
1820 return result.toOwnedSliceSentinel(0);
1821}
1822
1823pub fn wtf16LeToWtf8(wtf8: []u8, wtf16le: []const u16) usize {
1824 return utf16LeToUtf8Impl(wtf8, wtf16le, .can_encode_surrogate_half) catch |err| switch (err) {};
1825}
1826
1827pub fn wtf8ToWtf16LeArrayList(result: *std.array_list.Managed(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void {
1828 try result.ensureUnusedCapacity(wtf8.len);
1829 return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half);
1830}
1831
1832pub fn wtf8ToWtf16LeAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 {
1833 // optimistically guess that it will not require surrogate pairs
1834 var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len);
1835 errdefer result.deinit();
1836
1837 try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
1838 return result.toOwnedSlice();
1839}
1840
1841pub fn wtf8ToWtf16LeAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u16 {
1842 // optimistically guess that it will not require surrogate pairs
1843 var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len + 1);
1844 errdefer result.deinit();
1845
1846 try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
1847 return result.toOwnedSliceSentinel(0);
1848}
1849
1850/// Returns index of next character. If exact fit, returned index equals output slice length.
1851/// Assumes there is enough space for the output.
1852pub fn wtf8ToWtf16Le(wtf16le: []u16, wtf8: []const u8) error{InvalidWtf8}!usize {
1853 return utf8ToUtf16LeImpl(wtf16le, wtf8, .can_encode_surrogate_half);
1854}
1855
1856/// Surrogate codepoints (U+D800 to U+DFFF) are replaced by the Unicode replacement
1857/// character (U+FFFD).
1858/// All surrogate codepoints and the replacement character are encoded as three
1859/// bytes, meaning the input and output slices will always be the same length.
1860/// In-place conversion is supported when `utf8` and `wtf8` refer to the same slice.
1861/// Note: If `wtf8` is entirely composed of well-formed UTF-8, then no conversion is necessary.
1862/// `utf8ValidateSlice` can be used to check if lossy conversion is worthwhile.
1863/// If `wtf8` is not valid WTF-8, then `error.InvalidWtf8` is returned.
1864pub fn wtf8ToUtf8Lossy(utf8: []u8, wtf8: []const u8) error{InvalidWtf8}!void {
1865 assert(utf8.len >= wtf8.len);
1866
1867 const in_place = utf8.ptr == wtf8.ptr;
1868 const replacement_char_bytes = comptime blk: {
1869 var buf: [3]u8 = undefined;
1870 assert((utf8Encode(replacement_character, &buf) catch unreachable) == 3);
1871 break :blk buf;
1872 };
1873
1874 var dest_i: usize = 0;
1875 const view = try Wtf8View.init(wtf8);
1876 var it = view.iterator();
1877 while (it.nextCodepointSlice()) |codepoint_slice| {
1878 // All surrogate codepoints are encoded as 3 bytes
1879 if (codepoint_slice.len == 3) {
1880 const codepoint = wtf8Decode(codepoint_slice) catch unreachable;
1881 if (isSurrogateCodepoint(codepoint)) {
1882 @memcpy(utf8[dest_i..][0..replacement_char_bytes.len], &replacement_char_bytes);
1883 dest_i += replacement_char_bytes.len;
1884 continue;
1885 }
1886 }
1887 if (!in_place) {
1888 @memcpy(utf8[dest_i..][0..codepoint_slice.len], codepoint_slice);
1889 }
1890 dest_i += codepoint_slice.len;
1891 }
1892}
1893
1894pub fn wtf8ToUtf8LossyAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u8 {
1895 const utf8 = try allocator.alloc(u8, wtf8.len);
1896 errdefer allocator.free(utf8);
1897
1898 try wtf8ToUtf8Lossy(utf8, wtf8);
1899
1900 return utf8;
1901}
1902
1903pub fn wtf8ToUtf8LossyAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u8 {
1904 const utf8 = try allocator.allocSentinel(u8, wtf8.len, 0);
1905 errdefer allocator.free(utf8);
1906
1907 try wtf8ToUtf8Lossy(utf8, wtf8);
1908
1909 return utf8;
1910}
1911
1912test wtf8ToUtf8Lossy {
1913 var buf: [32]u8 = undefined;
1914
1915 const invalid_utf8 = "\xff";
1916 try testing.expectError(error.InvalidWtf8, wtf8ToUtf8Lossy(&buf, invalid_utf8));
1917
1918 const ascii = "abcd";
1919 try wtf8ToUtf8Lossy(&buf, ascii);
1920 try testing.expectEqualStrings("abcd", buf[0..ascii.len]);
1921
1922 const high_surrogate_half = "ab\xed\xa0\xbdcd";
1923 try wtf8ToUtf8Lossy(&buf, high_surrogate_half);
1924 try testing.expectEqualStrings("ab\u{FFFD}cd", buf[0..high_surrogate_half.len]);
1925
1926 const low_surrogate_half = "ab\xed\xb2\xa9cd";
1927 try wtf8ToUtf8Lossy(&buf, low_surrogate_half);
1928 try testing.expectEqualStrings("ab\u{FFFD}cd", buf[0..low_surrogate_half.len]);
1929
1930 // If the WTF-8 is not well-formed, each surrogate half is converted into a separate
1931 // replacement character instead of being interpreted as a surrogate pair.
1932 const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
1933 try wtf8ToUtf8Lossy(&buf, encoded_surrogate_pair);
1934 try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", buf[0..encoded_surrogate_pair.len]);
1935
1936 // in place
1937 @memcpy(buf[0..low_surrogate_half.len], low_surrogate_half);
1938 const slice = buf[0..low_surrogate_half.len];
1939 try wtf8ToUtf8Lossy(slice, slice);
1940 try testing.expectEqualStrings("ab\u{FFFD}cd", slice);
1941}
1942
1943test wtf8ToUtf8LossyAlloc {
1944 const invalid_utf8 = "\xff";
1945 try testing.expectError(error.InvalidWtf8, wtf8ToUtf8LossyAlloc(testing.allocator, invalid_utf8));
1946
1947 {
1948 const ascii = "abcd";
1949 const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, ascii);
1950 defer testing.allocator.free(utf8);
1951 try testing.expectEqualStrings("abcd", utf8);
1952 }
1953
1954 {
1955 const surrogate_half = "ab\xed\xa0\xbdcd";
1956 const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, surrogate_half);
1957 defer testing.allocator.free(utf8);
1958 try testing.expectEqualStrings("ab\u{FFFD}cd", utf8);
1959 }
1960
1961 {
1962 // If the WTF-8 is not well-formed, each surrogate half is converted into a separate
1963 // replacement character instead of being interpreted as a surrogate pair.
1964 const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
1965 const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, encoded_surrogate_pair);
1966 defer testing.allocator.free(utf8);
1967 try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", utf8);
1968 }
1969}
1970
1971test wtf8ToUtf8LossyAllocZ {
1972 const invalid_utf8 = "\xff";
1973 try testing.expectError(error.InvalidWtf8, wtf8ToUtf8LossyAllocZ(testing.allocator, invalid_utf8));
1974
1975 {
1976 const ascii = "abcd";
1977 const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, ascii);
1978 defer testing.allocator.free(utf8);
1979 try testing.expectEqualStrings("abcd", utf8);
1980 }
1981
1982 {
1983 const surrogate_half = "ab\xed\xa0\xbdcd";
1984 const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, surrogate_half);
1985 defer testing.allocator.free(utf8);
1986 try testing.expectEqualStrings("ab\u{FFFD}cd", utf8);
1987 }
1988
1989 {
1990 // If the WTF-8 is not well-formed, each surrogate half is converted into a separate
1991 // replacement character instead of being interpreted as a surrogate pair.
1992 const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
1993 const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, encoded_surrogate_pair);
1994 defer testing.allocator.free(utf8);
1995 try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", utf8);
1996 }
1997}
1998
1999pub const Wtf16LeIterator = struct {
2000 bytes: []const u8,
2001 i: usize,
2002
2003 pub fn init(s: []const u16) Wtf16LeIterator {
2004 return Wtf16LeIterator{
2005 .bytes = mem.sliceAsBytes(s),
2006 .i = 0,
2007 };
2008 }
2009
2010 /// If the next codepoint is encoded by a surrogate pair, returns the
2011 /// codepoint that the surrogate pair represents.
2012 /// If the next codepoint is an unpaired surrogate, returns the codepoint
2013 /// of the unpaired surrogate.
2014 pub fn nextCodepoint(it: *Wtf16LeIterator) ?u21 {
2015 assert(it.i <= it.bytes.len);
2016 if (it.i == it.bytes.len) return null;
2017 var code_units: [2]u16 = undefined;
2018 code_units[0] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
2019 it.i += 2;
2020 surrogate_pair: {
2021 if (utf16IsHighSurrogate(code_units[0])) {
2022 if (it.i >= it.bytes.len) break :surrogate_pair;
2023 code_units[1] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
2024 const codepoint = utf16DecodeSurrogatePair(&code_units) catch break :surrogate_pair;
2025 it.i += 2;
2026 return codepoint;
2027 }
2028 }
2029 return code_units[0];
2030 }
2031};
2032
2033test "non-well-formed WTF-8 does not roundtrip" {
2034 // This encodes the surrogate pair U+D83D U+DCA9.
2035 // The well-formed version of this would be U+1F4A9 which is \xF0\x9F\x92\xA9.
2036 const non_well_formed_wtf8 = "\xed\xa0\xbd\xed\xb2\xa9";
2037
2038 var wtf16_buf: [2]u16 = undefined;
2039 const wtf16_len = try wtf8ToWtf16Le(&wtf16_buf, non_well_formed_wtf8);
2040 const wtf16 = wtf16_buf[0..wtf16_len];
2041
2042 try testing.expectEqualSlices(u16, &[_]u16{
2043 mem.nativeToLittle(u16, 0xD83D), // high surrogate
2044 mem.nativeToLittle(u16, 0xDCA9), // low surrogate
2045 }, wtf16);
2046
2047 var wtf8_buf: [4]u8 = undefined;
2048 const wtf8_len = wtf16LeToWtf8(&wtf8_buf, wtf16);
2049 const wtf8 = wtf8_buf[0..wtf8_len];
2050
2051 // Converting to WTF-16 and back results in well-formed WTF-8,
2052 // but it does not match the input WTF-8
2053 try testing.expectEqualSlices(u8, "\xf0\x9f\x92\xa9", wtf8);
2054}
2055
2056fn testRoundtripWtf8(wtf8: []const u8) !void {
2057 // Buffer
2058 {
2059 var wtf16_buf: [32]u16 = undefined;
2060 const wtf16_len = try wtf8ToWtf16Le(&wtf16_buf, wtf8);
2061 try testing.expectEqual(wtf16_len, calcWtf16LeLen(wtf8));
2062 const wtf16 = wtf16_buf[0..wtf16_len];
2063
2064 var roundtripped_buf: [32]u8 = undefined;
2065 const roundtripped_len = wtf16LeToWtf8(&roundtripped_buf, wtf16);
2066 const roundtripped = roundtripped_buf[0..roundtripped_len];
2067
2068 try testing.expectEqualSlices(u8, wtf8, roundtripped);
2069 }
2070 // Alloc
2071 {
2072 const wtf16 = try wtf8ToWtf16LeAlloc(testing.allocator, wtf8);
2073 defer testing.allocator.free(wtf16);
2074
2075 const roundtripped = try wtf16LeToWtf8Alloc(testing.allocator, wtf16);
2076 defer testing.allocator.free(roundtripped);
2077
2078 try testing.expectEqualSlices(u8, wtf8, roundtripped);
2079 }
2080 // AllocZ
2081 {
2082 const wtf16 = try wtf8ToWtf16LeAllocZ(testing.allocator, wtf8);
2083 defer testing.allocator.free(wtf16);
2084
2085 const roundtripped = try wtf16LeToWtf8AllocZ(testing.allocator, wtf16);
2086 defer testing.allocator.free(roundtripped);
2087
2088 try testing.expectEqualSlices(u8, wtf8, roundtripped);
2089 }
2090}
2091
2092test "well-formed WTF-8 roundtrips" {
2093 try testRoundtripWtf8("\xed\x9f\xbf"); // not a surrogate half
2094 try testRoundtripWtf8("\xed\xa0\xbd"); // high surrogate
2095 try testRoundtripWtf8("\xed\xb2\xa9"); // low surrogate
2096 try testRoundtripWtf8("\xed\xa0\xbd \xed\xb2\xa9"); // <high surrogate><space><low surrogate>
2097 try testRoundtripWtf8("\xed\xa0\x80\xed\xaf\xbf"); // <high surrogate><high surrogate>
2098 try testRoundtripWtf8("\xed\xa0\x80\xee\x80\x80"); // <high surrogate><not surrogate>
2099 try testRoundtripWtf8("\xed\x9f\xbf\xed\xb0\x80"); // <not surrogate><low surrogate>
2100 try testRoundtripWtf8("a\xed\xb0\x80"); // <not surrogate><low surrogate>
2101 try testRoundtripWtf8("\xf0\x9f\x92\xa9"); // U+1F4A9, encoded as a surrogate pair in WTF-16
2102}
2103
2104fn testRoundtripWtf16(wtf16le: []const u16) !void {
2105 // Buffer
2106 {
2107 var wtf8_buf: [32]u8 = undefined;
2108 const wtf8_len = wtf16LeToWtf8(&wtf8_buf, wtf16le);
2109 const wtf8 = wtf8_buf[0..wtf8_len];
2110
2111 var roundtripped_buf: [32]u16 = undefined;
2112 const roundtripped_len = try wtf8ToWtf16Le(&roundtripped_buf, wtf8);
2113 const roundtripped = roundtripped_buf[0..roundtripped_len];
2114
2115 try testing.expectEqualSlices(u16, wtf16le, roundtripped);
2116 }
2117 // Alloc
2118 {
2119 const wtf8 = try wtf16LeToWtf8Alloc(testing.allocator, wtf16le);
2120 defer testing.allocator.free(wtf8);
2121
2122 const roundtripped = try wtf8ToWtf16LeAlloc(testing.allocator, wtf8);
2123 defer testing.allocator.free(roundtripped);
2124
2125 try testing.expectEqualSlices(u16, wtf16le, roundtripped);
2126 }
2127 // AllocZ
2128 {
2129 const wtf8 = try wtf16LeToWtf8AllocZ(testing.allocator, wtf16le);
2130 defer testing.allocator.free(wtf8);
2131
2132 const roundtripped = try wtf8ToWtf16LeAllocZ(testing.allocator, wtf8);
2133 defer testing.allocator.free(roundtripped);
2134
2135 try testing.expectEqualSlices(u16, wtf16le, roundtripped);
2136 }
2137}
2138
2139test "well-formed WTF-16 roundtrips" {
2140 try testRoundtripWtf16(&[_]u16{
2141 mem.nativeToLittle(u16, 0xD83D), // high surrogate
2142 mem.nativeToLittle(u16, 0xDCA9), // low surrogate
2143 });
2144 try testRoundtripWtf16(&[_]u16{
2145 mem.nativeToLittle(u16, 0xD83D), // high surrogate
2146 mem.nativeToLittle(u16, ' '), // not surrogate
2147 mem.nativeToLittle(u16, 0xDCA9), // low surrogate
2148 });
2149 try testRoundtripWtf16(&[_]u16{
2150 mem.nativeToLittle(u16, 0xD800), // high surrogate
2151 mem.nativeToLittle(u16, 0xDBFF), // high surrogate
2152 });
2153 try testRoundtripWtf16(&[_]u16{
2154 mem.nativeToLittle(u16, 0xD800), // high surrogate
2155 mem.nativeToLittle(u16, 0xE000), // not surrogate
2156 });
2157 try testRoundtripWtf16(&[_]u16{
2158 mem.nativeToLittle(u16, 0xD7FF), // not surrogate
2159 mem.nativeToLittle(u16, 0xDC00), // low surrogate
2160 });
2161 try testRoundtripWtf16(&[_]u16{
2162 mem.nativeToLittle(u16, 0x61), // not surrogate
2163 mem.nativeToLittle(u16, 0xDC00), // low surrogate
2164 });
2165 try testRoundtripWtf16(&[_]u16{
2166 mem.nativeToLittle(u16, 0xDC00), // low surrogate
2167 });
2168}
2169
2170/// Returns the length, in bytes, that would be necessary to encode the
2171/// given WTF-16 LE slice as WTF-8.
2172pub fn calcWtf8Len(wtf16le: []const u16) usize {
2173 var it = Wtf16LeIterator.init(wtf16le);
2174 var num_wtf8_bytes: usize = 0;
2175 while (it.nextCodepoint()) |codepoint| {
2176 // Note: If utf8CodepointSequenceLength is ever changed to error on surrogate
2177 // codepoints, then it would no longer be eligible to be used in this context.
2178 num_wtf8_bytes += utf8CodepointSequenceLength(codepoint) catch |err| switch (err) {
2179 error.CodepointTooLarge => unreachable,
2180 };
2181 }
2182 return num_wtf8_bytes;
2183}
2184
2185fn testCalcWtf8Len() !void {
2186 const L = utf8ToUtf16LeStringLiteral;
2187 try testing.expectEqual(@as(usize, 1), calcWtf8Len(L("a")));
2188 try testing.expectEqual(@as(usize, 10), calcWtf8Len(L("abcdefghij")));
2189 // unpaired surrogate
2190 try testing.expectEqual(@as(usize, 3), calcWtf8Len(&[_]u16{
2191 mem.nativeToLittle(u16, 0xD800),
2192 }));
2193 try testing.expectEqual(@as(usize, 15), calcWtf8Len(L("こんにちは")));
2194 // First codepoints that are encoded as 1, 2, 3, and 4 bytes
2195 try testing.expectEqual(@as(usize, 1 + 2 + 3 + 4), calcWtf8Len(L("\u{0}\u{80}\u{800}\u{10000}")));
2196}
2197
2198test "calculate wtf8 string length of given wtf16 string" {
2199 try testCalcWtf8Len();
2200 try comptime testCalcWtf8Len();
2201}