authorgravatar for r00ster91@protonmail.comr00ster91 <r00ster91@protonmail.com> 2022-04-13 18:28:49+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-04-15 11:20:11+03:00
log62d717e2ffb1e9a1127652521de57c2e18cf7d3b
treea6f338f451583713189c43d5a44dd3e155965c35
parent618398b7d3c0df13dcb3d87540e400665b2c02dc

Add `std.unicode.replacement_character`

2 files changed, 10 insertions(+), 6 deletions(-)

lib/std/fmt.zig+3-3
...@@ -966,10 +966,10 @@ pub fn formatUnicodeCodepoint(...@@ -966,10 +966,10 @@ pub fn formatUnicodeCodepoint(
966 writer: anytype,966 writer: anytype,
967) !void {967) !void {
968 var buf: [4]u8 = undefined;968 var buf: [4]u8 = undefined;
969 const len = std.unicode.utf8Encode(c, &buf) catch |err| switch (err) {969 const len = unicode.utf8Encode(c, &buf) catch |err| switch (err) {
970 error.Utf8CannotEncodeSurrogateHalf, error.CodepointTooLarge => {970 error.Utf8CannotEncodeSurrogateHalf, error.CodepointTooLarge => {
971 // In case of error output the replacement char U+FFFD971 const len = unicode.utf8Encode(unicode.replacement_character, &buf) catch unreachable;
972 return formatBuf(&[_]u8{ 0xef, 0xbf, 0xbd }, options, writer);972 return formatBuf(buf[0..len], options, writer);
973 },973 },
974 };974 };
975 return formatBuf(buf[0..len], options, writer);975 return formatBuf(buf[0..len], options, writer);
lib/std/unicode.zig+7-3
...@@ -3,6 +3,11 @@ const assert = std.debug.assert;...@@ -3,6 +3,11 @@ const assert = std.debug.assert;
3const testing = std.testing;3const testing = std.testing;
4const mem = std.mem;4const mem = std.mem;
55
6/// Use this to replace an unknown, unrecognized, or unrepresentable character.
7///
8/// See also: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character
9pub const replacement_character: u21 = 0xFFFD;
10
6/// Returns how many bytes the UTF-8 representation would require11/// Returns how many bytes the UTF-8 representation would require
7/// for the given codepoint.12/// for the given codepoint.
8pub fn utf8CodepointSequenceLength(c: u21) !u3 {13pub fn utf8CodepointSequenceLength(c: u21) !u3 {
...@@ -777,15 +782,14 @@ fn formatUtf16le(...@@ -777,15 +782,14 @@ fn formatUtf16le(
777 options: std.fmt.FormatOptions,782 options: std.fmt.FormatOptions,
778 writer: anytype,783 writer: anytype,
779) !void {784) !void {
780 const unknown_codepoint = 0xfffd;
781 _ = fmt;785 _ = fmt;
782 _ = options;786 _ = options;
783 var buf: [300]u8 = undefined; // just a random size I chose787 var buf: [300]u8 = undefined; // just a random size I chose
784 var it = Utf16LeIterator.init(utf16le);788 var it = Utf16LeIterator.init(utf16le);
785 var u8len: usize = 0;789 var u8len: usize = 0;
786 while (it.nextCodepoint() catch unknown_codepoint) |codepoint| {790 while (it.nextCodepoint() catch replacement_character) |codepoint| {
787 u8len += utf8Encode(codepoint, buf[u8len..]) catch791 u8len += utf8Encode(codepoint, buf[u8len..]) catch
788 utf8Encode(unknown_codepoint, buf[u8len..]) catch unreachable;792 utf8Encode(replacement_character, buf[u8len..]) catch unreachable;
789 if (u8len + 3 >= buf.len) {793 if (u8len + 3 >= buf.len) {
790 try writer.writeAll(buf[0..u8len]);794 try writer.writeAll(buf[0..u8len]);
791 u8len = 0;795 u8len = 0;