authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-10-21 09:43:04+02:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-10-21 09:43:04+02:00
log8f0c6ce407f96d04e9f4b7b66b2ce18134775969
treee039a597c7095989eef6f6304f700632f336ce0e
parent3c08167cb766f8d00fc5a219e382abba12e57ede
parentb77411310c94db7f6f1678e189f9d8954828b78c

Merge pull request 'Allow \u escape sequences to encode surrogate code points' (#36965) from squeek502/zig:surrogates into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36965

3 files changed, 18 insertions(+), 9 deletions(-)

doc/langref.html.in+11-2
...@@ -755,12 +755,21 @@...@@ -755,12 +755,21 @@
755 </tr>755 </tr>
756 <tr>756 <tr>
757 <th scope="row"><code>\u{NNNNNN}</code></th>757 <th scope="row"><code>\u{NNNNNN}</code></th>
758 <td>hexadecimal Unicode scalar value UTF-8 encoded (1 or more digits)</td>758 <td>hexadecimal Unicode code point (1 or more digits)</td>
759 </tr>759 </tr>
760 </tbody>760 </tbody>
761 </table>761 </table>
762 </div>762 </div>
763 <p>Note that the maximum valid Unicode scalar value is {#syntax#}0x10ffff{#endsyntax#}.</p>763 <p>
764 Note that the highest Unicode code point is {#syntax#}0x10ffff{#endsyntax#}. Character
765 literals are integers ({#syntax#}'A'{#endsyntax#} = {#syntax#}65{#endsyntax#} and
766 {#syntax#}'\u{10000}'{#endsyntax#} = {#syntax#}0x10000{#endsyntax#} = {#syntax#}65536{#endsyntax#}),
767 and <code>\u</code> escapes in string literals are encoded as UTF-8
768 ({#syntax#}"A"{#endsyntax#} = {#syntax#}"\u{41}"{#endsyntax#} = {#syntax#}"\x41"{#endsyntax#}
769 and {#syntax#}"\u{10000}"{#endsyntax#} = {#syntax#}"\xF0\x90\x80\x80"{#endsyntax#}).
770 <code>\u</code> escapes can be used for surrogate code points, which are not Unicode
771 scalar values or technically allowed in UTF-8 as defined by the Unicode standard, but can be useful in practice.
772 </p>
764 {#header_close#}773 {#header_close#}
765 {#header_open|Multiline String Literals#}774 {#header_open|Multiline String Literals#}
766 <p>775 <p>
lib/compiler/Maker/Package/Manifest.zig+1-1
...@@ -544,7 +544,7 @@ const Parse = struct {...@@ -544,7 +544,7 @@ const Parse = struct {
544 try p.appendErrorOff(544 try p.appendErrorOff(
545 token,545 token,
546 offset + @as(u32, @intCast(bad_index)),546 offset + @as(u32, @intCast(bad_index)),
547 "unicode escape does not correspond to a valid unicode scalar value",547 "unicode escape does not correspond to a unicode code point",
548 .{},548 .{},
549 );549 );
550 },550 },
lib/std/zig/string_literal.zig+6-6
...@@ -1,6 +1,5 @@...@@ -1,6 +1,5 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const assert = std.debug.assert;2const assert = std.debug.assert;
3const utf8Encode = std.unicode.utf8Encode;
4const Writer = std.Io.Writer;3const Writer = std.Io.Writer;
54
6pub const ParseError = error{5pub const ParseError = error{
...@@ -63,7 +62,7 @@ pub const Error = union(enum) {...@@ -63,7 +62,7 @@ pub const Error = union(enum) {
63 .{self.raw_string[bad_index]},62 .{self.raw_string[bad_index]},
64 ),63 ),
65 .invalid_unicode_codepoint => try writer.writeAll(64 .invalid_unicode_codepoint => try writer.writeAll(
66 "unicode escape does not correspond to a valid unicode scalar value",65 "unicode escape does not correspond to a unicode code point",
67 ),66 ),
68 .expected_lbrace => |bad_index| try writer.print(67 .expected_lbrace => |bad_index| try writer.print(
69 "expected '{{', found '{c}'",68 "expected '{{', found '{c}'",
...@@ -334,8 +333,8 @@ pub fn parseWrite(writer: *Writer, bytes: []const u8) Writer.Error!Result {...@@ -334,8 +333,8 @@ pub fn parseWrite(writer: *Writer, bytes: []const u8) Writer.Error!Result {
334 .success => |codepoint| {333 .success => |codepoint| {
335 if (bytes[escape_char_index] == 'u') {334 if (bytes[escape_char_index] == 'u') {
336 var buf: [4]u8 = undefined;335 var buf: [4]u8 = undefined;
337 const len = utf8Encode(codepoint, &buf) catch {336 const len = std.unicode.wtf8Encode(codepoint, &buf) catch |err| switch (err) {
338 return .{ .failure = .{ .invalid_unicode_codepoint = escape_char_index + 1 } };337 error.CodepointTooLarge => unreachable, // Checked in `parseEscapeSequence`.
339 };338 };
340 try writer.writeAll(buf[0..len]);339 try writer.writeAll(buf[0..len]);
341 } else {340 } else {
...@@ -383,6 +382,7 @@ test parseAlloc {...@@ -383,6 +382,7 @@ test parseAlloc {
383 try expect(eql(u8, "\x12foo", try parseAlloc(alloc, "\"\\x12foo\"")));382 try expect(eql(u8, "\x12foo", try parseAlloc(alloc, "\"\\x12foo\"")));
384 try expect(eql(u8, "bytes\u{1234}foo", try parseAlloc(alloc, "\"bytes\\u{1234}foo\"")));383 try expect(eql(u8, "bytes\u{1234}foo", try parseAlloc(alloc, "\"bytes\\u{1234}foo\"")));
385 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"foo\"")));384 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"foo\"")));
386 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"f\x6f\x6f\"")));385 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"f\\x6f\\x6f\"")));
387 try expect(eql(u8, "f💯", try parseAlloc(alloc, "\"f\u{1f4af}\"")));386 try expect(eql(u8, "f💯", try parseAlloc(alloc, "\"f\\u{1f4af}\"")));
387 try expect(eql(u8, "\xed\xa0\x80", try parseAlloc(alloc, "\"\\u{d800}\"")));
388}388}