authorgravatar for oa223@cam.ac.ukOeistein Andersen <oa223@cam.ac.uk> 2025-03-05 12:06:23+00:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-09-26 17:29:55-07:00
loge1f7e3563fa84b08e707a3dcea610560d179c63d
tree1f1a84d58a8ca26432ff379f3467a450a4544764
parent5b9147ed59b22edf7f891cf30e8b77223121da4f

Allow \u escape in string literal to encode surrogate code point.

3 files changed, 16 insertions(+), 7 deletions(-)

doc/langref.html.in+11-2
...@@ -754,12 +754,21 @@...@@ -754,12 +754,21 @@
754 </tr>754 </tr>
755 <tr>755 <tr>
756 <th scope="row"><code>\u{NNNNNN}</code></th>756 <th scope="row"><code>\u{NNNNNN}</code></th>
757 <td>hexadecimal Unicode scalar value UTF-8 encoded (1 or more digits)</td>757 <td>hexadecimal Unicode code point (1 or more digits)</td>
758 </tr>758 </tr>
759 </tbody>759 </tbody>
760 </table>760 </table>
761 </div>761 </div>
762 <p>Note that the maximum valid Unicode scalar value is {#syntax#}0x10ffff{#endsyntax#}.</p>762 <p>
763 Note that the highest Unicode code point is {#syntax#}0x10ffff{#endsyntax#}. Character
764 literals are integers ({#syntax#}'A'{#endsyntax#} = {#syntax#}65{#endsyntax#} and
765 {#syntax#}'\u{10000}'{#endsyntax#} = {#syntax#}0x10000{#endsyntax#} = {#syntax#}65536{#endsyntax#}),
766 and <code>\u</code> escapes in string literals are encoded as UTF-8
767 ({#syntax#}"A"{#endsyntax#} = {#syntax#}"\u{41}"{#endsyntax#} = {#syntax#}"\x41"{#endsyntax#}
768 and {#syntax#}"\u{10000}"{#endsyntax#} = {#syntax#}"\xF0\x90\x80\x80"{#endsyntax#}).
769 <code>\u</code> escapes can be used for surrogate code points, which are not Unicode
770 scalar values or technically allowed in UTF-8 as defined by the Unicode standard, but can be useful in practice.
771 </p>
763 {#header_close#}772 {#header_close#}
764 {#header_open|Multiline String Literals#}773 {#header_open|Multiline String Literals#}
765 <p>774 <p>
lib/compiler/Maker/Package/Manifest.zig+1-1
...@@ -508,7 +508,7 @@ const Parse = struct {...@@ -508,7 +508,7 @@ const Parse = struct {
508 try p.appendErrorOff(508 try p.appendErrorOff(
509 token,509 token,
510 offset + @as(u32, @intCast(bad_index)),510 offset + @as(u32, @intCast(bad_index)),
511 "unicode escape does not correspond to a valid unicode scalar value",511 "unicode escape does not correspond to a unicode code point",
512 .{},512 .{},
513 );513 );
514 },514 },
lib/std/zig/string_literal.zig+4-4
...@@ -1,6 +1,5 @@...@@ -1,6 +1,5 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const assert = std.debug.assert;2const assert = std.debug.assert;
3const utf8Encode = std.unicode.utf8Encode;
4const Writer = std.Io.Writer;3const Writer = std.Io.Writer;
54
6pub const ParseError = error{5pub const ParseError = error{
...@@ -63,7 +62,7 @@ pub const Error = union(enum) {...@@ -63,7 +62,7 @@ pub const Error = union(enum) {
63 .{self.raw_string[bad_index]},62 .{self.raw_string[bad_index]},
64 ),63 ),
65 .invalid_unicode_codepoint => try writer.writeAll(64 .invalid_unicode_codepoint => try writer.writeAll(
66 "unicode escape does not correspond to a valid unicode scalar value",65 "unicode escape does not correspond to a unicode code point",
67 ),66 ),
68 .expected_lbrace => |bad_index| try writer.print(67 .expected_lbrace => |bad_index| try writer.print(
69 "expected '{{', found '{c}'",68 "expected '{{', found '{c}'",
...@@ -334,8 +333,8 @@ pub fn parseWrite(writer: *Writer, bytes: []const u8) Writer.Error!Result {...@@ -334,8 +333,8 @@ pub fn parseWrite(writer: *Writer, bytes: []const u8) Writer.Error!Result {
334 .success => |codepoint| {333 .success => |codepoint| {
335 if (bytes[escape_char_index] == 'u') {334 if (bytes[escape_char_index] == 'u') {
336 var buf: [4]u8 = undefined;335 var buf: [4]u8 = undefined;
337 const len = utf8Encode(codepoint, &buf) catch {336 const len = std.unicode.wtf8Encode(codepoint, &buf) catch |err| switch (err) {
338 return .{ .failure = .{ .invalid_unicode_codepoint = escape_char_index + 1 } };337 error.CodepointTooLarge => unreachable, // Checked in `parseEscapeSequence`.
339 };338 };
340 try writer.writeAll(buf[0..len]);339 try writer.writeAll(buf[0..len]);
341 } else {340 } else {
...@@ -385,4 +384,5 @@ test parseAlloc {...@@ -385,4 +384,5 @@ test parseAlloc {
385 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"foo\"")));384 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"foo\"")));
386 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"f\x6f\x6f\"")));385 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"f\x6f\x6f\"")));
387 try expect(eql(u8, "f💯", try parseAlloc(alloc, "\"f\u{1f4af}\"")));386 try expect(eql(u8, "f💯", try parseAlloc(alloc, "\"f\u{1f4af}\"")));
387 try expect(eql(u8, "\xed\xa0\x80", try parseAlloc(alloc, "\"\\u{d800}\"")));
388}388}