authorgravatar for oa223@cam.ac.ukOeistein Andersen <oa223@cam.ac.uk> 2025-03-05 12:06:23+00:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-09-26 17:29:55-07:00
loge1f7e3563fa84b08e707a3dcea610560d179c63d
tree1f1a84d58a8ca26432ff379f3467a450a4544764
parent5b9147ed59b22edf7f891cf30e8b77223121da4f

Allow \u escape in string literal to encode surrogate code point.

3 files changed, 16 insertions(+), 7 deletions(-)

doc/langref.html.in+11-2
......@@ -754,12 +754,21 @@
754754 </tr>
755755 <tr>
756756 <th scope="row"><code>\u{NNNNNN}</code></th>
757 <td>hexadecimal Unicode scalar value UTF-8 encoded (1 or more digits)</td>
757 <td>hexadecimal Unicode code point (1 or more digits)</td>
758758 </tr>
759759 </tbody>
760760 </table>
761761 </div>
762 <p>Note that the maximum valid Unicode scalar value is {#syntax#}0x10ffff{#endsyntax#}.</p>
762 <p>
763 Note that the highest Unicode code point is {#syntax#}0x10ffff{#endsyntax#}. Character
764 literals are integers ({#syntax#}'A'{#endsyntax#} = {#syntax#}65{#endsyntax#} and
765 {#syntax#}'\u{10000}'{#endsyntax#} = {#syntax#}0x10000{#endsyntax#} = {#syntax#}65536{#endsyntax#}),
766 and <code>\u</code> escapes in string literals are encoded as UTF-8
767 ({#syntax#}"A"{#endsyntax#} = {#syntax#}"\u{41}"{#endsyntax#} = {#syntax#}"\x41"{#endsyntax#}
768 and {#syntax#}"\u{10000}"{#endsyntax#} = {#syntax#}"\xF0\x90\x80\x80"{#endsyntax#}).
769 <code>\u</code> escapes can be used for surrogate code points, which are not Unicode
770 scalar values or technically allowed in UTF-8 as defined by the Unicode standard, but can be useful in practice.
771 </p>
763772 {#header_close#}
764773 {#header_open|Multiline String Literals#}
765774 <p>
lib/compiler/Maker/Package/Manifest.zig+1-1
......@@ -508,7 +508,7 @@ const Parse = struct {
508508 try p.appendErrorOff(
509509 token,
510510 offset + @as(u32, @intCast(bad_index)),
511 "unicode escape does not correspond to a valid unicode scalar value",
511 "unicode escape does not correspond to a unicode code point",
512512 .{},
513513 );
514514 },
lib/std/zig/string_literal.zig+4-4
......@@ -1,6 +1,5 @@
11const std = @import("../std.zig");
22const assert = std.debug.assert;
3const utf8Encode = std.unicode.utf8Encode;
43const Writer = std.Io.Writer;
54
65pub const ParseError = error{
......@@ -63,7 +62,7 @@ pub const Error = union(enum) {
6362 .{self.raw_string[bad_index]},
6463 ),
6564 .invalid_unicode_codepoint => try writer.writeAll(
66 "unicode escape does not correspond to a valid unicode scalar value",
65 "unicode escape does not correspond to a unicode code point",
6766 ),
6867 .expected_lbrace => |bad_index| try writer.print(
6968 "expected '{{', found '{c}'",
......@@ -334,8 +333,8 @@ pub fn parseWrite(writer: *Writer, bytes: []const u8) Writer.Error!Result {
334333 .success => |codepoint| {
335334 if (bytes[escape_char_index] == 'u') {
336335 var buf: [4]u8 = undefined;
337 const len = utf8Encode(codepoint, &buf) catch {
338 return .{ .failure = .{ .invalid_unicode_codepoint = escape_char_index + 1 } };
336 const len = std.unicode.wtf8Encode(codepoint, &buf) catch |err| switch (err) {
337 error.CodepointTooLarge => unreachable, // Checked in `parseEscapeSequence`.
339338 };
340339 try writer.writeAll(buf[0..len]);
341340 } else {
......@@ -385,4 +384,5 @@ test parseAlloc {
385384 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"foo\"")));
386385 try expect(eql(u8, "foo", try parseAlloc(alloc, "\"f\x6f\x6f\"")));
387386 try expect(eql(u8, "f💯", try parseAlloc(alloc, "\"f\u{1f4af}\"")));
387 try expect(eql(u8, "\xed\xa0\x80", try parseAlloc(alloc, "\"\\u{d800}\"")));
388388}