diff --git a/doc/langref.html.in b/doc/langref.html.in index 19f9d832298ea89faf70d0d8dc2033abc99f400f..090dd48232d136d5a830945942deeeddcc98c4a2 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -755,12 +755,21 @@
\u{NNNNNN}Note that the maximum valid Unicode scalar value is {#syntax#}0x10ffff{#endsyntax#}.
+
+ Note that the highest Unicode code point is {#syntax#}0x10ffff{#endsyntax#}. Character
+ literals are integers ({#syntax#}'A'{#endsyntax#} = {#syntax#}65{#endsyntax#} and
+ {#syntax#}'\u{10000}'{#endsyntax#} = {#syntax#}0x10000{#endsyntax#} = {#syntax#}65536{#endsyntax#}),
+ and \u escapes in string literals are encoded as UTF-8
+ ({#syntax#}"A"{#endsyntax#} = {#syntax#}"\u{41}"{#endsyntax#} = {#syntax#}"\x41"{#endsyntax#}
+ and {#syntax#}"\u{10000}"{#endsyntax#} = {#syntax#}"\xF0\x90\x80\x80"{#endsyntax#}).
+ \u escapes can be used for surrogate code points, which are not Unicode
+ scalar values or technically allowed in UTF-8 as defined by the Unicode standard, but can be useful in practice.
+
diff --git a/lib/compiler/Maker/Package/Manifest.zig b/lib/compiler/Maker/Package/Manifest.zig index 517c5fd3239077e2f5cf324d4667648e5417ebc3..7dd507a20c8ce9836b04ba2d8e4822e569ef5e91 100644 --- a/lib/compiler/Maker/Package/Manifest.zig +++ b/lib/compiler/Maker/Package/Manifest.zig @@ -544,7 +544,7 @@ const Parse = struct { try p.appendErrorOff( token, offset + @as(u32, @intCast(bad_index)), - "unicode escape does not correspond to a valid unicode scalar value", + "unicode escape does not correspond to a unicode code point", .{}, ); }, diff --git a/lib/std/zig/string_literal.zig b/lib/std/zig/string_literal.zig index 3699636c7965aa2c7cad9389bfc51d5eaa636faa..c51c2ab77c62faf84057d25f2739e37ece9fd0d6 100644 --- a/lib/std/zig/string_literal.zig +++ b/lib/std/zig/string_literal.zig @@ -1,6 +1,5 @@ const std = @import("../std.zig"); const assert = std.debug.assert; -const utf8Encode = std.unicode.utf8Encode; const Writer = std.Io.Writer; pub const ParseError = error{ @@ -63,7 +62,7 @@ pub const Error = union(enum) { .{self.raw_string[bad_index]}, ), .invalid_unicode_codepoint => try writer.writeAll( - "unicode escape does not correspond to a valid unicode scalar value", + "unicode escape does not correspond to a unicode code point", ), .expected_lbrace => |bad_index| try writer.print( "expected '{{', found '{c}'", @@ -334,8 +333,8 @@ pub fn parseWrite(writer: *Writer, bytes: []const u8) Writer.Error!Result { .success => |codepoint| { if (bytes[escape_char_index] == 'u') { var buf: [4]u8 = undefined; - const len = utf8Encode(codepoint, &buf) catch { - return .{ .failure = .{ .invalid_unicode_codepoint = escape_char_index + 1 } }; + const len = std.unicode.wtf8Encode(codepoint, &buf) catch |err| switch (err) { + error.CodepointTooLarge => unreachable, // Checked in `parseEscapeSequence`. }; try writer.writeAll(buf[0..len]); } else { @@ -383,6 +382,7 @@ test parseAlloc { try expect(eql(u8, "\x12foo", try parseAlloc(alloc, "\"\\x12foo\""))); try expect(eql(u8, "bytes\u{1234}foo", try parseAlloc(alloc, "\"bytes\\u{1234}foo\""))); try expect(eql(u8, "foo", try parseAlloc(alloc, "\"foo\""))); - try expect(eql(u8, "foo", try parseAlloc(alloc, "\"f\x6f\x6f\""))); - try expect(eql(u8, "f💯", try parseAlloc(alloc, "\"f\u{1f4af}\""))); + try expect(eql(u8, "foo", try parseAlloc(alloc, "\"f\\x6f\\x6f\""))); + try expect(eql(u8, "f💯", try parseAlloc(alloc, "\"f\\u{1f4af}\""))); + try expect(eql(u8, "\xed\xa0\x80", try parseAlloc(alloc, "\"\\u{d800}\""))); }