authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2021-02-24 08:26:13-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2021-02-24 08:26:13-05:00
log8b9434871ea437840d25f073b945466359f402f9
tree8ae023c3dd5f8821431156e5e0d155a1dd449953
parentd9e46dceeca3f66b87e6b2e36415417495d2d2a0
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Avoid concept of a "Unicode character" in documentation and error messages (#8059)

4 files changed, 28 insertions(+), 14 deletions(-)

doc/langref.html.in+22-8
......@@ -310,7 +310,7 @@ pub fn main() !void {
310310 <p>
311311 The two arguments passed to the <code>stdout.print()</code> function, <code>"Hello, {s}!\n"</code>
312312 and <code>.{"world"}</code>, are evaluated at {#link|compile-time|comptime#}. The code sample is
313 purposely written to show how to perform {#link|string|String Literals and Character Literals#}
313 purposely written to show how to perform {#link|string|String Literals and Unicode Code Point Literals#}
314314 substitution in the <code>print</code> function. The curly-braces inside of the first argument
315315 are substituted with the compile-time known value inside of the second argument
316316 (known as an {#link|anonymous struct literal|Anonymous Struct Literals#}). The <code>\n</code>
......@@ -682,18 +682,31 @@ pub fn main() void {
682682 </div>
683683 {#see_also|Optionals|undefined#}
684684 {#header_close#}
685 {#header_open|String Literals and Character Literals#}
685 {#header_open|String Literals and Unicode Code Point Literals#}
686686 <p>
687 String literals are single-item constant {#link|Pointers#} to null-terminated UTF-8 encoded byte arrays.
687 String literals are single-item constant {#link|Pointers#} to null-terminated byte arrays.
688688 The type of string literals encodes both the length, and the fact that they are null-terminated,
689689 and thus they can be {#link|coerced|Type Coercion#} to both {#link|Slices#} and
690690 {#link|Null-Terminated Pointers|Sentinel-Terminated Pointers#}.
691691 Dereferencing string literals converts them to {#link|Arrays#}.
692692 </p>
693693 <p>
694 Character literals have type {#syntax#}comptime_int{#endsyntax#}, the same as
694 The encoding of a string in Zig is de-facto assumed to be UTF-8.
695 Because Zig source code is {#link|UTF-8 encoded|Source Encoding#}, any non-ASCII bytes appearing within a string literal
696 in source code carry their UTF-8 meaning into the content of the string in the Zig program;
697 the bytes are not modified by the compiler.
698 However, it is possible to embbed non-UTF-8 bytes into a string literal using <code>\xNN</code> notation.
699 </p>
700 <p>
701 Unicode code point literals have type {#syntax#}comptime_int{#endsyntax#}, the same as
695702 {#link|Integer Literals#}. All {#link|Escape Sequences#} are valid in both string literals
696 and character literals.
703 and Unicode code point literals.
704 </p>
705 <p>
706 In many other programming languages, a Unicode code point literal is called a "character literal".
707 However, there is <a href="https://unicode.org/glossary">no precise technical definition of a "character"</a>
708 in recent versions of the Unicode specification (as of Unicode 13.0).
709 In Zig, a Unicode code point literal corresponds to the Unicode definition of a code point.
697710 </p>
698711 {#code_begin|test#}
699712const expect = @import("std").testing.expect;
......@@ -709,6 +722,7 @@ test "string literals" {
709722 expect('\u{1f4a9}' == 128169);
710723 expect('💯' == 128175);
711724 expect(mem.eql(u8, "hello", "h\x65llo"));
725 expect("\xff"[0] == 0xff); // non-UTF-8 strings are possible with \xNN notation.
712726}
713727 {#code_end#}
714728 {#see_also|Arrays|Zig Test|Source Encoding#}
......@@ -749,11 +763,11 @@ test "string literals" {
749763 </tr>
750764 <tr>
751765 <td><code>\xNN</code></td>
752 <td>hexadecimal 8-bit character code (2 digits)</td>
766 <td>hexadecimal 8-bit byte value (2 digits)</td>
753767 </tr>
754768 <tr>
755769 <td><code>\u{NNNNNN}</code></td>
756 <td>hexadecimal Unicode character code UTF-8 encoded (1 or more digits)</td>
770 <td>hexadecimal Unicode code point UTF-8 encoded (1 or more digits)</td>
757771 </tr>
758772 </table>
759773 </div>
......@@ -7414,7 +7428,7 @@ test "main" {
74147428 This function returns a compile time constant pointer to null-terminated,
74157429 fixed-size array with length equal to the byte count of the file given by
74167430 {#syntax#}path{#endsyntax#}. The contents of the array are the contents of the file.
7417 This is equivalent to a {#link|string literal|String Literals and Character Literals#}
7431 This is equivalent to a {#link|string literal|String Literals and Unicode Code Point Literals#}
74187432 with the file contents.
74197433 </p>
74207434 <p>
lib/std/zig/parser_test.zig+1-1
......@@ -680,7 +680,7 @@ test "zig fmt: enum literal inside array literal" {
680680 );
681681}
682682
683test "zig fmt: character literal larger than u8" {
683test "zig fmt: Unicode code point literal larger than u8" {
684684 try testCanonical(
685685 \\const x = '\u{01f4a9}';
686686 \\
lib/std/zig/tokenizer.zig+3-3
......@@ -1513,7 +1513,7 @@ test "tokenizer - unknown length pointer and then c pointer" {
15131513 });
15141514}
15151515
1516test "tokenizer - char literal with hex escape" {
1516test "tokenizer - code point literal with hex escape" {
15171517 testTokenize(
15181518 \\'\x1b'
15191519 , &[_]Token.Id{.CharLiteral});
......@@ -1522,7 +1522,7 @@ test "tokenizer - char literal with hex escape" {
15221522 , &[_]Token.Id{ .Invalid, .Invalid });
15231523}
15241524
1525test "tokenizer - char literal with unicode escapes" {
1525test "tokenizer - code point literal with unicode escapes" {
15261526 // Valid unicode escapes
15271527 testTokenize(
15281528 \\'\u{3}'
......@@ -1572,7 +1572,7 @@ test "tokenizer - char literal with unicode escapes" {
15721572 , &[_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });
15731573}
15741574
1575test "tokenizer - char literal with unicode code point" {
1575test "tokenizer - code point literal with unicode code point" {
15761576 testTokenize(
15771577 \\'💩'
15781578 , &[_]Token.Id{.CharLiteral});
src/stage1/tokenizer.cpp+2-2
......@@ -1447,7 +1447,7 @@ void tokenize(Buf *buf, Tokenization *out) {
14471447 tokenize_error(&t, "unterminated string");
14481448 break;
14491449 } else if (t.cur_tok->id == TokenIdCharLiteral) {
1450 tokenize_error(&t, "unterminated character literal");
1450 tokenize_error(&t, "unterminated Unicode code point literal");
14511451 break;
14521452 } else {
14531453 zig_unreachable();
......@@ -1456,7 +1456,7 @@ void tokenize(Buf *buf, Tokenization *out) {
14561456 case TokenizeStateCharLiteral:
14571457 case TokenizeStateCharLiteralEnd:
14581458 case TokenizeStateCharLiteralUnicode:
1459 tokenize_error(&t, "unterminated character literal");
1459 tokenize_error(&t, "unterminated Unicode code point literal");
14601460 break;
14611461 case TokenizeStateSymbol:
14621462 case TokenizeStateZero: