authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2021-02-24 08:26:13-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2021-02-24 08:26:13-05:00
log8b9434871ea437840d25f073b945466359f402f9
tree8ae023c3dd5f8821431156e5e0d155a1dd449953
parentd9e46dceeca3f66b87e6b2e36415417495d2d2a0
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Avoid concept of a "Unicode character" in documentation and error messages (#8059)

4 files changed, 28 insertions(+), 14 deletions(-)

doc/langref.html.in+22-8
...@@ -310,7 +310,7 @@ pub fn main() !void {...@@ -310,7 +310,7 @@ pub fn main() !void {
310 <p>310 <p>
311 The two arguments passed to the <code>stdout.print()</code> function, <code>"Hello, {s}!\n"</code>311 The two arguments passed to the <code>stdout.print()</code> function, <code>"Hello, {s}!\n"</code>
312 and <code>.{"world"}</code>, are evaluated at {#link|compile-time|comptime#}. The code sample is312 and <code>.{"world"}</code>, are evaluated at {#link|compile-time|comptime#}. The code sample is
313 purposely written to show how to perform {#link|string|String Literals and Character Literals#}313 purposely written to show how to perform {#link|string|String Literals and Unicode Code Point Literals#}
314 substitution in the <code>print</code> function. The curly-braces inside of the first argument314 substitution in the <code>print</code> function. The curly-braces inside of the first argument
315 are substituted with the compile-time known value inside of the second argument315 are substituted with the compile-time known value inside of the second argument
316 (known as an {#link|anonymous struct literal|Anonymous Struct Literals#}). The <code>\n</code>316 (known as an {#link|anonymous struct literal|Anonymous Struct Literals#}). The <code>\n</code>
...@@ -682,18 +682,31 @@ pub fn main() void {...@@ -682,18 +682,31 @@ pub fn main() void {
682 </div>682 </div>
683 {#see_also|Optionals|undefined#}683 {#see_also|Optionals|undefined#}
684 {#header_close#}684 {#header_close#}
685 {#header_open|String Literals and Character Literals#}685 {#header_open|String Literals and Unicode Code Point Literals#}
686 <p>686 <p>
687 String literals are single-item constant {#link|Pointers#} to null-terminated UTF-8 encoded byte arrays.687 String literals are single-item constant {#link|Pointers#} to null-terminated byte arrays.
688 The type of string literals encodes both the length, and the fact that they are null-terminated,688 The type of string literals encodes both the length, and the fact that they are null-terminated,
689 and thus they can be {#link|coerced|Type Coercion#} to both {#link|Slices#} and689 and thus they can be {#link|coerced|Type Coercion#} to both {#link|Slices#} and
690 {#link|Null-Terminated Pointers|Sentinel-Terminated Pointers#}.690 {#link|Null-Terminated Pointers|Sentinel-Terminated Pointers#}.
691 Dereferencing string literals converts them to {#link|Arrays#}.691 Dereferencing string literals converts them to {#link|Arrays#}.
692 </p>692 </p>
693 <p>693 <p>
694 Character literals have type {#syntax#}comptime_int{#endsyntax#}, the same as694 The encoding of a string in Zig is de-facto assumed to be UTF-8.
695 Because Zig source code is {#link|UTF-8 encoded|Source Encoding#}, any non-ASCII bytes appearing within a string literal
696 in source code carry their UTF-8 meaning into the content of the string in the Zig program;
697 the bytes are not modified by the compiler.
698 However, it is possible to embbed non-UTF-8 bytes into a string literal using <code>\xNN</code> notation.
699 </p>
700 <p>
701 Unicode code point literals have type {#syntax#}comptime_int{#endsyntax#}, the same as
695 {#link|Integer Literals#}. All {#link|Escape Sequences#} are valid in both string literals702 {#link|Integer Literals#}. All {#link|Escape Sequences#} are valid in both string literals
696 and character literals.703 and Unicode code point literals.
704 </p>
705 <p>
706 In many other programming languages, a Unicode code point literal is called a "character literal".
707 However, there is <a href="https://unicode.org/glossary">no precise technical definition of a "character"</a>
708 in recent versions of the Unicode specification (as of Unicode 13.0).
709 In Zig, a Unicode code point literal corresponds to the Unicode definition of a code point.
697 </p>710 </p>
698 {#code_begin|test#}711 {#code_begin|test#}
699const expect = @import("std").testing.expect;712const expect = @import("std").testing.expect;
...@@ -709,6 +722,7 @@ test "string literals" {...@@ -709,6 +722,7 @@ test "string literals" {
709 expect('\u{1f4a9}' == 128169);722 expect('\u{1f4a9}' == 128169);
710 expect('💯' == 128175);723 expect('💯' == 128175);
711 expect(mem.eql(u8, "hello", "h\x65llo"));724 expect(mem.eql(u8, "hello", "h\x65llo"));
725 expect("\xff"[0] == 0xff); // non-UTF-8 strings are possible with \xNN notation.
712}726}
713 {#code_end#}727 {#code_end#}
714 {#see_also|Arrays|Zig Test|Source Encoding#}728 {#see_also|Arrays|Zig Test|Source Encoding#}
...@@ -749,11 +763,11 @@ test "string literals" {...@@ -749,11 +763,11 @@ test "string literals" {
749 </tr>763 </tr>
750 <tr>764 <tr>
751 <td><code>\xNN</code></td>765 <td><code>\xNN</code></td>
752 <td>hexadecimal 8-bit character code (2 digits)</td>766 <td>hexadecimal 8-bit byte value (2 digits)</td>
753 </tr>767 </tr>
754 <tr>768 <tr>
755 <td><code>\u{NNNNNN}</code></td>769 <td><code>\u{NNNNNN}</code></td>
756 <td>hexadecimal Unicode character code UTF-8 encoded (1 or more digits)</td>770 <td>hexadecimal Unicode code point UTF-8 encoded (1 or more digits)</td>
757 </tr>771 </tr>
758 </table>772 </table>
759 </div>773 </div>
...@@ -7414,7 +7428,7 @@ test "main" {...@@ -7414,7 +7428,7 @@ test "main" {
7414 This function returns a compile time constant pointer to null-terminated,7428 This function returns a compile time constant pointer to null-terminated,
7415 fixed-size array with length equal to the byte count of the file given by7429 fixed-size array with length equal to the byte count of the file given by
7416 {#syntax#}path{#endsyntax#}. The contents of the array are the contents of the file.7430 {#syntax#}path{#endsyntax#}. The contents of the array are the contents of the file.
7417 This is equivalent to a {#link|string literal|String Literals and Character Literals#}7431 This is equivalent to a {#link|string literal|String Literals and Unicode Code Point Literals#}
7418 with the file contents.7432 with the file contents.
7419 </p>7433 </p>
7420 <p>7434 <p>
lib/std/zig/parser_test.zig+1-1
...@@ -680,7 +680,7 @@ test "zig fmt: enum literal inside array literal" {...@@ -680,7 +680,7 @@ test "zig fmt: enum literal inside array literal" {
680 );680 );
681}681}
682682
683test "zig fmt: character literal larger than u8" {683test "zig fmt: Unicode code point literal larger than u8" {
684 try testCanonical(684 try testCanonical(
685 \\const x = '\u{01f4a9}';685 \\const x = '\u{01f4a9}';
686 \\686 \\
lib/std/zig/tokenizer.zig+3-3
...@@ -1513,7 +1513,7 @@ test "tokenizer - unknown length pointer and then c pointer" {...@@ -1513,7 +1513,7 @@ test "tokenizer - unknown length pointer and then c pointer" {
1513 });1513 });
1514}1514}
15151515
1516test "tokenizer - char literal with hex escape" {1516test "tokenizer - code point literal with hex escape" {
1517 testTokenize(1517 testTokenize(
1518 \\'\x1b'1518 \\'\x1b'
1519 , &[_]Token.Id{.CharLiteral});1519 , &[_]Token.Id{.CharLiteral});
...@@ -1522,7 +1522,7 @@ test "tokenizer - char literal with hex escape" {...@@ -1522,7 +1522,7 @@ test "tokenizer - char literal with hex escape" {
1522 , &[_]Token.Id{ .Invalid, .Invalid });1522 , &[_]Token.Id{ .Invalid, .Invalid });
1523}1523}
15241524
1525test "tokenizer - char literal with unicode escapes" {1525test "tokenizer - code point literal with unicode escapes" {
1526 // Valid unicode escapes1526 // Valid unicode escapes
1527 testTokenize(1527 testTokenize(
1528 \\'\u{3}'1528 \\'\u{3}'
...@@ -1572,7 +1572,7 @@ test "tokenizer - char literal with unicode escapes" {...@@ -1572,7 +1572,7 @@ test "tokenizer - char literal with unicode escapes" {
1572 , &[_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });1572 , &[_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });
1573}1573}
15741574
1575test "tokenizer - char literal with unicode code point" {1575test "tokenizer - code point literal with unicode code point" {
1576 testTokenize(1576 testTokenize(
1577 \\'💩'1577 \\'💩'
1578 , &[_]Token.Id{.CharLiteral});1578 , &[_]Token.Id{.CharLiteral});
src/stage1/tokenizer.cpp+2-2
...@@ -1447,7 +1447,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1447,7 +1447,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1447 tokenize_error(&t, "unterminated string");1447 tokenize_error(&t, "unterminated string");
1448 break;1448 break;
1449 } else if (t.cur_tok->id == TokenIdCharLiteral) {1449 } else if (t.cur_tok->id == TokenIdCharLiteral) {
1450 tokenize_error(&t, "unterminated character literal");1450 tokenize_error(&t, "unterminated Unicode code point literal");
1451 break;1451 break;
1452 } else {1452 } else {
1453 zig_unreachable();1453 zig_unreachable();
...@@ -1456,7 +1456,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1456,7 +1456,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1456 case TokenizeStateCharLiteral:1456 case TokenizeStateCharLiteral:
1457 case TokenizeStateCharLiteralEnd:1457 case TokenizeStateCharLiteralEnd:
1458 case TokenizeStateCharLiteralUnicode:1458 case TokenizeStateCharLiteralUnicode:
1459 tokenize_error(&t, "unterminated character literal");1459 tokenize_error(&t, "unterminated Unicode code point literal");
1460 break;1460 break;
1461 case TokenizeStateSymbol:1461 case TokenizeStateSymbol:
1462 case TokenizeStateZero:1462 case TokenizeStateZero: