From 89953ec83d8afe4fed0fc9e3cdded09c7522bf86 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 Mar 2019 17:35:21 -0400 Subject: [PATCH] character literals: allow unicode escapes also make the documentation for character literals more clear. closes #2089 see #2097 --- doc/langref.html.in | 16 ++++++++++++--- src/all_types.hpp | 2 +- src/tokenizer.cpp | 17 ++++++++-------- src/tokenizer.hpp | 2 +- std/zig/parser_test.zig | 7 +++++++ std/zig/tokenizer.zig | 37 ++++++++++++++++++++--------------- test/stage1/behavior/misc.zig | 5 +++++ 7 files changed, 56 insertions(+), 30 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index c42eeaf88bb0d71d5989c9a2f835cd30f9b886bb..0de6650c56bee8e02b84d30506fdc01ef952e128 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -501,7 +501,16 @@ pub fn main() void { {#see_also|Optionals|undefined#} {#header_close#} - {#header_open|String Literals#} + {#header_open|String Literals and Character Literals#} +

+ String literals are UTF-8 encoded byte arrays. +

+

+ Character literals have type {#syntax#}comptime_int{#endsyntax#}, the same as + {#link|Integer Literals#}. All {#link|Escape Sequences#} are valid in both string literals + and character literals. Once https://github.com/ziglang/zig/issues/2097 is implemented, + character literals will be allowed to have a single UTF-8 encoded codepoint. +

{#code_begin|test#} const assert = @import("std").debug.assert; const mem = @import("std").mem; @@ -513,6 +522,7 @@ test "string literals" { assert(normal_bytes.len == 5); assert(normal_bytes[1] == 'e'); assert('e' == '\x65'); + assert('\U01f4a9' == 128169); assert(mem.eql(u8, "hello", "h\x65llo")); // A C string literal is a null terminated pointer. @@ -521,7 +531,7 @@ test "string literals" { assert(null_terminated_bytes[5] == 0); } {#code_end#} - {#see_also|Arrays|Zig Test#} + {#see_also|Arrays|Zig Test|Source Encoding#} {#header_open|Escape Sequences#}
@@ -8530,7 +8540,7 @@ pub fn main() void { ); } {#code_end#} - {#see_also|String Literals#} + {#see_also|String Literals and Character Literals#} {#header_close#} {#header_open|Import from C Header File#} diff --git a/src/all_types.hpp b/src/all_types.hpp index bd4b802e73548c0c80b1de271a57fdba4cb7f641..b49b42d4951a0ef26d8e375a4917c285528415d2 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -845,7 +845,7 @@ struct AstNodeStringLiteral { }; struct AstNodeCharLiteral { - uint8_t value; + uint32_t value; }; struct AstNodeFloatLiteral { diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index dc9d61aa2203c17f1c6ac823c1cbf8eddc08ceec..7d41343e3a1ecd9c398731df9beabf588ef1c151 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -1103,11 +1103,15 @@ void tokenize(Buf *buf, Tokenization *out) { if (t.char_code_index >= t.char_code_end) { if (t.unicode) { - if (t.char_code <= 0x7f) { + if (t.char_code > 0x10ffff) { + tokenize_error(&t, "unicode value out of range: %x", t.char_code); + } + if (t.cur_tok->id == TokenIdCharLiteral) { + t.cur_tok->data.char_lit.c = t.char_code; + t.state = TokenizeStateCharLiteralEnd; + } else if (t.char_code <= 0x7f) { // 00000000 00000000 00000000 0xxxxxxx handle_string_escape(&t, (uint8_t)t.char_code); - } else if (t.cur_tok->id == TokenIdCharLiteral) { - tokenize_error(&t, "unicode value too large for character literal: %x", t.char_code); } else if (t.char_code <= 0x7ff) { // 00000000 00000000 00000xxx xx000000 handle_string_escape(&t, (uint8_t)(0xc0 | (t.char_code >> 6))); @@ -1129,14 +1133,9 @@ void tokenize(Buf *buf, Tokenization *out) { handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 6) & 0x3f))); // 00000000 00000000 00000000 00xxxxxx handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f))); - } else { - tokenize_error(&t, "unicode value out of range: %x", t.char_code); } } else { - if (t.cur_tok->id == TokenIdCharLiteral && t.char_code > UINT8_MAX) { - tokenize_error(&t, "value too large for character literal: '%x'", - t.char_code); - } + assert(t.char_code <= 255); handle_string_escape(&t, (uint8_t)t.char_code); } } diff --git a/src/tokenizer.hpp b/src/tokenizer.hpp index 62117b577938e9e51071125a4b1cd183b56adb4e..13ab0352d924b5557142e046c1ecca42f535a31f 100644 --- a/src/tokenizer.hpp +++ b/src/tokenizer.hpp @@ -148,7 +148,7 @@ struct TokenStrLit { }; struct TokenCharLit { - uint8_t c; + uint32_t c; }; struct Token { diff --git a/std/zig/parser_test.zig b/std/zig/parser_test.zig index 508813759c367dcc7b0b00347a878d3dc78f0798..8b9c0c2d64cfe2cf855a2f6defae82d94edfaf81 100644 --- a/std/zig/parser_test.zig +++ b/std/zig/parser_test.zig @@ -1,3 +1,10 @@ +test "zig fmt: character literal larger than u8" { + try testCanonical( + \\const x = '\U01f4a9'; + \\ + ); +} + test "zig fmt: infix operator and then multiline string literal" { try testCanonical( \\const x = "" ++ diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 2159371ccff386898723452975688bc334f9a6c5..19d64514a1cd6a9388c091347f27b2436aef43f4 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -236,8 +236,7 @@ pub const Tokenizer = struct { MultilineStringLiteralLine, CharLiteral, CharLiteralBackslash, - CharLiteralEscape1, - CharLiteralEscape2, + CharLiteralHexEscape, CharLiteralEnd, Backslash, Equal, @@ -293,6 +292,8 @@ pub const Tokenizer = struct { .start = self.index, .end = undefined, }; + var seen_escape_digits: usize = undefined; + var expected_escape_digits: usize = undefined; while (self.index < self.buffer.len) : (self.index += 1) { const c = self.buffer[self.index]; switch (state) { @@ -658,26 +659,31 @@ pub const Tokenizer = struct { break; }, 'x' => { - state = State.CharLiteralEscape1; + state = State.CharLiteralHexEscape; + seen_escape_digits = 0; + expected_escape_digits = 2; }, - else => { - state = State.CharLiteralEnd; + 'u' => { + state = State.CharLiteralHexEscape; + seen_escape_digits = 0; + expected_escape_digits = 4; }, - }, - - State.CharLiteralEscape1 => switch (c) { - '0'...'9', 'a'...'z', 'A'...'F' => { - state = State.CharLiteralEscape2; + 'U' => { + state = State.CharLiteralHexEscape; + seen_escape_digits = 0; + expected_escape_digits = 6; }, else => { - result.id = Token.Id.Invalid; - break; + state = State.CharLiteralEnd; }, }, - State.CharLiteralEscape2 => switch (c) { + State.CharLiteralHexEscape => switch (c) { '0'...'9', 'a'...'z', 'A'...'F' => { - state = State.CharLiteralEnd; + seen_escape_digits += 1; + if (seen_escape_digits == expected_escape_digits) { + state = State.CharLiteralEnd; + } }, else => { result.id = Token.Id.Invalid; @@ -1045,8 +1051,7 @@ pub const Tokenizer = struct { State.Backslash, State.CharLiteral, State.CharLiteralBackslash, - State.CharLiteralEscape1, - State.CharLiteralEscape2, + State.CharLiteralHexEscape, State.CharLiteralEnd, State.StringLiteralBackslash, State.LBracketStar, diff --git a/test/stage1/behavior/misc.zig b/test/stage1/behavior/misc.zig index 36246162f52dce56e269da81127eb60565c2a1fd..fd407821e678bd9f523a0daa8fc9b4347d23f8d8 100644 --- a/test/stage1/behavior/misc.zig +++ b/test/stage1/behavior/misc.zig @@ -699,3 +699,8 @@ test "thread local variable" { S.t += 1; expect(S.t == 1235); } + +test "unicode escape in character literal" { + var a: u24 = '\U01f4a9'; + expect(a == 128169); +} -- 2.54.0