Commit 89953ec83d8afe4fed0fc9e3cdded09c7522bf86 | zig

author	Andrew Kelley <andrew@ziglang.org> 2019-03-23 17:35:21-04:00
committer	Andrew Kelley <andrew@ziglang.org> 2019-03-23 17:35:21-04:00
log	89953ec83d8afe4fed0fc9e3cdded09c7522bf86
tree	42f76e6da37b08e0022af060545c5e5d3f6bd476
parent	55cb9ef138c7cf0a23e7f852a82884612a3ca663
signature	Commit is signed but in an unrecognized format.

character literals: allow unicode escapes

also make the documentation for character literals more clear.
closes #2089

see #2097

7 files changed, 56 insertions(+), 30 deletions(-)

doc
- langref.html.in
src
std
- zig
  - parser_test.zig
  - tokenizer.zig
test
- stage1
  - behavior
    misc.zig

doc/langref.html.in+13-3

...	...	@@ -501,7 +501,16 @@ pub fn main() void {
501	501	</div>
502	502	{#see_also\|Optionals\|undefined#}
503	503	{#header_close#}
504		{#header_open\|String Literals#}
	504	{#header_open\|String Literals and Character Literals#}
	505	<p>
	506	String literals are UTF-8 encoded byte arrays.
	507	</p>
	508	<p>
	509	Character literals have type {#syntax#}comptime_int{#endsyntax#}, the same as
	510	{#link\|Integer Literals#}. All {#link\|Escape Sequences#} are valid in both string literals
	511	and character literals. Once https://github.com/ziglang/zig/issues/2097 is implemented,
	512	character literals will be allowed to have a single UTF-8 encoded codepoint.
	513	</p>
505	514	{#code_begin\|test#}
506	515	const assert = @import("std").debug.assert;
507	516	const mem = @import("std").mem;
...	...	@@ -513,6 +522,7 @@ test "string literals" {
513	522	assert(normal_bytes.len == 5);
514	523	assert(normal_bytes[1] == 'e');
515	524	assert('e' == '\x65');
	525	assert('\U01f4a9' == 128169);
516	526	assert(mem.eql(u8, "hello", "h\x65llo"));
517	527
518	528	// A C string literal is a null terminated pointer.
...	...	@@ -521,7 +531,7 @@ test "string literals" {
521	531	assert(null_terminated_bytes[5] == 0);
522	532	}
523	533	{#code_end#}
524		{#see_also\|Arrays\|Zig Test#}
	534	{#see_also\|Arrays\|Zig Test\|Source Encoding#}
525	535	{#header_open\|Escape Sequences#}
526	536	<div class="table-wrapper">
527	537	<table>
...	...	@@ -8530,7 +8540,7 @@ pub fn main() void {
8530	8540	);
8531	8541	}
8532	8542	{#code_end#}
8533		{#see_also\|String Literals#}
	8543	{#see_also\|String Literals and Character Literals#}
8534	8544	{#header_close#}
8535	8545
8536	8546	{#header_open\|Import from C Header File#}

src/all_types.hpp+1-1

...	...	@@ -845,7 +845,7 @@ struct AstNodeStringLiteral {
845	845	};
846	846
847	847	struct AstNodeCharLiteral {
848		uint8_t value;
	848	uint32_t value;
849	849	};
850	850
851	851	struct AstNodeFloatLiteral {

src/tokenizer.cpp+8-9

...	...	@@ -1103,11 +1103,15 @@ void tokenize(Buf buf, Tokenization out) {
1103	1103
1104	1104	if (t.char_code_index >= t.char_code_end) {
1105	1105	if (t.unicode) {
1106		if (t.char_code <= 0x7f) {
	1106	if (t.char_code > 0x10ffff) {
	1107	tokenize_error(&t, "unicode value out of range: %x", t.char_code);
	1108	}
	1109	if (t.cur_tok->id == TokenIdCharLiteral) {
	1110	t.cur_tok->data.char_lit.c = t.char_code;
	1111	t.state = TokenizeStateCharLiteralEnd;
	1112	} else if (t.char_code <= 0x7f) {
1107	1113	// 00000000 00000000 00000000 0xxxxxxx
1108	1114	handle_string_escape(&t, (uint8_t)t.char_code);
1109		} else if (t.cur_tok->id == TokenIdCharLiteral) {
1110		tokenize_error(&t, "unicode value too large for character literal: %x", t.char_code);
1111	1115	} else if (t.char_code <= 0x7ff) {
1112	1116	// 00000000 00000000 00000xxx xx000000
1113	1117	handle_string_escape(&t, (uint8_t)(0xc0 \| (t.char_code >> 6)));
...	...	@@ -1129,14 +1133,9 @@ void tokenize(Buf buf, Tokenization out) {
1129	1133	handle_string_escape(&t, (uint8_t)(0x80 \| ((t.char_code >> 6) & 0x3f)));
1130	1134	// 00000000 00000000 00000000 00xxxxxx
1131	1135	handle_string_escape(&t, (uint8_t)(0x80 \| (t.char_code & 0x3f)));
1132		} else {
1133		tokenize_error(&t, "unicode value out of range: %x", t.char_code);
1134	1136	}
1135	1137	} else {
1136		if (t.cur_tok->id == TokenIdCharLiteral && t.char_code > UINT8_MAX) {
1137		tokenize_error(&t, "value too large for character literal: '%x'",
1138		t.char_code);
1139		}
	1138	assert(t.char_code <= 255);
1140	1139	handle_string_escape(&t, (uint8_t)t.char_code);
1141	1140	}
1142	1141	}

src/tokenizer.hpp+1-1

...	...	@@ -148,7 +148,7 @@ struct TokenStrLit {
148	148	};
149	149
150	150	struct TokenCharLit {
151		uint8_t c;
	151	uint32_t c;
152	152	};
153	153
154	154	struct Token {

std/zig/parser_test.zig+7

...	...	@@ -1,3 +1,10 @@
	1	test "zig fmt: character literal larger than u8" {
	2	try testCanonical(
	3	\\const x = '\U01f4a9';
	4	\\
	5	);
	6	}
	7
1	8	test "zig fmt: infix operator and then multiline string literal" {
2	9	try testCanonical(
3	10	\\const x = "" ++

std/zig/tokenizer.zig+21-16

...	...	@@ -236,8 +236,7 @@ pub const Tokenizer = struct {
236	236	MultilineStringLiteralLine,
237	237	CharLiteral,
238	238	CharLiteralBackslash,
239		CharLiteralEscape1,
240		CharLiteralEscape2,
	239	CharLiteralHexEscape,
241	240	CharLiteralEnd,
242	241	Backslash,
243	242	Equal,
...	...	@@ -293,6 +292,8 @@ pub const Tokenizer = struct {
293	292	.start = self.index,
294	293	.end = undefined,
295	294	};
	295	var seen_escape_digits: usize = undefined;
	296	var expected_escape_digits: usize = undefined;
296	297	while (self.index < self.buffer.len) : (self.index += 1) {
297	298	const c = self.buffer[self.index];
298	299	switch (state) {
...	...	@@ -658,26 +659,31 @@ pub const Tokenizer = struct {
658	659	break;
659	660	},
660	661	'x' => {
661		state = State.CharLiteralEscape1;
	662	state = State.CharLiteralHexEscape;
	663	seen_escape_digits = 0;
	664	expected_escape_digits = 2;
662	665	},
663		else => {
664		state = State.CharLiteralEnd;
	666	'u' => {
	667	state = State.CharLiteralHexEscape;
	668	seen_escape_digits = 0;
	669	expected_escape_digits = 4;
665	670	},
666		},
667
668		State.CharLiteralEscape1 => switch (c) {
669		'0'...'9', 'a'...'z', 'A'...'F' => {
670		state = State.CharLiteralEscape2;
	671	'U' => {
	672	state = State.CharLiteralHexEscape;
	673	seen_escape_digits = 0;
	674	expected_escape_digits = 6;
671	675	},
672	676	else => {
673		result.id = Token.Id.Invalid;
674		break;
	677	state = State.CharLiteralEnd;
675	678	},
676	679	},
677	680
678		State.CharLiteralEscape2 => switch (c) {
	681	State.CharLiteralHexEscape => switch (c) {
679	682	'0'...'9', 'a'...'z', 'A'...'F' => {
680		state = State.CharLiteralEnd;
	683	seen_escape_digits += 1;
	684	if (seen_escape_digits == expected_escape_digits) {
	685	state = State.CharLiteralEnd;
	686	}
681	687	},
682	688	else => {
683	689	result.id = Token.Id.Invalid;
...	...	@@ -1045,8 +1051,7 @@ pub const Tokenizer = struct {
1045	1051	State.Backslash,
1046	1052	State.CharLiteral,
1047	1053	State.CharLiteralBackslash,
1048		State.CharLiteralEscape1,
1049		State.CharLiteralEscape2,
	1054	State.CharLiteralHexEscape,
1050	1055	State.CharLiteralEnd,
1051	1056	State.StringLiteralBackslash,
1052	1057	State.LBracketStar,

test/stage1/behavior/misc.zig+5

...	...	@@ -699,3 +699,8 @@ test "thread local variable" {
699	699	S.t += 1;
700	700	expect(S.t == 1235);
701	701	}
	702
	703	test "unicode escape in character literal" {
	704	var a: u24 = '\U01f4a9';
	705	expect(a == 128169);
	706	}