authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-05-24 21:51:58-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-05-24 21:51:58-04:00
logb74dda34b6a8b5f04d1865e2f23aab43229815f9
tree038e0da38722232944287cce84ab6100f3d56415
parent43085417bec447ab31f3454e180213f102885cc8

std.zig.tokenizer: support hex escape in char literals

2 files changed, 47 insertions(+), 0 deletions(-)

std/zig/parser_test.zig+13
...@@ -1,3 +1,16 @@...@@ -1,3 +1,16 @@
1test "zig fmt: float literal with exponent" {
2 try testCanonical(
3 \\test "aoeu" {
4 \\ switch (state) {
5 \\ TermState.Start => switch (c) {
6 \\ '\x1b' => state = TermState.Escape,
7 \\ else => try out.writeByte(c),
8 \\ },
9 \\ }
10 \\}
11 \\
12 );
13}
1test "zig fmt: float literal with exponent" {14test "zig fmt: float literal with exponent" {
2 try testCanonical(15 try testCanonical(
3 \\pub const f64_true_min = 4.94065645841246544177e-324;16 \\pub const f64_true_min = 4.94065645841246544177e-324;
std/zig/tokenizer.zig+34
...@@ -220,6 +220,8 @@ pub const Tokenizer = struct {...@@ -220,6 +220,8 @@ pub const Tokenizer = struct {
220 MultilineStringLiteralLineBackslash,220 MultilineStringLiteralLineBackslash,
221 CharLiteral,221 CharLiteral,
222 CharLiteralBackslash,222 CharLiteralBackslash,
223 CharLiteralEscape1,
224 CharLiteralEscape2,
223 CharLiteralEnd,225 CharLiteralEnd,
224 Backslash,226 Backslash,
225 Equal,227 Equal,
...@@ -612,9 +614,32 @@ pub const Tokenizer = struct {...@@ -612,9 +614,32 @@ pub const Tokenizer = struct {
612 result.id = Token.Id.Invalid;614 result.id = Token.Id.Invalid;
613 break;615 break;
614 },616 },
617 'x' => {
618 state = State.CharLiteralEscape1;
619 },
620 else => {
621 state = State.CharLiteralEnd;
622 },
623 },
624
625 State.CharLiteralEscape1 => switch (c) {
626 '0'...'9', 'a'...'z', 'A'...'F' => {
627 state = State.CharLiteralEscape2;
628 },
615 else => {629 else => {
630 result.id = Token.Id.Invalid;
631 break;
632 },
633 },
634
635 State.CharLiteralEscape2 => switch (c) {
636 '0'...'9', 'a'...'z', 'A'...'F' => {
616 state = State.CharLiteralEnd;637 state = State.CharLiteralEnd;
617 },638 },
639 else => {
640 result.id = Token.Id.Invalid;
641 break;
642 },
618 },643 },
619644
620 State.CharLiteralEnd => switch (c) {645 State.CharLiteralEnd => switch (c) {
...@@ -988,6 +1013,8 @@ pub const Tokenizer = struct {...@@ -988,6 +1013,8 @@ pub const Tokenizer = struct {
988 State.MultilineStringLiteralLineBackslash,1013 State.MultilineStringLiteralLineBackslash,
989 State.CharLiteral,1014 State.CharLiteral,
990 State.CharLiteralBackslash,1015 State.CharLiteralBackslash,
1016 State.CharLiteralEscape1,
1017 State.CharLiteralEscape2,
991 State.CharLiteralEnd,1018 State.CharLiteralEnd,
992 State.StringLiteralBackslash => {1019 State.StringLiteralBackslash => {
993 result.id = Token.Id.Invalid;1020 result.id = Token.Id.Invalid;
...@@ -1127,6 +1154,13 @@ test "tokenizer" {...@@ -1127,6 +1154,13 @@ test "tokenizer" {
1127 });1154 });
1128}1155}
11291156
1157test "tokenizer - char literal with hex escape" {
1158 testTokenize( \\'\x1b'
1159 , []Token.Id {
1160 Token.Id.CharLiteral,
1161 });
1162}
1163
1130test "tokenizer - float literal e exponent" {1164test "tokenizer - float literal e exponent" {
1131 testTokenize("a = 4.94065645841246544177e-324;\n", []Token.Id {1165 testTokenize("a = 4.94065645841246544177e-324;\n", []Token.Id {
1132 Token.Id.Identifier,1166 Token.Id.Identifier,