authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-05-24 21:51:58-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-05-24 21:51:58-04:00
logb74dda34b6a8b5f04d1865e2f23aab43229815f9
tree038e0da38722232944287cce84ab6100f3d56415
parent43085417bec447ab31f3454e180213f102885cc8

std.zig.tokenizer: support hex escape in char literals

2 files changed, 47 insertions(+), 0 deletions(-)

std/zig/parser_test.zig+13
......@@ -1,3 +1,16 @@
1test "zig fmt: float literal with exponent" {
2 try testCanonical(
3 \\test "aoeu" {
4 \\ switch (state) {
5 \\ TermState.Start => switch (c) {
6 \\ '\x1b' => state = TermState.Escape,
7 \\ else => try out.writeByte(c),
8 \\ },
9 \\ }
10 \\}
11 \\
12 );
13}
114test "zig fmt: float literal with exponent" {
215 try testCanonical(
316 \\pub const f64_true_min = 4.94065645841246544177e-324;
std/zig/tokenizer.zig+34
......@@ -220,6 +220,8 @@ pub const Tokenizer = struct {
220220 MultilineStringLiteralLineBackslash,
221221 CharLiteral,
222222 CharLiteralBackslash,
223 CharLiteralEscape1,
224 CharLiteralEscape2,
223225 CharLiteralEnd,
224226 Backslash,
225227 Equal,
......@@ -612,9 +614,32 @@ pub const Tokenizer = struct {
612614 result.id = Token.Id.Invalid;
613615 break;
614616 },
617 'x' => {
618 state = State.CharLiteralEscape1;
619 },
620 else => {
621 state = State.CharLiteralEnd;
622 },
623 },
624
625 State.CharLiteralEscape1 => switch (c) {
626 '0'...'9', 'a'...'z', 'A'...'F' => {
627 state = State.CharLiteralEscape2;
628 },
615629 else => {
630 result.id = Token.Id.Invalid;
631 break;
632 },
633 },
634
635 State.CharLiteralEscape2 => switch (c) {
636 '0'...'9', 'a'...'z', 'A'...'F' => {
616637 state = State.CharLiteralEnd;
617638 },
639 else => {
640 result.id = Token.Id.Invalid;
641 break;
642 },
618643 },
619644
620645 State.CharLiteralEnd => switch (c) {
......@@ -988,6 +1013,8 @@ pub const Tokenizer = struct {
9881013 State.MultilineStringLiteralLineBackslash,
9891014 State.CharLiteral,
9901015 State.CharLiteralBackslash,
1016 State.CharLiteralEscape1,
1017 State.CharLiteralEscape2,
9911018 State.CharLiteralEnd,
9921019 State.StringLiteralBackslash => {
9931020 result.id = Token.Id.Invalid;
......@@ -1127,6 +1154,13 @@ test "tokenizer" {
11271154 });
11281155}
11291156
1157test "tokenizer - char literal with hex escape" {
1158 testTokenize( \\'\x1b'
1159 , []Token.Id {
1160 Token.Id.CharLiteral,
1161 });
1162}
1163
11301164test "tokenizer - float literal e exponent" {
11311165 testTokenize("a = 4.94065645841246544177e-324;\n", []Token.Id {
11321166 Token.Id.Identifier,