authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-09-27 17:17:27+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-28 14:16:26-04:00
log5c6cd5e2c9e8b2d0feb0026bad7c201035a175b4
tree78552e0b970107df097bc7699e0be5996aa2831b
parente60939bfaafc9e6b3ccdc172009b950fc7a3eab1

stage{1,2}: Fix parsing of range literals

stage1 was unable to parse ranges whose starting point was written in binary/octal as the first dot in '...' was incorrectly interpreted as decimal point. stage2 forgot to reset the literal type to IntegerLiteral when it discovered the dot was not a decimal point. I've only stumbled across this bug because zig fmt keeps formatting the ranges without any space around the ...

2 files changed, 12 insertions(+), 3 deletions(-)

lib/std/zig/tokenizer.zig+9
...@@ -1195,6 +1195,7 @@ pub const Tokenizer = struct {...@@ -1195,6 +1195,7 @@ pub const Tokenizer = struct {
1195 },1195 },
1196 .num_dot_hex => switch (c) {1196 .num_dot_hex => switch (c) {
1197 '.' => {1197 '.' => {
1198 result.id = .IntegerLiteral;
1198 self.index -= 1;1199 self.index -= 1;
1199 state = .start;1200 state = .start;
1200 break;1201 break;
...@@ -1758,6 +1759,14 @@ test "correctly parse pointer assignment" {...@@ -1758,6 +1759,14 @@ test "correctly parse pointer assignment" {
1758 });1759 });
1759}1760}
17601761
1762test "tokenizer - range literals" {
1763 testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
1764 testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral });
1765 testTokenize("0x00...0x09", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
1766 testTokenize("0b00...0b11", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
1767 testTokenize("0o00...0o11", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
1768}
1769
1761test "tokenizer - number literals decimal" {1770test "tokenizer - number literals decimal" {
1762 testTokenize("0", &[_]Token.Id{.IntegerLiteral});1771 testTokenize("0", &[_]Token.Id{.IntegerLiteral});
1763 testTokenize("1", &[_]Token.Id{.IntegerLiteral});1772 testTokenize("1", &[_]Token.Id{.IntegerLiteral});
src/tokenizer.cpp+3-3
...@@ -1225,9 +1225,6 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1225,9 +1225,6 @@ void tokenize(Buf *buf, Tokenization *out) {
1225 invalid_char_error(&t, c);1225 invalid_char_error(&t, c);
1226 break;1226 break;
1227 }1227 }
1228 if (t.radix != 16 && t.radix != 10) {
1229 invalid_char_error(&t, c);
1230 }
1231 t.state = TokenizeStateNumberDot;1228 t.state = TokenizeStateNumberDot;
1232 break;1229 break;
1233 }1230 }
...@@ -1281,6 +1278,9 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1281,6 +1278,9 @@ void tokenize(Buf *buf, Tokenization *out) {
1281 t.state = TokenizeStateStart;1278 t.state = TokenizeStateStart;
1282 continue;1279 continue;
1283 }1280 }
1281 if (t.radix != 16 && t.radix != 10) {
1282 invalid_char_error(&t, c);
1283 }
1284 t.pos -= 1;1284 t.pos -= 1;
1285 t.state = TokenizeStateFloatFractionNoUnderscore;1285 t.state = TokenizeStateFloatFractionNoUnderscore;
1286 assert(t.cur_tok->id == TokenIdIntLiteral);1286 assert(t.cur_tok->id == TokenIdIntLiteral);