| author |  Travis Staloch <twostepted@gmail.com> 2021-09-08 15:58:37-07:00 |
| committer |  Andrew Kelley <andrew@ziglang.org> 2021-09-28 17:03:43-07:00 |
| log | dcbc52ec85d00fbd3603c314aaaab98fb3866892 |
| tree | cda1709af1515d51ada13319119f9f79544ff4f3 |
| parent | b9a95f2dd94e6175322d3388c3936eb600ec90ea |
sat-arithmetic: correctly tokenize <<|, <<|=
- set state rather than result.tag in tokenizer.zig
- add test to tokenizer.zig for <<, <<|, <<|=3 files changed, 10 insertions(+), 5 deletions(-)
lib- std
- src
- Air.zig
- stage1
lib/std/zig/tokenizer.zig+7-1
| ... | ... | @@ -1007,7 +1007,7 @@ pub const Tokenizer = struct { |
| 1007 | 1007 | break; |
| 1008 | 1008 | }, |
| 1009 | 1009 | '|' => { |
| 1010 | result.tag = .angle_bracket_angle_bracket_left_pipe; | |
| 1010 | state = .angle_bracket_angle_bracket_left_pipe; | |
| 1011 | 1011 | }, |
| 1012 | 1012 | else => { |
| 1013 | 1013 | result.tag = .angle_bracket_angle_bracket_left; |
| ... | ... | @@ -2015,6 +2015,12 @@ test "tokenizer - invalid token with unfinished escape right before eof" { |
| 2015 | 2015 | try testTokenize("'\\u", &.{.invalid}); |
| 2016 | 2016 | } |
| 2017 | 2017 | |
| 2018 | test "tokenizer - saturating" { | |
| 2019 | try testTokenize("<<", &.{.angle_bracket_angle_bracket_left}); | |
| 2020 | try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe}); | |
| 2021 | try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal}); | |
| 2022 | } | |
| 2023 | ||
| 2018 | 2024 | fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void { |
| 2019 | 2025 | var tokenizer = Tokenizer.init(source); |
| 2020 | 2026 | for (expected_tokens) |expected_token_id| { |
src/Air.zig+3-3
| ... | ... | @@ -44,7 +44,7 @@ pub const Inst = struct { |
| 44 | 44 | /// is the same as both operands. |
| 45 | 45 | /// Uses the `bin_op` field. |
| 46 | 46 | addwrap, |
| 47 | /// Saturating integer addition. | |
| 47 | /// Saturating integer addition. | |
| 48 | 48 | /// Both operands are guaranteed to be the same type, and the result type |
| 49 | 49 | /// is the same as both operands. |
| 50 | 50 | /// Uses the `bin_op` field. |
| ... | ... | @@ -59,7 +59,7 @@ pub const Inst = struct { |
| 59 | 59 | /// is the same as both operands. |
| 60 | 60 | /// Uses the `bin_op` field. |
| 61 | 61 | subwrap, |
| 62 | /// Saturating integer subtraction. | |
| 62 | /// Saturating integer subtraction. | |
| 63 | 63 | /// Both operands are guaranteed to be the same type, and the result type |
| 64 | 64 | /// is the same as both operands. |
| 65 | 65 | /// Uses the `bin_op` field. |
| ... | ... | @@ -74,7 +74,7 @@ pub const Inst = struct { |
| 74 | 74 | /// is the same as both operands. |
| 75 | 75 | /// Uses the `bin_op` field. |
| 76 | 76 | mulwrap, |
| 77 | /// Saturating integer multiplication. | |
| 77 | /// Saturating integer multiplication. | |
| 78 | 78 | /// Both operands are guaranteed to be the same type, and the result type |
| 79 | 79 | /// is the same as both operands. |
| 80 | 80 | /// Uses the `bin_op` field. |
src/stage1/tokenizer.cpp-1
| ... | ... | @@ -995,7 +995,6 @@ void tokenize(const char *source, Tokenization *out) { |
| 995 | 995 | t.state = TokenizeState_start; |
| 996 | 996 | break; |
| 997 | 997 | case '|': |
| 998 | // t.out->ids.last() = TokenIdBitShiftLeftPipe; | |
| 999 | 998 | t.state = TokenizeState_angle_bracket_angle_bracket_left_pipe; |
| 1000 | 999 | break; |
| 1001 | 1000 | default: |