Commit dcbc52ec85d00fbd3603c314aaaab98fb3866892 | zig

author	Travis Staloch <twostepted@gmail.com> 2021-09-08 15:58:37-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2021-09-28 17:03:43-07:00
log	dcbc52ec85d00fbd3603c314aaaab98fb3866892
tree	cda1709af1515d51ada13319119f9f79544ff4f3
parent	b9a95f2dd94e6175322d3388c3936eb600ec90ea

sat-arithmetic: correctly tokenize <<|, <<|=

- set state rather than result.tag in tokenizer.zig
- add test to tokenizer.zig for <<, <<|, <<|=

3 files changed, 10 insertions(+), 5 deletions(-)

lib
- std
  - zig
    tokenizer.zig
src
- Air.zig
- stage1
  - tokenizer.cpp

lib/std/zig/tokenizer.zig+7-1

 @@ -1007,7 +1007,7 @@ pub const Tokenizer = struct {
                         break;
                     },
                     '|' => {
-                        result.tag = .angle_bracket_angle_bracket_left_pipe;
+                        state = .angle_bracket_angle_bracket_left_pipe;
                     },
                     else => {
                         result.tag = .angle_bracket_angle_bracket_left;
 @@ -2015,6 +2015,12 @@ test "tokenizer - invalid token with unfinished escape right before eof" {
     try testTokenize("'\\u", &.{.invalid});
 }
+test "tokenizer - saturating" {
+    try testTokenize("<<", &.{.angle_bracket_angle_bracket_left});
+    try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe});
+    try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal});
+}
 fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void {
     var tokenizer = Tokenizer.init(source);
     for (expected_tokens) |expected_token_id| {

src/Air.zig+3-3

src/stage1/tokenizer.cpp-1

...	@@ -995,7 +995,6 @@ void tokenize(const char source, Tokenization out) {	...	@@ -995,7 +995,6 @@ void tokenize(const char source, Tokenization out) {
995	t.state = TokenizeState_start;	995	t.state = TokenizeState_start;
996	break;	996	break;
997	case '\|':	997	case '\|':
998	// t.out->ids.last() = TokenIdBitShiftLeftPipe;
999	t.state = TokenizeState_angle_bracket_angle_bracket_left_pipe;	998	t.state = TokenizeState_angle_bracket_angle_bracket_left_pipe;
1000	break;	999	break;
1001	default:	1000	default:

...	@@ -44,7 +44,7 @@ pub const Inst = struct {	...	@@ -44,7 +44,7 @@ pub const Inst = struct {
44	/// is the same as both operands.	44	/// is the same as both operands.
45	/// Uses the `bin_op` field.	45	/// Uses the `bin_op` field.
46	addwrap,	46	addwrap,
47	/// Saturating integer addition.	47	/// Saturating integer addition.
48	/// Both operands are guaranteed to be the same type, and the result type	48	/// Both operands are guaranteed to be the same type, and the result type
49	/// is the same as both operands.	49	/// is the same as both operands.
50	/// Uses the `bin_op` field.	50	/// Uses the `bin_op` field.
...	@@ -59,7 +59,7 @@ pub const Inst = struct {	...	@@ -59,7 +59,7 @@ pub const Inst = struct {
59	/// is the same as both operands.	59	/// is the same as both operands.
60	/// Uses the `bin_op` field.	60	/// Uses the `bin_op` field.
61	subwrap,	61	subwrap,
62	/// Saturating integer subtraction.	62	/// Saturating integer subtraction.
63	/// Both operands are guaranteed to be the same type, and the result type	63	/// Both operands are guaranteed to be the same type, and the result type
64	/// is the same as both operands.	64	/// is the same as both operands.
65	/// Uses the `bin_op` field.	65	/// Uses the `bin_op` field.
...	@@ -74,7 +74,7 @@ pub const Inst = struct {	...	@@ -74,7 +74,7 @@ pub const Inst = struct {
74	/// is the same as both operands.	74	/// is the same as both operands.
75	/// Uses the `bin_op` field.	75	/// Uses the `bin_op` field.
76	mulwrap,	76	mulwrap,
77	/// Saturating integer multiplication.	77	/// Saturating integer multiplication.
78	/// Both operands are guaranteed to be the same type, and the result type	78	/// Both operands are guaranteed to be the same type, and the result type
79	/// is the same as both operands.	79	/// is the same as both operands.
80	/// Uses the `bin_op` field.	80	/// Uses the `bin_op` field.