authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-13 10:28:55-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-13 10:28:55-05:00
log2dcff95bd2fb8f377491ac48b0ecf961183abcd7
tree8fc8547d6f4bb443d01ff8a5923b6e4332a344e2
parentdfbb8254ca97154b5314bde03655417c1dca86ae

self hosted: add tokenizer test fix eof handling

1 files changed, 30 insertions(+), 3 deletions(-)

std/zig/tokenizer.zig+30-3
......@@ -72,6 +72,8 @@ pub const Token = struct {
7272 Eof,
7373 Builtin,
7474 Bang,
75 Pipe,
76 PipeEqual,
7577 Equal,
7678 EqualEqual,
7779 BangEqual,
......@@ -193,6 +195,7 @@ pub const Tokenizer = struct {
193195 StringLiteralBackslash,
194196 Equal,
195197 Bang,
198 Pipe,
196199 Minus,
197200 Slash,
198201 LineComment,
......@@ -248,6 +251,9 @@ pub const Tokenizer = struct {
248251 '!' => {
249252 state = State.Bang;
250253 },
254 '|' => {
255 state = State.Pipe;
256 },
251257 '(' => {
252258 result.id = Token.Id.LParen;
253259 self.index += 1;
......@@ -394,6 +400,18 @@ pub const Tokenizer = struct {
394400 },
395401 },
396402
403 State.Pipe => switch (c) {
404 '=' => {
405 result.id = Token.Id.PipeEqual;
406 self.index += 1;
407 break;
408 },
409 else => {
410 result.id = Token.Id.Pipe;
411 break;
412 },
413 },
414
397415 State.Equal => switch (c) {
398416 '=' => {
399417 result.id = Token.Id.EqualEqual;
......@@ -525,9 +543,7 @@ pub const Tokenizer = struct {
525543 else => break,
526544 },
527545 }
528 }
529 result.end = self.index;
530 if (self.index == self.buffer.len) {
546 } else if (self.index == self.buffer.len) {
531547 switch (state) {
532548 State.Start,
533549 State.C,
......@@ -578,6 +594,9 @@ pub const Tokenizer = struct {
578594 State.Period2 => {
579595 result.id = Token.Id.Ellipsis2;
580596 },
597 State.Pipe => {
598 result.id = Token.Id.Pipe;
599 },
581600 }
582601 }
583602 if (result.id == Token.Id.Eof) {
......@@ -587,6 +606,7 @@ pub const Tokenizer = struct {
587606 }
588607 }
589608
609 result.end = self.index;
590610 return result;
591611 }
592612
......@@ -716,6 +736,13 @@ test "tokenizer - string identifier and builtin fns" {
716736 );
717737}
718738
739test "tokenizer - pipe and then invalid" {
740 testTokenize("||=", []Token.Id{
741 Token.Id.Pipe,
742 Token.Id.PipeEqual,
743 });
744}
745
719746fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
720747 var tokenizer = Tokenizer.init(source);
721748 for (expected_tokens) |expected_token_id| {