authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2017-12-23 18:35:45-07:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2017-12-23 18:35:45-07:00
log0082989f22165c6607a02528f868204285ad982a
treebbdbd48f5f9e48b95172bcf361fb66ecc2a778f0
parent45ab752f9acd5247cd970ab01a388390ac3bdd94

[self-hosted] tokenizer error for ascii control codes

1 files changed, 69 insertions(+), 24 deletions(-)

src-self-hosted/tokenizer.zig+69-24
...@@ -141,6 +141,7 @@ pub const Tokenizer = struct {...@@ -141,6 +141,7 @@ pub const Tokenizer = struct {
141 buffer: []const u8,141 buffer: []const u8,
142 index: usize,142 index: usize,
143 actual_file_end: usize,143 actual_file_end: usize,
144 pending_invalid_token: ?Token,
144145
145 pub const Location = struct {146 pub const Location = struct {
146 line: usize,147 line: usize,
...@@ -179,24 +180,18 @@ pub const Tokenizer = struct {...@@ -179,24 +180,18 @@ pub const Tokenizer = struct {
179 }180 }
180181
181 pub fn init(buffer: []const u8) -> Tokenizer {182 pub fn init(buffer: []const u8) -> Tokenizer {
182 if (buffer.len == 0 or buffer[buffer.len - 1] == '\n') {183 var source_len = buffer.len;
183 return Tokenizer {184 while (source_len > 0) : (source_len -= 1) {
184 .buffer = buffer,185 if (buffer[source_len - 1] == '\n') break;
185 .index = 0,
186 .actual_file_end = buffer.len,
187 };
188 } else {
189 // last line is incomplete, so skip it, and give an error when we get there.186 // last line is incomplete, so skip it, and give an error when we get there.
190 var source_len = buffer.len;
191 while (source_len > 0) : (source_len -= 1) {
192 if (buffer[source_len - 1] == '\n') break;
193 }
194 return Tokenizer {
195 .buffer = buffer[0..source_len],
196 .index = 0,
197 .actual_file_end = buffer.len,
198 };
199 }187 }
188
189 return Tokenizer {
190 .buffer = buffer[0..source_len],
191 .index = 0,
192 .actual_file_end = buffer.len,
193 .pending_invalid_token = null,
194 };
200 }195 }
201196
202 const State = enum {197 const State = enum {
...@@ -223,6 +218,10 @@ pub const Tokenizer = struct {...@@ -223,6 +218,10 @@ pub const Tokenizer = struct {
223 };218 };
224219
225 pub fn next(self: &Tokenizer) -> Token {220 pub fn next(self: &Tokenizer) -> Token {
221 if (self.pending_invalid_token) |token| {
222 self.pending_invalid_token = null;
223 return token;
224 }
226 var state = State.Start;225 var state = State.Start;
227 var result = Token {226 var result = Token {
228 .id = Token.Id.Eof,227 .id = Token.Id.Eof,
...@@ -368,7 +367,7 @@ pub const Tokenizer = struct {...@@ -368,7 +367,7 @@ pub const Tokenizer = struct {
368 break;367 break;
369 },368 },
370 '\n' => break, // Look for this error later.369 '\n' => break, // Look for this error later.
371 else => {},370 else => self.checkLiteralCharacter(),
372 },371 },
373372
374 State.StringLiteralBackslash => switch (c) {373 State.StringLiteralBackslash => switch (c) {
...@@ -455,7 +454,7 @@ pub const Tokenizer = struct {...@@ -455,7 +454,7 @@ pub const Tokenizer = struct {
455 .end = undefined,454 .end = undefined,
456 };455 };
457 },456 },
458 else => {},457 else => self.checkLiteralCharacter(),
459 },458 },
460 State.Zero => switch (c) {459 State.Zero => switch (c) {
461 'b', 'o', 'x' => {460 'b', 'o', 'x' => {
...@@ -513,10 +512,16 @@ pub const Tokenizer = struct {...@@ -513,10 +512,16 @@ pub const Tokenizer = struct {
513 }512 }
514 }513 }
515 result.end = self.index;514 result.end = self.index;
516 if (result.id == Token.Id.Eof and self.actual_file_end != self.buffer.len) {515 if (result.id == Token.Id.Eof) {
517 // instead of an Eof, give an error token516 if (self.pending_invalid_token) |token| {
518 result.id = Token.Id.NoEolAtEof;517 self.pending_invalid_token = null;
519 result.end = self.actual_file_end;518 return token;
519 }
520 if (self.actual_file_end != self.buffer.len) {
521 // instead of an Eof, give an error token
522 result.id = Token.Id.NoEolAtEof;
523 result.end = self.actual_file_end;
524 }
520 }525 }
521 return result;526 return result;
522 }527 }
...@@ -524,12 +529,29 @@ pub const Tokenizer = struct {...@@ -524,12 +529,29 @@ pub const Tokenizer = struct {
524 pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) -> []const u8 {529 pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) -> []const u8 {
525 return self.buffer[token.start..token.end];530 return self.buffer[token.start..token.end];
526 }531 }
532
533 fn checkLiteralCharacter(self: &Tokenizer) {
534 if (self.pending_invalid_token != null) return;
535 const c0 = self.buffer[self.index];
536 if (c0 < 0x20 or c0 == 0x7f) {
537 // ascii control codes are never allowed
538 // (note that \n was checked before we got here)
539 self.pending_invalid_token = Token {
540 .id = Token.Id.Invalid,
541 .start = self.index,
542 .end = self.index + 1,
543 };
544 return;
545 }
546 }
527};547};
528548
529549
530550
531test "tokenizer" {551test "tokenizer" {
532 // source must end with eol552 // source must end with eol
553 testTokenize("", []Token.Id {
554 }, true);
533 testTokenize("no newline", []Token.Id {555 testTokenize("no newline", []Token.Id {
534 }, false);556 }, false);
535 testTokenize("test\n", []Token.Id {557 testTokenize("test\n", []Token.Id {
...@@ -538,6 +560,29 @@ test "tokenizer" {...@@ -538,6 +560,29 @@ test "tokenizer" {
538 testTokenize("test\nno newline", []Token.Id {560 testTokenize("test\nno newline", []Token.Id {
539 Token.Id.Keyword_test,561 Token.Id.Keyword_test,
540 }, false);562 }, false);
563
564 // invalid token characters
565 testTokenize("#\n", []Token.Id {
566 Token.Id.Invalid,
567 }, true);
568 testTokenize("`\n", []Token.Id {
569 Token.Id.Invalid,
570 }, true);
571
572 // invalid literal/comment characters
573 testTokenize("\"\x00\"\n", []Token.Id {
574 Token.Id { .StringLiteral = Token.StrLitKind.Normal },
575 Token.Id.Invalid,
576 }, true);
577 testTokenize("//\x00\n", []Token.Id {
578 Token.Id.Invalid,
579 }, true);
580 testTokenize("//\x1f\n", []Token.Id {
581 Token.Id.Invalid,
582 }, true);
583 testTokenize("//\x7f\n", []Token.Id {
584 Token.Id.Invalid,
585 }, true);
541}586}
542587
543fn testTokenize(source: []const u8, expected_tokens: []const Token.Id, expected_eol_at_eof: bool) {588fn testTokenize(source: []const u8, expected_tokens: []const Token.Id, expected_eol_at_eof: bool) {
...@@ -546,8 +591,8 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id, expected_...@@ -546,8 +591,8 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id, expected_
546 const token = tokenizer.next();591 const token = tokenizer.next();
547 std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));592 std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
548 switch (expected_token_id) {593 switch (expected_token_id) {
549 Token.Id.StringLiteral => |kind| {594 Token.Id.StringLiteral => |expected_kind| {
550 @panic("TODO: how do i test this?");595 std.debug.assert(expected_kind == switch (token.id) { Token.Id.StringLiteral => |kind| kind, else => unreachable });
551 },596 },
552 else => {},597 else => {},
553 }598 }