authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-12 21:25:38-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-12 21:26:15-05:00
logdfbb8254ca97154b5314bde03655417c1dca86ae
tree0360cdd64d2289a6321133e4e07e4eb09a26177d
parentb4e44c4e80fd4f477a46251fdaf485a742ffe6f7

fix self hosted tokenizer handling of EOF

2 files changed, 107 insertions(+), 27 deletions(-)

std/zig/ast.zig+17-15
...@@ -18,6 +18,7 @@ pub const Node = struct {...@@ -18,6 +18,7 @@ pub const Node = struct {
18 PrefixOp,18 PrefixOp,
19 IntegerLiteral,19 IntegerLiteral,
20 FloatLiteral,20 FloatLiteral,
21 BuiltinCall,
21 };22 };
2223
23 pub fn iterate(base: &Node, index: usize) ?&Node {24 pub fn iterate(base: &Node, index: usize) ?&Node {
...@@ -32,21 +33,7 @@ pub const Node = struct {...@@ -32,21 +33,7 @@ pub const Node = struct {
32 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),33 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),
33 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),34 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
34 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),35 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
35 };36 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
36 }
37
38 pub fn destroy(base: &Node, allocator: &mem.Allocator) void {
39 return switch (base.id) {
40 Id.Root => allocator.destroy(@fieldParentPtr(NodeRoot, "base", base)),
41 Id.VarDecl => allocator.destroy(@fieldParentPtr(NodeVarDecl, "base", base)),
42 Id.Identifier => allocator.destroy(@fieldParentPtr(NodeIdentifier, "base", base)),
43 Id.FnProto => allocator.destroy(@fieldParentPtr(NodeFnProto, "base", base)),
44 Id.ParamDecl => allocator.destroy(@fieldParentPtr(NodeParamDecl, "base", base)),
45 Id.Block => allocator.destroy(@fieldParentPtr(NodeBlock, "base", base)),
46 Id.InfixOp => allocator.destroy(@fieldParentPtr(NodeInfixOp, "base", base)),
47 Id.PrefixOp => allocator.destroy(@fieldParentPtr(NodePrefixOp, "base", base)),
48 Id.IntegerLiteral => allocator.destroy(@fieldParentPtr(NodeIntegerLiteral, "base", base)),
49 Id.FloatLiteral => allocator.destroy(@fieldParentPtr(NodeFloatLiteral, "base", base)),
50 };37 };
51 }38 }
52};39};
...@@ -269,3 +256,18 @@ pub const NodeFloatLiteral = struct {...@@ -269,3 +256,18 @@ pub const NodeFloatLiteral = struct {
269 return null;256 return null;
270 }257 }
271};258};
259
260pub const NodeBuiltinCall = struct {
261 base: Node,
262 builtin_token: Token,
263 params: ArrayList(&Node),
264
265 pub fn iterate(self: &NodeBuiltinCall, index: usize) ?&Node {
266 var i = index;
267
268 if (i < self.params.len) return self.params.at(i);
269 i -= self.params.len;
270
271 return null;
272 }
273};
std/zig/tokenizer.zig+90-12
...@@ -68,6 +68,7 @@ pub const Token = struct {...@@ -68,6 +68,7 @@ pub const Token = struct {
68 Invalid,68 Invalid,
69 Identifier,69 Identifier,
70 StringLiteral: StrLitKind,70 StringLiteral: StrLitKind,
71 StringIdentifier,
71 Eof,72 Eof,
72 Builtin,73 Builtin,
73 Bang,74 Bang,
...@@ -205,6 +206,7 @@ pub const Tokenizer = struct {...@@ -205,6 +206,7 @@ pub const Tokenizer = struct {
205 Ampersand,206 Ampersand,
206 Period,207 Period,
207 Period2,208 Period2,
209 SawAtSign,
208 };210 };
209211
210 pub fn next(self: &Tokenizer) Token {212 pub fn next(self: &Tokenizer) Token {
...@@ -238,8 +240,7 @@ pub const Tokenizer = struct {...@@ -238,8 +240,7 @@ pub const Tokenizer = struct {
238 result.id = Token.Id.Identifier;240 result.id = Token.Id.Identifier;
239 },241 },
240 '@' => {242 '@' => {
241 state = State.Builtin;243 state = State.SawAtSign;
242 result.id = Token.Id.Builtin;
243 },244 },
244 '=' => {245 '=' => {
245 state = State.Equal;246 state = State.Equal;
...@@ -313,6 +314,20 @@ pub const Tokenizer = struct {...@@ -313,6 +314,20 @@ pub const Tokenizer = struct {
313 break;314 break;
314 },315 },
315 },316 },
317
318 State.SawAtSign => switch (c) {
319 '"' => {
320 result.id = Token.Id.StringIdentifier;
321 state = State.StringLiteral;
322 },
323 else => {
324 // reinterpret as a builtin
325 self.index -= 1;
326 state = State.Builtin;
327 result.id = Token.Id.Builtin;
328 },
329 },
330
316 State.Ampersand => switch (c) {331 State.Ampersand => switch (c) {
317 '=' => {332 '=' => {
318 result.id = Token.Id.AmpersandEqual;333 result.id = Token.Id.AmpersandEqual;
...@@ -512,7 +527,59 @@ pub const Tokenizer = struct {...@@ -512,7 +527,59 @@ pub const Tokenizer = struct {
512 }527 }
513 }528 }
514 result.end = self.index;529 result.end = self.index;
530 if (self.index == self.buffer.len) {
531 switch (state) {
532 State.Start,
533 State.C,
534 State.IntegerLiteral,
535 State.IntegerLiteralWithRadix,
536 State.FloatFraction,
537 State.FloatExponentNumber,
538 State.StringLiteral, // find this error later
539 State.Builtin => {},
540
541 State.Identifier => {
542 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
543 result.id = id;
544 }
545 },
546 State.LineComment => {
547 result.id = Token.Id.Eof;
548 },
549
550 State.NumberDot,
551 State.FloatExponentUnsigned,
552 State.SawAtSign,
553 State.StringLiteralBackslash => {
554 result.id = Token.Id.Invalid;
555 },
515556
557 State.Equal => {
558 result.id = Token.Id.Equal;
559 },
560 State.Bang => {
561 result.id = Token.Id.Bang;
562 },
563 State.Minus => {
564 result.id = Token.Id.Minus;
565 },
566 State.Slash => {
567 result.id = Token.Id.Slash;
568 },
569 State.Zero => {
570 result.id = Token.Id.IntegerLiteral;
571 },
572 State.Ampersand => {
573 result.id = Token.Id.Ampersand;
574 },
575 State.Period => {
576 result.id = Token.Id.Period;
577 },
578 State.Period2 => {
579 result.id = Token.Id.Ellipsis2;
580 },
581 }
582 }
516 if (result.id == Token.Id.Eof) {583 if (result.id == Token.Id.Eof) {
517 if (self.pending_invalid_token) |token| {584 if (self.pending_invalid_token) |token| {
518 self.pending_invalid_token = null;585 self.pending_invalid_token = null;
...@@ -551,7 +618,7 @@ pub const Tokenizer = struct {...@@ -551,7 +618,7 @@ pub const Tokenizer = struct {
551 } else {618 } else {
552 // check utf8-encoded character.619 // check utf8-encoded character.
553 const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;620 const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
554 if (self.index + length >= self.buffer.len) {621 if (self.index + length > self.buffer.len) {
555 return u3(self.buffer.len - self.index);622 return u3(self.buffer.len - self.index);
556 }623 }
557 const bytes = self.buffer[self.index..self.index + length];624 const bytes = self.buffer[self.index..self.index + length];
...@@ -632,15 +699,25 @@ test "tokenizer - illegal unicode codepoints" {...@@ -632,15 +699,25 @@ test "tokenizer - illegal unicode codepoints" {
632 testTokenize("//\xe2\x80\xaa", []Token.Id{});699 testTokenize("//\xe2\x80\xaa", []Token.Id{});
633}700}
634701
702test "tokenizer - string identifier and builtin fns" {
703 testTokenize(
704 \\const @"if" = @import("std");
705 ,
706 []Token.Id{
707 Token.Id.Keyword_const,
708 Token.Id.StringIdentifier,
709 Token.Id.Equal,
710 Token.Id.Builtin,
711 Token.Id.LParen,
712 Token.Id {.StringLiteral = Token.StrLitKind.Normal},
713 Token.Id.RParen,
714 Token.Id.Semicolon,
715 }
716 );
717}
718
635fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {719fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
636 // (test authors, just make this bigger if you need it)720 var tokenizer = Tokenizer.init(source);
637 var padded_source: [0x100]u8 = undefined;
638 std.mem.copy(u8, padded_source[0..source.len], source);
639 padded_source[source.len + 0] = '\n';
640 padded_source[source.len + 1] = '\n';
641 padded_source[source.len + 2] = '\n';
642
643 var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]);
644 for (expected_tokens) |expected_token_id| {721 for (expected_tokens) |expected_token_id| {
645 const token = tokenizer.next();722 const token = tokenizer.next();
646 std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));723 std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
...@@ -651,5 +728,6 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {...@@ -651,5 +728,6 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
651 else => {},728 else => {},
652 }729 }
653 }730 }
654 std.debug.assert(tokenizer.next().id == Token.Id.Eof);731 const last_token = tokenizer.next();
732 std.debug.assert(last_token.id == Token.Id.Eof);
655}733}