authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-07-19 17:54:06-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-07-19 17:54:06-04:00
log8ea8cff4912bf66b8e95b7518877c7f55c7a9356
tree86e7d61f3ed055c54dbe49a5ed4ab9f2942d8d17
parent111d3792e0f2d92f7b9f73fb8265ee17fc229eb4
signaturelock-open Commit is signed but in an unrecognized format.

slightly simpler implementation

4 files changed, 135 insertions(+), 141 deletions(-)

doc/docgen.zig+118-120
...@@ -742,101 +742,101 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok...@@ -742,101 +742,101 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok
742 const token = tokenizer.next();742 const token = tokenizer.next();
743 try writeEscaped(out, src[index..token.start]);743 try writeEscaped(out, src[index..token.start]);
744 switch (token.id) {744 switch (token.id) {
745 std.zig.Token.Id.Eof => break,745 .Eof => break,
746746
747 std.zig.Token.Id.Keyword_align,747 .Keyword_align,
748 std.zig.Token.Id.Keyword_and,748 .Keyword_and,
749 std.zig.Token.Id.Keyword_asm,749 .Keyword_asm,
750 std.zig.Token.Id.Keyword_async,750 .Keyword_async,
751 std.zig.Token.Id.Keyword_await,751 .Keyword_await,
752 std.zig.Token.Id.Keyword_break,752 .Keyword_break,
753 std.zig.Token.Id.Keyword_cancel,753 .Keyword_cancel,
754 std.zig.Token.Id.Keyword_catch,754 .Keyword_catch,
755 std.zig.Token.Id.Keyword_comptime,755 .Keyword_comptime,
756 std.zig.Token.Id.Keyword_const,756 .Keyword_const,
757 std.zig.Token.Id.Keyword_continue,757 .Keyword_continue,
758 std.zig.Token.Id.Keyword_defer,758 .Keyword_defer,
759 std.zig.Token.Id.Keyword_else,759 .Keyword_else,
760 std.zig.Token.Id.Keyword_enum,760 .Keyword_enum,
761 std.zig.Token.Id.Keyword_errdefer,761 .Keyword_errdefer,
762 std.zig.Token.Id.Keyword_error,762 .Keyword_error,
763 std.zig.Token.Id.Keyword_export,763 .Keyword_export,
764 std.zig.Token.Id.Keyword_extern,764 .Keyword_extern,
765 std.zig.Token.Id.Keyword_for,765 .Keyword_for,
766 std.zig.Token.Id.Keyword_if,766 .Keyword_if,
767 std.zig.Token.Id.Keyword_inline,767 .Keyword_inline,
768 std.zig.Token.Id.Keyword_nakedcc,768 .Keyword_nakedcc,
769 std.zig.Token.Id.Keyword_noalias,769 .Keyword_noalias,
770 std.zig.Token.Id.Keyword_or,770 .Keyword_or,
771 std.zig.Token.Id.Keyword_orelse,771 .Keyword_orelse,
772 std.zig.Token.Id.Keyword_packed,772 .Keyword_packed,
773 std.zig.Token.Id.Keyword_promise,773 .Keyword_promise,
774 std.zig.Token.Id.Keyword_pub,774 .Keyword_pub,
775 std.zig.Token.Id.Keyword_resume,775 .Keyword_resume,
776 std.zig.Token.Id.Keyword_return,776 .Keyword_return,
777 std.zig.Token.Id.Keyword_linksection,777 .Keyword_linksection,
778 std.zig.Token.Id.Keyword_stdcallcc,778 .Keyword_stdcallcc,
779 std.zig.Token.Id.Keyword_struct,779 .Keyword_struct,
780 std.zig.Token.Id.Keyword_suspend,780 .Keyword_suspend,
781 std.zig.Token.Id.Keyword_switch,781 .Keyword_switch,
782 std.zig.Token.Id.Keyword_test,782 .Keyword_test,
783 std.zig.Token.Id.Keyword_threadlocal,783 .Keyword_threadlocal,
784 std.zig.Token.Id.Keyword_try,784 .Keyword_try,
785 std.zig.Token.Id.Keyword_union,785 .Keyword_union,
786 std.zig.Token.Id.Keyword_unreachable,786 .Keyword_unreachable,
787 std.zig.Token.Id.Keyword_usingnamespace,787 .Keyword_usingnamespace,
788 std.zig.Token.Id.Keyword_var,788 .Keyword_var,
789 std.zig.Token.Id.Keyword_volatile,789 .Keyword_volatile,
790 std.zig.Token.Id.Keyword_allowzero,790 .Keyword_allowzero,
791 std.zig.Token.Id.Keyword_while,791 .Keyword_while,
792 => {792 => {
793 try out.write("<span class=\"tok-kw\">");793 try out.write("<span class=\"tok-kw\">");
794 try writeEscaped(out, src[token.start..token.end]);794 try writeEscaped(out, src[token.start..token.end]);
795 try out.write("</span>");795 try out.write("</span>");
796 },796 },
797797
798 std.zig.Token.Id.Keyword_fn => {798 .Keyword_fn => {
799 try out.write("<span class=\"tok-kw\">");799 try out.write("<span class=\"tok-kw\">");
800 try writeEscaped(out, src[token.start..token.end]);800 try writeEscaped(out, src[token.start..token.end]);
801 try out.write("</span>");801 try out.write("</span>");
802 next_tok_is_fn = true;802 next_tok_is_fn = true;
803 },803 },
804804
805 std.zig.Token.Id.Keyword_undefined,805 .Keyword_undefined,
806 std.zig.Token.Id.Keyword_null,806 .Keyword_null,
807 std.zig.Token.Id.Keyword_true,807 .Keyword_true,
808 std.zig.Token.Id.Keyword_false,808 .Keyword_false,
809 => {809 => {
810 try out.write("<span class=\"tok-null\">");810 try out.write("<span class=\"tok-null\">");
811 try writeEscaped(out, src[token.start..token.end]);811 try writeEscaped(out, src[token.start..token.end]);
812 try out.write("</span>");812 try out.write("</span>");
813 },813 },
814814
815 std.zig.Token.Id.StringLiteral,815 .StringLiteral,
816 std.zig.Token.Id.MultilineStringLiteralLine,816 .MultilineStringLiteralLine,
817 std.zig.Token.Id.CharLiteral,817 .CharLiteral,
818 => {818 => {
819 try out.write("<span class=\"tok-str\">");819 try out.write("<span class=\"tok-str\">");
820 try writeEscaped(out, src[token.start..token.end]);820 try writeEscaped(out, src[token.start..token.end]);
821 try out.write("</span>");821 try out.write("</span>");
822 },822 },
823823
824 std.zig.Token.Id.Builtin => {824 .Builtin => {
825 try out.write("<span class=\"tok-builtin\">");825 try out.write("<span class=\"tok-builtin\">");
826 try writeEscaped(out, src[token.start..token.end]);826 try writeEscaped(out, src[token.start..token.end]);
827 try out.write("</span>");827 try out.write("</span>");
828 },828 },
829829
830 std.zig.Token.Id.LineComment,830 .LineComment,
831 std.zig.Token.Id.DocComment,831 .DocComment,
832 std.zig.Token.Id.ShebangLine,832 .ShebangLine,
833 => {833 => {
834 try out.write("<span class=\"tok-comment\">");834 try out.write("<span class=\"tok-comment\">");
835 try writeEscaped(out, src[token.start..token.end]);835 try writeEscaped(out, src[token.start..token.end]);
836 try out.write("</span>");836 try out.write("</span>");
837 },837 },
838838
839 std.zig.Token.Id.Identifier => {839 .Identifier => {
840 if (prev_tok_was_fn) {840 if (prev_tok_was_fn) {
841 try out.write("<span class=\"tok-fn\">");841 try out.write("<span class=\"tok-fn\">");
842 try writeEscaped(out, src[token.start..token.end]);842 try writeEscaped(out, src[token.start..token.end]);
...@@ -864,74 +864,72 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok...@@ -864,74 +864,72 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok
864 }864 }
865 },865 },
866866
867 std.zig.Token.Id.IntegerLiteral,867 .IntegerLiteral,
868 std.zig.Token.Id.FloatLiteral,868 .FloatLiteral,
869 => {869 => {
870 try out.write("<span class=\"tok-number\">");870 try out.write("<span class=\"tok-number\">");
871 try writeEscaped(out, src[token.start..token.end]);871 try writeEscaped(out, src[token.start..token.end]);
872 try out.write("</span>");872 try out.write("</span>");
873 },873 },
874874
875 std.zig.Token.Id.Bang,875 .Bang,
876 std.zig.Token.Id.Pipe,876 .Pipe,
877 std.zig.Token.Id.PipePipe,877 .PipePipe,
878 std.zig.Token.Id.PipeEqual,878 .PipeEqual,
879 std.zig.Token.Id.Equal,879 .Equal,
880 std.zig.Token.Id.EqualEqual,880 .EqualEqual,
881 std.zig.Token.Id.EqualAngleBracketRight,881 .EqualAngleBracketRight,
882 std.zig.Token.Id.BangEqual,882 .BangEqual,
883 std.zig.Token.Id.LParen,883 .LParen,
884 std.zig.Token.Id.RParen,884 .RParen,
885 std.zig.Token.Id.Semicolon,885 .Semicolon,
886 std.zig.Token.Id.Percent,886 .Percent,
887 std.zig.Token.Id.PercentEqual,887 .PercentEqual,
888 std.zig.Token.Id.LBrace,888 .LBrace,
889 std.zig.Token.Id.RBrace,889 .RBrace,
890 std.zig.Token.Id.LBracket,890 .LBracket,
891 std.zig.Token.Id.RBracket,891 .RBracket,
892 std.zig.Token.Id.Period,892 .Period,
893 std.zig.Token.Id.Ellipsis2,893 .Ellipsis2,
894 std.zig.Token.Id.Ellipsis3,894 .Ellipsis3,
895 std.zig.Token.Id.Caret,895 .Caret,
896 std.zig.Token.Id.CaretEqual,896 .CaretEqual,
897 std.zig.Token.Id.Plus,897 .Plus,
898 std.zig.Token.Id.PlusPlus,898 .PlusPlus,
899 std.zig.Token.Id.PlusEqual,899 .PlusEqual,
900 std.zig.Token.Id.PlusPercent,900 .PlusPercent,
901 std.zig.Token.Id.PlusPercentEqual,901 .PlusPercentEqual,
902 std.zig.Token.Id.Minus,902 .Minus,
903 std.zig.Token.Id.MinusEqual,903 .MinusEqual,
904 std.zig.Token.Id.MinusPercent,904 .MinusPercent,
905 std.zig.Token.Id.MinusPercentEqual,905 .MinusPercentEqual,
906 std.zig.Token.Id.Asterisk,906 .Asterisk,
907 std.zig.Token.Id.AsteriskEqual,907 .AsteriskEqual,
908 std.zig.Token.Id.AsteriskAsterisk,908 .AsteriskAsterisk,
909 std.zig.Token.Id.AsteriskPercent,909 .AsteriskPercent,
910 std.zig.Token.Id.AsteriskPercentEqual,910 .AsteriskPercentEqual,
911 std.zig.Token.Id.Arrow,911 .Arrow,
912 std.zig.Token.Id.Colon,912 .Colon,
913 std.zig.Token.Id.Slash,913 .Slash,
914 std.zig.Token.Id.SlashEqual,914 .SlashEqual,
915 std.zig.Token.Id.Comma,915 .Comma,
916 std.zig.Token.Id.Ampersand,916 .Ampersand,
917 std.zig.Token.Id.AmpersandEqual,917 .AmpersandEqual,
918 std.zig.Token.Id.QuestionMark,918 .QuestionMark,
919 std.zig.Token.Id.AngleBracketLeft,919 .AngleBracketLeft,
920 std.zig.Token.Id.AngleBracketLeftEqual,920 .AngleBracketLeftEqual,
921 std.zig.Token.Id.AngleBracketAngleBracketLeft,921 .AngleBracketAngleBracketLeft,
922 std.zig.Token.Id.AngleBracketAngleBracketLeftEqual,922 .AngleBracketAngleBracketLeftEqual,
923 std.zig.Token.Id.AngleBracketRight,923 .AngleBracketRight,
924 std.zig.Token.Id.AngleBracketRightEqual,924 .AngleBracketRightEqual,
925 std.zig.Token.Id.AngleBracketAngleBracketRight,925 .AngleBracketAngleBracketRight,
926 std.zig.Token.Id.AngleBracketAngleBracketRightEqual,926 .AngleBracketAngleBracketRightEqual,
927 std.zig.Token.Id.Tilde,927 .Tilde,
928 std.zig.Token.Id.BracketStarBracket,928 .BracketStarBracket,
929 std.zig.Token.Id.BracketStarCBracket,929 .BracketStarCBracket,
930 => try writeEscaped(out, src[token.start..token.end]),930 => try writeEscaped(out, src[token.start..token.end]),
931931
932 std.zig.Token.Id.AmpersandAmpersand,932 .Invalid, .Invalid_ampersands => return parseError(
933 std.zig.Token.Id.Invalid,
934 => return parseError(
935 docgen_tokenizer,933 docgen_tokenizer,
936 source_token,934 source_token,
937 "syntax error",935 "syntax error",
std/zig/ast.zig+13-6
...@@ -113,7 +113,6 @@ pub const Tree = struct {...@@ -113,7 +113,6 @@ pub const Tree = struct {
113113
114pub const Error = union(enum) {114pub const Error = union(enum) {
115 InvalidToken: InvalidToken,115 InvalidToken: InvalidToken,
116 InvalidAmpersandAmpersand: InvalidAmpersandAmpersand,
117 ExpectedContainerMembers: ExpectedContainerMembers,116 ExpectedContainerMembers: ExpectedContainerMembers,
118 ExpectedStringLiteral: ExpectedStringLiteral,117 ExpectedStringLiteral: ExpectedStringLiteral,
119 ExpectedIntegerLiteral: ExpectedIntegerLiteral,118 ExpectedIntegerLiteral: ExpectedIntegerLiteral,
...@@ -161,7 +160,6 @@ pub const Error = union(enum) {...@@ -161,7 +160,6 @@ pub const Error = union(enum) {
161 pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {160 pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {
162 switch (self.*) {161 switch (self.*) {
163 .InvalidToken => |*x| return x.render(tokens, stream),162 .InvalidToken => |*x| return x.render(tokens, stream),
164 .InvalidAmpersandAmpersand => |*x| return x.render(tokens, stream),
165 .ExpectedContainerMembers => |*x| return x.render(tokens, stream),163 .ExpectedContainerMembers => |*x| return x.render(tokens, stream),
166 .ExpectedStringLiteral => |*x| return x.render(tokens, stream),164 .ExpectedStringLiteral => |*x| return x.render(tokens, stream),
167 .ExpectedIntegerLiteral => |*x| return x.render(tokens, stream),165 .ExpectedIntegerLiteral => |*x| return x.render(tokens, stream),
...@@ -211,7 +209,6 @@ pub const Error = union(enum) {...@@ -211,7 +209,6 @@ pub const Error = union(enum) {
211 pub fn loc(self: *const Error) TokenIndex {209 pub fn loc(self: *const Error) TokenIndex {
212 switch (self.*) {210 switch (self.*) {
213 .InvalidToken => |x| return x.token,211 .InvalidToken => |x| return x.token,
214 .InvalidAmpersandAmpersand => |x| return x.token,
215 .ExpectedContainerMembers => |x| return x.token,212 .ExpectedContainerMembers => |x| return x.token,
216 .ExpectedStringLiteral => |x| return x.token,213 .ExpectedStringLiteral => |x| return x.token,
217 .ExpectedIntegerLiteral => |x| return x.token,214 .ExpectedIntegerLiteral => |x| return x.token,
...@@ -292,7 +289,6 @@ pub const Error = union(enum) {...@@ -292,7 +289,6 @@ pub const Error = union(enum) {
292 pub const ExpectedDerefOrUnwrap = SingleTokenError("Expected pointer dereference or optional unwrap, found {}");289 pub const ExpectedDerefOrUnwrap = SingleTokenError("Expected pointer dereference or optional unwrap, found {}");
293 pub const ExpectedSuffixOp = SingleTokenError("Expected pointer dereference, optional unwrap, or field access, found {}");290 pub const ExpectedSuffixOp = SingleTokenError("Expected pointer dereference, optional unwrap, or field access, found {}");
294291
295 pub const InvalidAmpersandAmpersand = SimpleError("Invalid token '&&', 'and' performs boolean AND");
296 pub const ExpectedParamType = SimpleError("Expected parameter type");292 pub const ExpectedParamType = SimpleError("Expected parameter type");
297 pub const ExpectedPubItem = SimpleError("Pub must be followed by fn decl, var decl, or container member");293 pub const ExpectedPubItem = SimpleError("Pub must be followed by fn decl, var decl, or container member");
298 pub const UnattachedDocComment = SimpleError("Unattached documentation comment");294 pub const UnattachedDocComment = SimpleError("Unattached documentation comment");
...@@ -322,8 +318,19 @@ pub const Error = union(enum) {...@@ -322,8 +318,19 @@ pub const Error = union(enum) {
322 expected_id: Token.Id,318 expected_id: Token.Id,
323319
324 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {320 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {
325 const token_name = @tagName(tokens.at(self.token).id);321 const found_token = tokens.at(self.token);
326 return stream.print("expected {}, found {}", @tagName(self.expected_id), token_name);322 switch (found_token.id) {
323 .Invalid_ampersands => {
324 return stream.print("`&&` is invalid. Note that `and` is boolean AND.");
325 },
326 .Invalid => {
327 return stream.print("expected {}, found invalid bytes", @tagName(self.expected_id));
328 },
329 else => {
330 const token_name = @tagName(found_token.id);
331 return stream.print("expected {}, found {}", @tagName(self.expected_id), token_name);
332 },
333 }
327 }334 }
328 };335 };
329336
std/zig/parse.zig+1-13
...@@ -774,7 +774,7 @@ fn parseBoolAndExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -774,7 +774,7 @@ fn parseBoolAndExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
774 arena,774 arena,
775 it,775 it,
776 tree,776 tree,
777 parseAmpersandAmpersandOp,777 SimpleBinOpParseFn(.Keyword_and, .BoolAnd),
778 parseCompareExpr,778 parseCompareExpr,
779 .Infinitely,779 .Infinitely,
780 );780 );
...@@ -2687,18 +2687,6 @@ fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) No...@@ -2687,18 +2687,6 @@ fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) No
26872687
2688// Helper parsers not included in the grammar2688// Helper parsers not included in the grammar
26892689
2690fn parseAmpersandAmpersandOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2691 const op_parse_and = SimpleBinOpParseFn(.Keyword_and, Node.InfixOp.Op.BoolAnd);
2692 const op_token = eatToken(it, .AmpersandAmpersand);
2693 if (op_token != null) {
2694 try tree.errors.push(AstError{
2695 .InvalidAmpersandAmpersand = AstError.InvalidAmpersandAmpersand{ .token = it.index },
2696 });
2697 return error.ParseError;
2698 }
2699 return op_parse_and(arena, it, tree);
2700}
2701
2702fn parseBuiltinCall(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {2690fn parseBuiltinCall(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2703 const token = eatToken(it, .Builtin) orelse return null;2691 const token = eatToken(it, .Builtin) orelse return null;
2704 const params = (try parseFnCallArguments(arena, it, tree)) orelse {2692 const params = (try parseFnCallArguments(arena, it, tree)) orelse {
std/zig/tokenizer.zig+3-2
...@@ -77,6 +77,7 @@ pub const Token = struct {...@@ -77,6 +77,7 @@ pub const Token = struct {
7777
78 pub const Id = enum {78 pub const Id = enum {
79 Invalid,79 Invalid,
80 Invalid_ampersands,
80 Identifier,81 Identifier,
81 StringLiteral,82 StringLiteral,
82 MultilineStringLiteralLine,83 MultilineStringLiteralLine,
...@@ -125,7 +126,6 @@ pub const Token = struct {...@@ -125,7 +126,6 @@ pub const Token = struct {
125 SlashEqual,126 SlashEqual,
126 Comma,127 Comma,
127 Ampersand,128 Ampersand,
128 AmpersandAmpersand,
129 AmpersandEqual,129 AmpersandEqual,
130 QuestionMark,130 QuestionMark,
131 AngleBracketLeft,131 AngleBracketLeft,
...@@ -486,7 +486,8 @@ pub const Tokenizer = struct {...@@ -486,7 +486,8 @@ pub const Tokenizer = struct {
486486
487 State.Ampersand => switch (c) {487 State.Ampersand => switch (c) {
488 '&' => {488 '&' => {
489 result.id = Token.Id.AmpersandAmpersand;489 result.id = Token.Id.Invalid_ampersands;
490 self.index += 1;
490 break;491 break;
491 },492 },
492 '=' => {493 '=' => {