authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-07-19 17:54:06-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-07-19 17:54:06-04:00
log8ea8cff4912bf66b8e95b7518877c7f55c7a9356
tree86e7d61f3ed055c54dbe49a5ed4ab9f2942d8d17
parent111d3792e0f2d92f7b9f73fb8265ee17fc229eb4
signaturelock-open Commit is signed but in an unrecognized format.

slightly simpler implementation

4 files changed, 135 insertions(+), 141 deletions(-)

doc/docgen.zig+118-120
......@@ -742,101 +742,101 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok
742742 const token = tokenizer.next();
743743 try writeEscaped(out, src[index..token.start]);
744744 switch (token.id) {
745 std.zig.Token.Id.Eof => break,
746
747 std.zig.Token.Id.Keyword_align,
748 std.zig.Token.Id.Keyword_and,
749 std.zig.Token.Id.Keyword_asm,
750 std.zig.Token.Id.Keyword_async,
751 std.zig.Token.Id.Keyword_await,
752 std.zig.Token.Id.Keyword_break,
753 std.zig.Token.Id.Keyword_cancel,
754 std.zig.Token.Id.Keyword_catch,
755 std.zig.Token.Id.Keyword_comptime,
756 std.zig.Token.Id.Keyword_const,
757 std.zig.Token.Id.Keyword_continue,
758 std.zig.Token.Id.Keyword_defer,
759 std.zig.Token.Id.Keyword_else,
760 std.zig.Token.Id.Keyword_enum,
761 std.zig.Token.Id.Keyword_errdefer,
762 std.zig.Token.Id.Keyword_error,
763 std.zig.Token.Id.Keyword_export,
764 std.zig.Token.Id.Keyword_extern,
765 std.zig.Token.Id.Keyword_for,
766 std.zig.Token.Id.Keyword_if,
767 std.zig.Token.Id.Keyword_inline,
768 std.zig.Token.Id.Keyword_nakedcc,
769 std.zig.Token.Id.Keyword_noalias,
770 std.zig.Token.Id.Keyword_or,
771 std.zig.Token.Id.Keyword_orelse,
772 std.zig.Token.Id.Keyword_packed,
773 std.zig.Token.Id.Keyword_promise,
774 std.zig.Token.Id.Keyword_pub,
775 std.zig.Token.Id.Keyword_resume,
776 std.zig.Token.Id.Keyword_return,
777 std.zig.Token.Id.Keyword_linksection,
778 std.zig.Token.Id.Keyword_stdcallcc,
779 std.zig.Token.Id.Keyword_struct,
780 std.zig.Token.Id.Keyword_suspend,
781 std.zig.Token.Id.Keyword_switch,
782 std.zig.Token.Id.Keyword_test,
783 std.zig.Token.Id.Keyword_threadlocal,
784 std.zig.Token.Id.Keyword_try,
785 std.zig.Token.Id.Keyword_union,
786 std.zig.Token.Id.Keyword_unreachable,
787 std.zig.Token.Id.Keyword_usingnamespace,
788 std.zig.Token.Id.Keyword_var,
789 std.zig.Token.Id.Keyword_volatile,
790 std.zig.Token.Id.Keyword_allowzero,
791 std.zig.Token.Id.Keyword_while,
745 .Eof => break,
746
747 .Keyword_align,
748 .Keyword_and,
749 .Keyword_asm,
750 .Keyword_async,
751 .Keyword_await,
752 .Keyword_break,
753 .Keyword_cancel,
754 .Keyword_catch,
755 .Keyword_comptime,
756 .Keyword_const,
757 .Keyword_continue,
758 .Keyword_defer,
759 .Keyword_else,
760 .Keyword_enum,
761 .Keyword_errdefer,
762 .Keyword_error,
763 .Keyword_export,
764 .Keyword_extern,
765 .Keyword_for,
766 .Keyword_if,
767 .Keyword_inline,
768 .Keyword_nakedcc,
769 .Keyword_noalias,
770 .Keyword_or,
771 .Keyword_orelse,
772 .Keyword_packed,
773 .Keyword_promise,
774 .Keyword_pub,
775 .Keyword_resume,
776 .Keyword_return,
777 .Keyword_linksection,
778 .Keyword_stdcallcc,
779 .Keyword_struct,
780 .Keyword_suspend,
781 .Keyword_switch,
782 .Keyword_test,
783 .Keyword_threadlocal,
784 .Keyword_try,
785 .Keyword_union,
786 .Keyword_unreachable,
787 .Keyword_usingnamespace,
788 .Keyword_var,
789 .Keyword_volatile,
790 .Keyword_allowzero,
791 .Keyword_while,
792792 => {
793793 try out.write("<span class=\"tok-kw\">");
794794 try writeEscaped(out, src[token.start..token.end]);
795795 try out.write("</span>");
796796 },
797797
798 std.zig.Token.Id.Keyword_fn => {
798 .Keyword_fn => {
799799 try out.write("<span class=\"tok-kw\">");
800800 try writeEscaped(out, src[token.start..token.end]);
801801 try out.write("</span>");
802802 next_tok_is_fn = true;
803803 },
804804
805 std.zig.Token.Id.Keyword_undefined,
806 std.zig.Token.Id.Keyword_null,
807 std.zig.Token.Id.Keyword_true,
808 std.zig.Token.Id.Keyword_false,
805 .Keyword_undefined,
806 .Keyword_null,
807 .Keyword_true,
808 .Keyword_false,
809809 => {
810810 try out.write("<span class=\"tok-null\">");
811811 try writeEscaped(out, src[token.start..token.end]);
812812 try out.write("</span>");
813813 },
814814
815 std.zig.Token.Id.StringLiteral,
816 std.zig.Token.Id.MultilineStringLiteralLine,
817 std.zig.Token.Id.CharLiteral,
815 .StringLiteral,
816 .MultilineStringLiteralLine,
817 .CharLiteral,
818818 => {
819819 try out.write("<span class=\"tok-str\">");
820820 try writeEscaped(out, src[token.start..token.end]);
821821 try out.write("</span>");
822822 },
823823
824 std.zig.Token.Id.Builtin => {
824 .Builtin => {
825825 try out.write("<span class=\"tok-builtin\">");
826826 try writeEscaped(out, src[token.start..token.end]);
827827 try out.write("</span>");
828828 },
829829
830 std.zig.Token.Id.LineComment,
831 std.zig.Token.Id.DocComment,
832 std.zig.Token.Id.ShebangLine,
830 .LineComment,
831 .DocComment,
832 .ShebangLine,
833833 => {
834834 try out.write("<span class=\"tok-comment\">");
835835 try writeEscaped(out, src[token.start..token.end]);
836836 try out.write("</span>");
837837 },
838838
839 std.zig.Token.Id.Identifier => {
839 .Identifier => {
840840 if (prev_tok_was_fn) {
841841 try out.write("<span class=\"tok-fn\">");
842842 try writeEscaped(out, src[token.start..token.end]);
......@@ -864,74 +864,72 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok
864864 }
865865 },
866866
867 std.zig.Token.Id.IntegerLiteral,
868 std.zig.Token.Id.FloatLiteral,
867 .IntegerLiteral,
868 .FloatLiteral,
869869 => {
870870 try out.write("<span class=\"tok-number\">");
871871 try writeEscaped(out, src[token.start..token.end]);
872872 try out.write("</span>");
873873 },
874874
875 std.zig.Token.Id.Bang,
876 std.zig.Token.Id.Pipe,
877 std.zig.Token.Id.PipePipe,
878 std.zig.Token.Id.PipeEqual,
879 std.zig.Token.Id.Equal,
880 std.zig.Token.Id.EqualEqual,
881 std.zig.Token.Id.EqualAngleBracketRight,
882 std.zig.Token.Id.BangEqual,
883 std.zig.Token.Id.LParen,
884 std.zig.Token.Id.RParen,
885 std.zig.Token.Id.Semicolon,
886 std.zig.Token.Id.Percent,
887 std.zig.Token.Id.PercentEqual,
888 std.zig.Token.Id.LBrace,
889 std.zig.Token.Id.RBrace,
890 std.zig.Token.Id.LBracket,
891 std.zig.Token.Id.RBracket,
892 std.zig.Token.Id.Period,
893 std.zig.Token.Id.Ellipsis2,
894 std.zig.Token.Id.Ellipsis3,
895 std.zig.Token.Id.Caret,
896 std.zig.Token.Id.CaretEqual,
897 std.zig.Token.Id.Plus,
898 std.zig.Token.Id.PlusPlus,
899 std.zig.Token.Id.PlusEqual,
900 std.zig.Token.Id.PlusPercent,
901 std.zig.Token.Id.PlusPercentEqual,
902 std.zig.Token.Id.Minus,
903 std.zig.Token.Id.MinusEqual,
904 std.zig.Token.Id.MinusPercent,
905 std.zig.Token.Id.MinusPercentEqual,
906 std.zig.Token.Id.Asterisk,
907 std.zig.Token.Id.AsteriskEqual,
908 std.zig.Token.Id.AsteriskAsterisk,
909 std.zig.Token.Id.AsteriskPercent,
910 std.zig.Token.Id.AsteriskPercentEqual,
911 std.zig.Token.Id.Arrow,
912 std.zig.Token.Id.Colon,
913 std.zig.Token.Id.Slash,
914 std.zig.Token.Id.SlashEqual,
915 std.zig.Token.Id.Comma,
916 std.zig.Token.Id.Ampersand,
917 std.zig.Token.Id.AmpersandEqual,
918 std.zig.Token.Id.QuestionMark,
919 std.zig.Token.Id.AngleBracketLeft,
920 std.zig.Token.Id.AngleBracketLeftEqual,
921 std.zig.Token.Id.AngleBracketAngleBracketLeft,
922 std.zig.Token.Id.AngleBracketAngleBracketLeftEqual,
923 std.zig.Token.Id.AngleBracketRight,
924 std.zig.Token.Id.AngleBracketRightEqual,
925 std.zig.Token.Id.AngleBracketAngleBracketRight,
926 std.zig.Token.Id.AngleBracketAngleBracketRightEqual,
927 std.zig.Token.Id.Tilde,
928 std.zig.Token.Id.BracketStarBracket,
929 std.zig.Token.Id.BracketStarCBracket,
875 .Bang,
876 .Pipe,
877 .PipePipe,
878 .PipeEqual,
879 .Equal,
880 .EqualEqual,
881 .EqualAngleBracketRight,
882 .BangEqual,
883 .LParen,
884 .RParen,
885 .Semicolon,
886 .Percent,
887 .PercentEqual,
888 .LBrace,
889 .RBrace,
890 .LBracket,
891 .RBracket,
892 .Period,
893 .Ellipsis2,
894 .Ellipsis3,
895 .Caret,
896 .CaretEqual,
897 .Plus,
898 .PlusPlus,
899 .PlusEqual,
900 .PlusPercent,
901 .PlusPercentEqual,
902 .Minus,
903 .MinusEqual,
904 .MinusPercent,
905 .MinusPercentEqual,
906 .Asterisk,
907 .AsteriskEqual,
908 .AsteriskAsterisk,
909 .AsteriskPercent,
910 .AsteriskPercentEqual,
911 .Arrow,
912 .Colon,
913 .Slash,
914 .SlashEqual,
915 .Comma,
916 .Ampersand,
917 .AmpersandEqual,
918 .QuestionMark,
919 .AngleBracketLeft,
920 .AngleBracketLeftEqual,
921 .AngleBracketAngleBracketLeft,
922 .AngleBracketAngleBracketLeftEqual,
923 .AngleBracketRight,
924 .AngleBracketRightEqual,
925 .AngleBracketAngleBracketRight,
926 .AngleBracketAngleBracketRightEqual,
927 .Tilde,
928 .BracketStarBracket,
929 .BracketStarCBracket,
930930 => try writeEscaped(out, src[token.start..token.end]),
931931
932 std.zig.Token.Id.AmpersandAmpersand,
933 std.zig.Token.Id.Invalid,
934 => return parseError(
932 .Invalid, .Invalid_ampersands => return parseError(
935933 docgen_tokenizer,
936934 source_token,
937935 "syntax error",
std/zig/ast.zig+13-6
......@@ -113,7 +113,6 @@ pub const Tree = struct {
113113
114114pub const Error = union(enum) {
115115 InvalidToken: InvalidToken,
116 InvalidAmpersandAmpersand: InvalidAmpersandAmpersand,
117116 ExpectedContainerMembers: ExpectedContainerMembers,
118117 ExpectedStringLiteral: ExpectedStringLiteral,
119118 ExpectedIntegerLiteral: ExpectedIntegerLiteral,
......@@ -161,7 +160,6 @@ pub const Error = union(enum) {
161160 pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {
162161 switch (self.*) {
163162 .InvalidToken => |*x| return x.render(tokens, stream),
164 .InvalidAmpersandAmpersand => |*x| return x.render(tokens, stream),
165163 .ExpectedContainerMembers => |*x| return x.render(tokens, stream),
166164 .ExpectedStringLiteral => |*x| return x.render(tokens, stream),
167165 .ExpectedIntegerLiteral => |*x| return x.render(tokens, stream),
......@@ -211,7 +209,6 @@ pub const Error = union(enum) {
211209 pub fn loc(self: *const Error) TokenIndex {
212210 switch (self.*) {
213211 .InvalidToken => |x| return x.token,
214 .InvalidAmpersandAmpersand => |x| return x.token,
215212 .ExpectedContainerMembers => |x| return x.token,
216213 .ExpectedStringLiteral => |x| return x.token,
217214 .ExpectedIntegerLiteral => |x| return x.token,
......@@ -292,7 +289,6 @@ pub const Error = union(enum) {
292289 pub const ExpectedDerefOrUnwrap = SingleTokenError("Expected pointer dereference or optional unwrap, found {}");
293290 pub const ExpectedSuffixOp = SingleTokenError("Expected pointer dereference, optional unwrap, or field access, found {}");
294291
295 pub const InvalidAmpersandAmpersand = SimpleError("Invalid token '&&', 'and' performs boolean AND");
296292 pub const ExpectedParamType = SimpleError("Expected parameter type");
297293 pub const ExpectedPubItem = SimpleError("Pub must be followed by fn decl, var decl, or container member");
298294 pub const UnattachedDocComment = SimpleError("Unattached documentation comment");
......@@ -322,8 +318,19 @@ pub const Error = union(enum) {
322318 expected_id: Token.Id,
323319
324320 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {
325 const token_name = @tagName(tokens.at(self.token).id);
326 return stream.print("expected {}, found {}", @tagName(self.expected_id), token_name);
321 const found_token = tokens.at(self.token);
322 switch (found_token.id) {
323 .Invalid_ampersands => {
324 return stream.print("`&&` is invalid. Note that `and` is boolean AND.");
325 },
326 .Invalid => {
327 return stream.print("expected {}, found invalid bytes", @tagName(self.expected_id));
328 },
329 else => {
330 const token_name = @tagName(found_token.id);
331 return stream.print("expected {}, found {}", @tagName(self.expected_id), token_name);
332 },
333 }
327334 }
328335 };
329336
std/zig/parse.zig+1-13
......@@ -774,7 +774,7 @@ fn parseBoolAndExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
774774 arena,
775775 it,
776776 tree,
777 parseAmpersandAmpersandOp,
777 SimpleBinOpParseFn(.Keyword_and, .BoolAnd),
778778 parseCompareExpr,
779779 .Infinitely,
780780 );
......@@ -2687,18 +2687,6 @@ fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) No
26872687
26882688// Helper parsers not included in the grammar
26892689
2690fn parseAmpersandAmpersandOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2691 const op_parse_and = SimpleBinOpParseFn(.Keyword_and, Node.InfixOp.Op.BoolAnd);
2692 const op_token = eatToken(it, .AmpersandAmpersand);
2693 if (op_token != null) {
2694 try tree.errors.push(AstError{
2695 .InvalidAmpersandAmpersand = AstError.InvalidAmpersandAmpersand{ .token = it.index },
2696 });
2697 return error.ParseError;
2698 }
2699 return op_parse_and(arena, it, tree);
2700}
2701
27022690fn parseBuiltinCall(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
27032691 const token = eatToken(it, .Builtin) orelse return null;
27042692 const params = (try parseFnCallArguments(arena, it, tree)) orelse {
std/zig/tokenizer.zig+3-2
......@@ -77,6 +77,7 @@ pub const Token = struct {
7777
7878 pub const Id = enum {
7979 Invalid,
80 Invalid_ampersands,
8081 Identifier,
8182 StringLiteral,
8283 MultilineStringLiteralLine,
......@@ -125,7 +126,6 @@ pub const Token = struct {
125126 SlashEqual,
126127 Comma,
127128 Ampersand,
128 AmpersandAmpersand,
129129 AmpersandEqual,
130130 QuestionMark,
131131 AngleBracketLeft,
......@@ -486,7 +486,8 @@ pub const Tokenizer = struct {
486486
487487 State.Ampersand => switch (c) {
488488 '&' => {
489 result.id = Token.Id.AmpersandAmpersand;
489 result.id = Token.Id.Invalid_ampersands;
490 self.index += 1;
490491 break;
491492 },
492493 '=' => {