authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-10-06 16:39:27-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-10-06 16:39:27-04:00
log8e2c441b2eae7af27fe311c4539f6932a7f47cea
tree9bb0bd70760ab320431ada61169520689a65a4c1
parent86171afb9b7d7f1c1d5444f894e273ca40e87147
signaturelock-open Commit is signed but in an unrecognized format.

stage1 parser supports doc comments

8 files changed, 133 insertions(+), 17 deletions(-)

doc/langref.html.in+1-1
......@@ -10097,7 +10097,7 @@ TopLevelComptime &lt;- KEYWORD_comptime BlockExpr
1009710097TopLevelDecl
1009810098 &lt;- (KEYWORD_export / KEYWORD_extern STRINGLITERAL? / KEYWORD_inline)? FnProto (SEMICOLON / Block)
1009910099 / (KEYWORD_export / KEYWORD_extern STRINGLITERAL?)? KEYWORD_threadlocal? VarDecl
10100 / KEYWORD_use Expr SEMICOLON
10100 / KEYWORD_usingnamespace Expr SEMICOLON
1010110101
1010210102FnProto &lt;- FnCC? KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)
1010310103
src/all_types.hpp+25-9
......@@ -589,6 +589,7 @@ enum NodeType {
589589 NodeTypeIfErrorExpr,
590590 NodeTypeIfOptional,
591591 NodeTypeErrorSetDecl,
592 NodeTypeErrorSetField,
592593 NodeTypeResume,
593594 NodeTypeAwaitExpr,
594595 NodeTypeSuspend,
......@@ -612,16 +613,10 @@ enum FnInline {
612613};
613614
614615struct AstNodeFnProto {
615 VisibMod visib_mod;
616616 Buf *name;
617617 ZigList<AstNode *> params;
618618 AstNode *return_type;
619619 Token *return_var_token;
620 bool is_var_args;
621 bool is_extern;
622 bool is_export;
623 FnInline fn_inline;
624 CallingConvention cc;
625620 AstNode *fn_def_node;
626621 // populated if this is an extern declaration
627622 Buf *lib_name;
......@@ -629,8 +624,16 @@ struct AstNodeFnProto {
629624 AstNode *align_expr;
630625 // populated if the "section(S)" is present
631626 AstNode *section_expr;
627 Buf doc_comments;
632628
629 FnInline fn_inline;
630 CallingConvention cc;
631
632 VisibMod visib_mod;
633633 bool auto_err_set;
634 bool is_var_args;
635 bool is_extern;
636 bool is_export;
634637};
635638
636639struct AstNodeFnDef {
......@@ -642,6 +645,7 @@ struct AstNodeParamDecl {
642645 Buf *name;
643646 AstNode *type;
644647 Token *var_token;
648 Buf doc_comments;
645649 bool is_noalias;
646650 bool is_comptime;
647651 bool is_var_args;
......@@ -684,6 +688,7 @@ struct AstNodeVariableDeclaration {
684688 // populated if the "section(S)" is present
685689 AstNode *section_expr;
686690 Token *threadlocal_tok;
691 Buf doc_comments;
687692
688693 VisibMod visib_mod;
689694 bool is_const;
......@@ -957,25 +962,35 @@ enum ContainerLayout {
957962};
958963
959964struct AstNodeContainerDecl {
960 ContainerKind kind;
965 AstNode *init_arg_expr; // enum(T), struct(endianness), or union(T), or union(enum(T))
961966 ZigList<AstNode *> fields;
962967 ZigList<AstNode *> decls;
968
969 ContainerKind kind;
963970 ContainerLayout layout;
964 AstNode *init_arg_expr; // enum(T), struct(endianness), or union(T), or union(enum(T))
971
965972 bool auto_enum, is_root; // union(enum)
966973};
967974
975struct AstNodeErrorSetField {
976 Buf doc_comments;
977 AstNode *field_name;
978};
979
968980struct AstNodeErrorSetDecl {
981 // Each AstNode could be AstNodeErrorSetField or just AstNodeSymbolExpr to save memory
969982 ZigList<AstNode *> decls;
970983};
971984
972985struct AstNodeStructField {
973 VisibMod visib_mod;
974986 Buf *name;
975987 AstNode *type;
976988 AstNode *value;
977989 // populated if the "align(A)" is present
978990 AstNode *align_expr;
991 Buf doc_comments;
992
993 VisibMod visib_mod;
979994};
980995
981996struct AstNodeStringLiteral {
......@@ -1126,6 +1141,7 @@ struct AstNode {
11261141 AstNodeInferredArrayType inferred_array_type;
11271142 AstNodeErrorType error_type;
11281143 AstNodeErrorSetDecl err_set_decl;
1144 AstNodeErrorSetField err_set_field;
11291145 AstNodeResumeExpr resume_expr;
11301146 AstNodeAwaitExpr await_expr;
11311147 AstNodeSuspend suspend;
src/analyze.cpp+1
......@@ -3572,6 +3572,7 @@ void scan_decls(CodeGen *g, ScopeDecls *decls_scope, AstNode *node) {
35723572 case NodeTypeSuspend:
35733573 case NodeTypeEnumLiteral:
35743574 case NodeTypeAnyFrameType:
3575 case NodeTypeErrorSetField:
35753576 zig_unreachable();
35763577 }
35773578}
src/ast_render.cpp+3
......@@ -266,6 +266,8 @@ static const char *node_type_str(NodeType node_type) {
266266 return "AnyFrameType";
267267 case NodeTypeEnumLiteral:
268268 return "EnumLiteral";
269 case NodeTypeErrorSetField:
270 return "ErrorSetField";
269271 }
270272 zig_unreachable();
271273}
......@@ -1177,6 +1179,7 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) {
11771179 case NodeTypeTestDecl:
11781180 case NodeTypeStructField:
11791181 case NodeTypeUsingNamespace:
1182 case NodeTypeErrorSetField:
11801183 zig_panic("TODO more ast rendering");
11811184 }
11821185}
src/ir.cpp+10-1
......@@ -7945,7 +7945,15 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A
79457945 ErrorTableEntry **errors = allocate<ErrorTableEntry *>(irb->codegen->errors_by_index.length + err_count);
79467946
79477947 for (uint32_t i = 0; i < err_count; i += 1) {
7948 AstNode *symbol_node = node->data.err_set_decl.decls.at(i);
7948 AstNode *field_node = node->data.err_set_decl.decls.at(i);
7949 AstNode *symbol_node;
7950 if (field_node->type == NodeTypeSymbol) {
7951 symbol_node = field_node;
7952 } else if (field_node->type == NodeTypeErrorSetField) {
7953 symbol_node = field_node->data.err_set_field.field_name;
7954 } else {
7955 zig_unreachable();
7956 }
79497957 assert(symbol_node->type == NodeTypeSymbol);
79507958 Buf *err_name = symbol_node->data.symbol_expr.symbol;
79517959 ErrorTableEntry *err = allocate<ErrorTableEntry>(1);
......@@ -8116,6 +8124,7 @@ static IrInstruction *ir_gen_node_raw(IrBuilder *irb, AstNode *node, Scope *scop
81168124 case NodeTypeSwitchProng:
81178125 case NodeTypeSwitchRange:
81188126 case NodeTypeStructField:
8127 case NodeTypeErrorSetField:
81198128 case NodeTypeFnDef:
81208129 case NodeTypeTestDecl:
81218130 zig_unreachable();
src/parser.cpp+41-4
......@@ -37,7 +37,7 @@ static AstNode *ast_parse_root(ParseContext *pc);
3737static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc);
3838static AstNode *ast_parse_test_decl(ParseContext *pc);
3939static AstNode *ast_parse_top_level_comptime(ParseContext *pc);
40static AstNode *ast_parse_top_level_decl(ParseContext *pc, VisibMod visib_mod);
40static AstNode *ast_parse_top_level_decl(ParseContext *pc, VisibMod visib_mod, Buf *doc_comments);
4141static AstNode *ast_parse_fn_proto(ParseContext *pc);
4242static AstNode *ast_parse_var_decl(ParseContext *pc);
4343static AstNode *ast_parse_container_field(ParseContext *pc);
......@@ -497,6 +497,19 @@ static AstNode *ast_parse_root(ParseContext *pc) {
497497 return node;
498498}
499499
500static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) {
501 Token *doc_token = nullptr;
502 while ((doc_token = eat_token_if(pc, TokenIdDocComment))) {
503 if (buf->list.length == 0) {
504 buf_resize(buf, 0);
505 }
506 // chops off '///' and '\n'
507 buf_append_mem(buf, buf_ptr(pc->buf) + doc_token->start_pos + 3,
508 doc_token->end_pos - doc_token->start_pos - 4);
509 }
510 return doc_token;
511}
512
500513// ContainerMembers
501514// <- TestDecl ContainerMembers
502515// / TopLevelComptime ContainerMembers
......@@ -519,10 +532,13 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
519532 continue;
520533 }
521534
535 Buf doc_comment_buf = BUF_INIT;
536 ast_parse_doc_comments(pc, &doc_comment_buf);
537
522538 Token *visib_token = eat_token_if(pc, TokenIdKeywordPub);
523539 VisibMod visib_mod = visib_token != nullptr ? VisibModPub : VisibModPrivate;
524540
525 AstNode *top_level_decl = ast_parse_top_level_decl(pc, visib_mod);
541 AstNode *top_level_decl = ast_parse_top_level_decl(pc, visib_mod, &doc_comment_buf);
526542 if (top_level_decl != nullptr) {
527543 res.decls.append(top_level_decl);
528544 continue;
......@@ -532,6 +548,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
532548 if (container_field != nullptr) {
533549 assert(container_field->type == NodeTypeStructField);
534550 container_field->data.struct_field.visib_mod = visib_mod;
551 container_field->data.struct_field.doc_comments = doc_comment_buf;
535552 res.fields.append(container_field);
536553 if (eat_token_if(pc, TokenIdComma) != nullptr) {
537554 continue;
......@@ -581,7 +598,7 @@ static AstNode *ast_parse_top_level_comptime(ParseContext *pc) {
581598// <- (KEYWORD_export / KEYWORD_extern STRINGLITERAL? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)
582599// / (KEYWORD_export / KEYWORD_extern STRINGLITERAL?)? KEYWORD_threadlocal? VarDecl
583600// / KEYWORD_use Expr SEMICOLON
584static AstNode *ast_parse_top_level_decl(ParseContext *pc, VisibMod visib_mod) {
601static AstNode *ast_parse_top_level_decl(ParseContext *pc, VisibMod visib_mod, Buf *doc_comments) {
585602 Token *first = eat_token_if(pc, TokenIdKeywordExport);
586603 if (first == nullptr)
587604 first = eat_token_if(pc, TokenIdKeywordExtern);
......@@ -603,6 +620,7 @@ static AstNode *ast_parse_top_level_decl(ParseContext *pc, VisibMod visib_mod) {
603620 var_decl->column = first->start_column;
604621 var_decl->data.variable_declaration.threadlocal_tok = thread_local_kw;
605622 var_decl->data.variable_declaration.visib_mod = visib_mod;
623 var_decl->data.variable_declaration.doc_comments = *doc_comments;
606624 var_decl->data.variable_declaration.is_extern = first->id == TokenIdKeywordExtern;
607625 var_decl->data.variable_declaration.is_export = first->id == TokenIdKeywordExport;
608626 var_decl->data.variable_declaration.lib_name = token_buf(lib_name);
......@@ -623,6 +641,7 @@ static AstNode *ast_parse_top_level_decl(ParseContext *pc, VisibMod visib_mod) {
623641 fn_proto->line = first->start_line;
624642 fn_proto->column = first->start_column;
625643 fn_proto->data.fn_proto.visib_mod = visib_mod;
644 fn_proto->data.fn_proto.doc_comments = *doc_comments;
626645 fn_proto->data.fn_proto.is_extern = first->id == TokenIdKeywordExtern;
627646 fn_proto->data.fn_proto.is_export = first->id == TokenIdKeywordExport;
628647 switch (first->id) {
......@@ -657,6 +676,7 @@ static AstNode *ast_parse_top_level_decl(ParseContext *pc, VisibMod visib_mod) {
657676 if (var_decl != nullptr) {
658677 assert(var_decl->type == NodeTypeVariableDeclaration);
659678 var_decl->data.variable_declaration.visib_mod = visib_mod;
679 var_decl->data.variable_declaration.doc_comments = *doc_comments;
660680 var_decl->data.variable_declaration.threadlocal_tok = thread_local_kw;
661681 return var_decl;
662682 }
......@@ -672,6 +692,7 @@ static AstNode *ast_parse_top_level_decl(ParseContext *pc, VisibMod visib_mod) {
672692
673693 assert(fn_proto->type == NodeTypeFnProto);
674694 fn_proto->data.fn_proto.visib_mod = visib_mod;
695 fn_proto->data.fn_proto.doc_comments = *doc_comments;
675696 AstNode *res = fn_proto;
676697 if (body != nullptr) {
677698 res = ast_create_node_copy_line_info(pc, NodeTypeFnDef, fn_proto);
......@@ -1719,11 +1740,20 @@ static AstNode *ast_parse_error_set_decl(ParseContext *pc) {
17191740 }
17201741
17211742 ZigList<AstNode *> decls = ast_parse_list<AstNode>(pc, TokenIdComma, [](ParseContext *context) {
1743 Buf doc_comment_buf = BUF_INIT;
1744 Token *doc_token = ast_parse_doc_comments(context, &doc_comment_buf);
17221745 Token *ident = eat_token_if(context, TokenIdSymbol);
17231746 if (ident == nullptr)
17241747 return (AstNode*)nullptr;
17251748
1726 return token_symbol(context, ident);
1749 AstNode *symbol_node = token_symbol(context, ident);
1750 if (doc_token == nullptr)
1751 return symbol_node;
1752
1753 AstNode *field_node = ast_create_node(context, NodeTypeErrorSetField, doc_token);
1754 field_node->data.err_set_field.field_name = symbol_node;
1755 field_node->data.err_set_field.doc_comments = doc_comment_buf;
1756 return field_node;
17271757 });
17281758 expect_token(pc, TokenIdRBrace);
17291759
......@@ -2057,6 +2087,9 @@ static Optional<AstNodeFnProto> ast_parse_fn_cc(ParseContext *pc) {
20572087
20582088// ParamDecl <- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType
20592089static AstNode *ast_parse_param_decl(ParseContext *pc) {
2090 Buf doc_comments = BUF_INIT;
2091 ast_parse_doc_comments(pc, &doc_comments);
2092
20602093 Token *first = eat_token_if(pc, TokenIdKeywordNoAlias);
20612094 if (first == nullptr)
20622095 first = eat_token_if(pc, TokenIdKeywordCompTime);
......@@ -2089,6 +2122,7 @@ static AstNode *ast_parse_param_decl(ParseContext *pc) {
20892122 res->line = first->start_line;
20902123 res->column = first->start_column;
20912124 res->data.param_decl.name = token_buf(name);
2125 res->data.param_decl.doc_comments = doc_comments;
20922126 res->data.param_decl.is_noalias = first->id == TokenIdKeywordNoAlias;
20932127 res->data.param_decl.is_comptime = first->id == TokenIdKeywordCompTime;
20942128 return res;
......@@ -3029,6 +3063,9 @@ void ast_visit_node_children(AstNode *node, void (*visit)(AstNode **, void *cont
30293063 case NodeTypeErrorSetDecl:
30303064 visit_node_list(&node->data.err_set_decl.decls, visit, context);
30313065 break;
3066 case NodeTypeErrorSetField:
3067 visit_field(&node->data.err_set_field.field_name, visit, context);
3068 break;
30323069 case NodeTypeResume:
30333070 visit_field(&node->data.resume_expr.expr, visit, context);
30343071 break;
src/tokenizer.cpp+51-2
......@@ -196,6 +196,8 @@ enum TokenizeState {
196196 TokenizeStateSawStar,
197197 TokenizeStateSawStarPercent,
198198 TokenizeStateSawSlash,
199 TokenizeStateSawSlash2,
200 TokenizeStateSawSlash3,
199201 TokenizeStateSawBackslash,
200202 TokenizeStateSawPercent,
201203 TokenizeStateSawPlus,
......@@ -206,6 +208,7 @@ enum TokenizeState {
206208 TokenizeStateSawCaret,
207209 TokenizeStateSawBar,
208210 TokenizeStateSawBarBar,
211 TokenizeStateDocComment,
209212 TokenizeStateLineComment,
210213 TokenizeStateLineString,
211214 TokenizeStateLineStringEnd,
......@@ -910,8 +913,7 @@ void tokenize(Buf *buf, Tokenization *out) {
910913 case TokenizeStateSawSlash:
911914 switch (c) {
912915 case '/':
913 cancel_token(&t);
914 t.state = TokenizeStateLineComment;
916 t.state = TokenizeStateSawSlash2;
915917 break;
916918 case '=':
917919 set_token_id(&t, t.cur_tok, TokenIdDivEq);
......@@ -925,6 +927,38 @@ void tokenize(Buf *buf, Tokenization *out) {
925927 continue;
926928 }
927929 break;
930 case TokenizeStateSawSlash2:
931 switch (c) {
932 case '/':
933 t.state = TokenizeStateSawSlash3;
934 break;
935 case '\n':
936 cancel_token(&t);
937 t.state = TokenizeStateStart;
938 break;
939 default:
940 cancel_token(&t);
941 t.state = TokenizeStateLineComment;
942 break;
943 }
944 break;
945 case TokenizeStateSawSlash3:
946 switch (c) {
947 case '/':
948 cancel_token(&t);
949 t.state = TokenizeStateLineComment;
950 break;
951 case '\n':
952 set_token_id(&t, t.cur_tok, TokenIdDocComment);
953 end_token(&t);
954 t.state = TokenizeStateStart;
955 break;
956 default:
957 set_token_id(&t, t.cur_tok, TokenIdDocComment);
958 t.state = TokenizeStateDocComment;
959 break;
960 }
961 break;
928962 case TokenizeStateSawBackslash:
929963 switch (c) {
930964 case '\\':
......@@ -1004,6 +1038,17 @@ void tokenize(Buf *buf, Tokenization *out) {
10041038 break;
10051039 }
10061040 break;
1041 case TokenizeStateDocComment:
1042 switch (c) {
1043 case '\n':
1044 end_token(&t);
1045 t.state = TokenizeStateStart;
1046 break;
1047 default:
1048 // do nothing
1049 break;
1050 }
1051 break;
10071052 case TokenizeStateSymbolFirstC:
10081053 switch (c) {
10091054 case '"':
......@@ -1466,6 +1511,7 @@ void tokenize(Buf *buf, Tokenization *out) {
14661511 case TokenizeStateLineStringEnd:
14671512 case TokenizeStateSawBarBar:
14681513 case TokenizeStateLBracket:
1514 case TokenizeStateDocComment:
14691515 end_token(&t);
14701516 break;
14711517 case TokenizeStateSawDotDot:
......@@ -1478,6 +1524,8 @@ void tokenize(Buf *buf, Tokenization *out) {
14781524 tokenize_error(&t, "unexpected EOF");
14791525 break;
14801526 case TokenizeStateLineComment:
1527 case TokenizeStateSawSlash2:
1528 case TokenizeStateSawSlash3:
14811529 break;
14821530 }
14831531 if (t.state != TokenizeStateError) {
......@@ -1524,6 +1572,7 @@ const char * token_name(TokenId id) {
15241572 case TokenIdComma: return ",";
15251573 case TokenIdDash: return "-";
15261574 case TokenIdDivEq: return "/=";
1575 case TokenIdDocComment: return "DocComment";
15271576 case TokenIdDot: return ".";
15281577 case TokenIdEllipsis2: return "..";
15291578 case TokenIdEllipsis3: return "...";
src/tokenizer.hpp+1
......@@ -42,6 +42,7 @@ enum TokenId {
4242 TokenIdComma,
4343 TokenIdDash,
4444 TokenIdDivEq,
45 TokenIdDocComment,
4546 TokenIdDot,
4647 TokenIdEllipsis2,
4748 TokenIdEllipsis3,