authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-11-15 14:12:14+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-11-15 15:02:51+02:00
loge509d21f39af726da3c6001b0c20f2c765be07a5
treeeff4c0f8081c81065a9bbf9d1ab0f4e3d7aec0ab
parentb92f42d1f4b91bb343558f72af84ea052da4ac98
signaturelock-open Commit is signed but in an unrecognized format.

implemented container doc comments in stage 1

4 files changed, 56 insertions(+), 1 deletions(-)

src/all_types.hpp+1
...@@ -968,6 +968,7 @@ struct AstNodeContainerDecl {...@@ -968,6 +968,7 @@ struct AstNodeContainerDecl {
968 AstNode *init_arg_expr; // enum(T), struct(endianness), or union(T), or union(enum(T))968 AstNode *init_arg_expr; // enum(T), struct(endianness), or union(T), or union(enum(T))
969 ZigList<AstNode *> fields;969 ZigList<AstNode *> fields;
970 ZigList<AstNode *> decls;970 ZigList<AstNode *> decls;
971 Buf doc_comments;
971972
972 ContainerKind kind;973 ContainerKind kind;
973 ContainerLayout layout;974 ContainerLayout layout;
src/parser.cpp+22-1
...@@ -493,6 +493,7 @@ static AstNode *ast_parse_root(ParseContext *pc) {...@@ -493,6 +493,7 @@ static AstNode *ast_parse_root(ParseContext *pc) {
493 node->data.container_decl.layout = ContainerLayoutAuto;493 node->data.container_decl.layout = ContainerLayoutAuto;
494 node->data.container_decl.kind = ContainerKindStruct;494 node->data.container_decl.kind = ContainerKindStruct;
495 node->data.container_decl.is_root = true;495 node->data.container_decl.is_root = true;
496 node->data.container_decl.doc_comments = members.doc_comments;
496497
497 return node;498 return node;
498}499}
...@@ -514,6 +515,21 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) {...@@ -514,6 +515,21 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) {
514 return first_doc_token;515 return first_doc_token;
515}516}
516517
518static void ast_parse_container_doc_comments(ParseContext *pc, Buf *buf) {
519 if (buf_len(buf) != 0 && peek_token(pc)->id == TokenIdContainerDocComment) {
520 buf_append_char(buf, '\n');
521 }
522 Token *doc_token = nullptr;
523 while ((doc_token = eat_token_if(pc, TokenIdContainerDocComment))) {
524 if (buf->list.length == 0) {
525 buf_resize(buf, 0);
526 }
527 // chops off '//!' but leaves '\n'
528 buf_append_mem(buf, buf_ptr(pc->buf) + doc_token->start_pos + 3,
529 doc_token->end_pos - doc_token->start_pos - 3);
530 }
531}
532
517// ContainerMembers533// ContainerMembers
518// <- TestDecl ContainerMembers534// <- TestDecl ContainerMembers
519// / TopLevelComptime ContainerMembers535// / TopLevelComptime ContainerMembers
...@@ -523,7 +539,11 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) {...@@ -523,7 +539,11 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) {
523// /539// /
524static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {540static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
525 AstNodeContainerDecl res = {};541 AstNodeContainerDecl res = {};
542 Buf tld_doc_comment_buf = BUF_INIT;
543 buf_resize(&tld_doc_comment_buf, 0);
526 for (;;) {544 for (;;) {
545 ast_parse_container_doc_comments(pc, &tld_doc_comment_buf);
546
527 AstNode *test_decl = ast_parse_test_decl(pc);547 AstNode *test_decl = ast_parse_test_decl(pc);
528 if (test_decl != nullptr) {548 if (test_decl != nullptr) {
529 res.decls.append(test_decl);549 res.decls.append(test_decl);
...@@ -566,7 +586,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {...@@ -566,7 +586,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
566586
567 break;587 break;
568 }588 }
569589 res.doc_comments = tld_doc_comment_buf;
570 return res;590 return res;
571}591}
572592
...@@ -2797,6 +2817,7 @@ static AstNode *ast_parse_container_decl_auto(ParseContext *pc) {...@@ -2797,6 +2817,7 @@ static AstNode *ast_parse_container_decl_auto(ParseContext *pc) {
27972817
2798 res->data.container_decl.fields = members.fields;2818 res->data.container_decl.fields = members.fields;
2799 res->data.container_decl.decls = members.decls;2819 res->data.container_decl.decls = members.decls;
2820 res->data.container_decl.doc_comments = members.doc_comments;
2800 return res;2821 return res;
2801}2822}
28022823
src/tokenizer.cpp+32
...@@ -198,6 +198,7 @@ enum TokenizeState {...@@ -198,6 +198,7 @@ enum TokenizeState {
198 TokenizeStateSawSlash,198 TokenizeStateSawSlash,
199 TokenizeStateSawSlash2,199 TokenizeStateSawSlash2,
200 TokenizeStateSawSlash3,200 TokenizeStateSawSlash3,
201 TokenizeStateSawSlashBang,
201 TokenizeStateSawBackslash,202 TokenizeStateSawBackslash,
202 TokenizeStateSawPercent,203 TokenizeStateSawPercent,
203 TokenizeStateSawPlus,204 TokenizeStateSawPlus,
...@@ -209,6 +210,7 @@ enum TokenizeState {...@@ -209,6 +210,7 @@ enum TokenizeState {
209 TokenizeStateSawBar,210 TokenizeStateSawBar,
210 TokenizeStateSawBarBar,211 TokenizeStateSawBarBar,
211 TokenizeStateDocComment,212 TokenizeStateDocComment,
213 TokenizeStateContainerDocComment,
212 TokenizeStateLineComment,214 TokenizeStateLineComment,
213 TokenizeStateLineString,215 TokenizeStateLineString,
214 TokenizeStateLineStringEnd,216 TokenizeStateLineStringEnd,
...@@ -938,6 +940,9 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -938,6 +940,9 @@ void tokenize(Buf *buf, Tokenization *out) {
938 case '/':940 case '/':
939 t.state = TokenizeStateSawSlash3;941 t.state = TokenizeStateSawSlash3;
940 break;942 break;
943 case '!':
944 t.state = TokenizeStateSawSlashBang;
945 break;
941 case '\n':946 case '\n':
942 cancel_token(&t);947 cancel_token(&t);
943 t.state = TokenizeStateStart;948 t.state = TokenizeStateStart;
...@@ -965,6 +970,19 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -965,6 +970,19 @@ void tokenize(Buf *buf, Tokenization *out) {
965 break;970 break;
966 }971 }
967 break;972 break;
973 case TokenizeStateSawSlashBang:
974 switch (c) {
975 case '\n':
976 set_token_id(&t, t.cur_tok, TokenIdContainerDocComment);
977 end_token(&t);
978 t.state = TokenizeStateStart;
979 break;
980 default:
981 set_token_id(&t, t.cur_tok, TokenIdContainerDocComment);
982 t.state = TokenizeStateContainerDocComment;
983 break;
984 }
985 break;
968 case TokenizeStateSawBackslash:986 case TokenizeStateSawBackslash:
969 switch (c) {987 switch (c) {
970 case '\\':988 case '\\':
...@@ -1055,6 +1073,17 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1055,6 +1073,17 @@ void tokenize(Buf *buf, Tokenization *out) {
1055 break;1073 break;
1056 }1074 }
1057 break;1075 break;
1076 case TokenizeStateContainerDocComment:
1077 switch (c) {
1078 case '\n':
1079 end_token(&t);
1080 t.state = TokenizeStateStart;
1081 break;
1082 default:
1083 // do nothing
1084 break;
1085 }
1086 break;
1058 case TokenizeStateSymbolFirstC:1087 case TokenizeStateSymbolFirstC:
1059 switch (c) {1088 switch (c) {
1060 case '"':1089 case '"':
...@@ -1545,6 +1574,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1545,6 +1574,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1545 case TokenizeStateSawBarBar:1574 case TokenizeStateSawBarBar:
1546 case TokenizeStateLBracket:1575 case TokenizeStateLBracket:
1547 case TokenizeStateDocComment:1576 case TokenizeStateDocComment:
1577 case TokenizeStateContainerDocComment:
1548 end_token(&t);1578 end_token(&t);
1549 break;1579 break;
1550 case TokenizeStateSawDotDot:1580 case TokenizeStateSawDotDot:
...@@ -1559,6 +1589,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1559,6 +1589,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1559 case TokenizeStateLineComment:1589 case TokenizeStateLineComment:
1560 case TokenizeStateSawSlash2:1590 case TokenizeStateSawSlash2:
1561 case TokenizeStateSawSlash3:1591 case TokenizeStateSawSlash3:
1592 case TokenizeStateSawSlashBang:
1562 break;1593 break;
1563 }1594 }
1564 if (t.state != TokenizeStateError) {1595 if (t.state != TokenizeStateError) {
...@@ -1606,6 +1637,7 @@ const char * token_name(TokenId id) {...@@ -1606,6 +1637,7 @@ const char * token_name(TokenId id) {
1606 case TokenIdDash: return "-";1637 case TokenIdDash: return "-";
1607 case TokenIdDivEq: return "/=";1638 case TokenIdDivEq: return "/=";
1608 case TokenIdDocComment: return "DocComment";1639 case TokenIdDocComment: return "DocComment";
1640 case TokenIdContainerDocComment: return "ContainerDocComment";
1609 case TokenIdDot: return ".";1641 case TokenIdDot: return ".";
1610 case TokenIdDotStar: return ".*";1642 case TokenIdDotStar: return ".*";
1611 case TokenIdEllipsis2: return "..";1643 case TokenIdEllipsis2: return "..";
src/tokenizer.hpp+1
...@@ -43,6 +43,7 @@ enum TokenId {...@@ -43,6 +43,7 @@ enum TokenId {
43 TokenIdDash,43 TokenIdDash,
44 TokenIdDivEq,44 TokenIdDivEq,
45 TokenIdDocComment,45 TokenIdDocComment,
46 TokenIdContainerDocComment,
46 TokenIdDot,47 TokenIdDot,
47 TokenIdDotStar,48 TokenIdDotStar,
48 TokenIdEllipsis2,49 TokenIdEllipsis2,