authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-06 23:59:40-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-06 23:59:40-07:00
logae0e9685c690e9ddb31a28e5a9f827ea8da111ca
treeb79d30e0ecde77b181b7bb3e20f6f773073c07a7
parent72be61fc0a0cb552c8eec1d9d4b8ccacf64f491c

parser generator supports sub rules

3 files changed, 171 insertions(+), 32 deletions(-)

src/grammar.txt+15-11
......@@ -1,46 +1,50 @@
1Root : many(FnDecl) token(EOF) {
1Root<node> : many(FnDecl) token(EOF) {
22 $$ = ast_create_root($1);
33};
44
5FnDecl : token(Fn) token(Symbol) token(LParen) list(ParamDecl, token(Comma)) token(RParen) option(token(Arrow) Type) Block {
5FnDecl<node> : token(Fn) token(Symbol) token(LParen) list(ParamDecl, token(Comma)) token(RParen) option(ReturnType) Block {
66 $$ = ast_create_fn_decl($2, $4, $6, $7);
77};
88
9ParamDecl : token(Symbol) token(Colon) Type {
9ReturnType<node> : token(Arrow) Type {
10 $$ = $2;
11};
12
13ParamDecl<node> : token(Symbol) token(Colon) Type {
1014 $$ = ast_create_param_decl($1, $2);
1115};
1216
13Type : token(Symbol) {
17Type<node> : token(Symbol) {
1418 $$ = ast_create_symbol_type($1);
1519} | PointerType {
1620 $$ = $1;
1721};
1822
19PointerType : token(Star) token(Const) Type {
23PointerType<node> : token(Star) token(Const) Type {
2024 $$ = ast_create_pointer_type($2, $3);
2125} | token(Star) token(Mut) Type {
2226 $$ = ast_create_pointer_type($2, $3);
2327};
2428
25Block : token(LBrace) many(Statement) option(Expression) token(RBrace) {
29Block<node> : token(LBrace) many(Statement) option(Expression) token(RBrace) {
2630 $$ = ast_create_block($2, $3);
2731};
2832
29Statement : ExpressionStatement {
33Statement<node> : ExpressionStatement {
3034 $$ = $1;
3135} | ReturnStatement {
3236 $$ = $1;
3337};
3438
35ExpressionStatement : Expression token(Semicolon) {
39ExpressionStatement<node> : Expression token(Semicolon) {
3640 $$ = ast_create_expression_statement($1);
3741};
3842
39ReturnStatement : token(Return) Expression token(Semicolon) {
43ReturnStatement<node> : token(Return) Expression token(Semicolon) {
4044 $$ = ast_create_return_statement($2);
4145};
4246
43Expression : token(Number) {
47Expression<node> : token(Number) {
4448 $$ = ast_create_number($1);
4549} | token(String) {
4650 $$ = ast_create_string($1);
......@@ -48,6 +52,6 @@ Expression : token(Number) {
4852 $$ = $1;
4953};
5054
51FnCall : token(Symbol) token(LParen) list(Expression, token(Comma)) token(RParen) {
55FnCall<node> : token(Symbol) token(LParen) list(Expression, token(Comma)) token(RParen) {
5256 $$ = ast_create_fn_call($1, $3);
5357};
src/parsergen.cpp+153-18
......@@ -136,6 +136,7 @@ struct RuleTuple {
136136 Buf name;
137137 ZigList<RuleNode *> children;
138138 Buf body;
139 Buf union_field_name;
139140};
140141
141142struct RuleMany {
......@@ -161,6 +162,9 @@ struct RuleList {
161162
162163struct RuleSubRule {
163164 RuleNode *child;
165
166 // for lexer use only
167 Buf name;
164168};
165169
166170enum RuleNodeType {
......@@ -175,6 +179,8 @@ enum RuleNodeType {
175179
176180struct RuleNode {
177181 RuleNodeType type;
182 int lex_line;
183 int lex_column;
178184 union {
179185 RuleTuple tuple;
180186 RuleMany many;
......@@ -205,6 +211,7 @@ struct CodeGenCapture {
205211 Buf *body;
206212 bool is_root;
207213 Buf *field_names;
214 Buf *union_field_name;
208215};
209216
210217struct CodeGen {
......@@ -225,6 +232,8 @@ struct ParserState {
225232enum LexState {
226233 LexStateStart,
227234 LexStateRuleName,
235 LexStateRuleFieldNameStart,
236 LexStateRuleFieldName,
228237 LexStateWaitForColon,
229238 LexStateTupleRule,
230239 LexStateFnName,
......@@ -232,6 +241,7 @@ enum LexState {
232241 LexStateToken,
233242 LexStateBody,
234243 LexStateEndOrOr,
244 LexStateSubTupleName,
235245};
236246
237247struct LexStack {
......@@ -258,6 +268,8 @@ struct Gen {
258268 int lex_token_name_begin;
259269 int lex_body_begin;
260270 int lex_body_end;
271 int lex_sub_tuple_begin;
272 int lex_field_name_begin;
261273};
262274
263275static ParserState *create_state(Gen *g) {
......@@ -303,12 +315,13 @@ static void state_add_push_node(ParserState *state) {
303315 state_add_code(state, code);
304316}
305317
306static CodeGen *codegen_create_capture(Buf *body, bool is_root, int field_name_count) {
318static CodeGen *codegen_create_capture(Buf *body, bool is_root, int field_name_count, Buf *union_field_name) {
307319 CodeGen *code = allocate<CodeGen>(1);
308320 code->type = CodeGenTypeCapture;
309321 code->capture.body = body;
310322 code->capture.is_root = is_root;
311323 code->capture.field_names = allocate<Buf>(field_name_count);
324 code->capture.union_field_name = union_field_name;
312325 return code;
313326}
314327
......@@ -325,6 +338,7 @@ static void state_add_eat_token(ParserState *state) {
325338}
326339
327340static void gen(Gen *g, RuleNode *node, Buf *out_field_name) {
341 assert(node);
328342 switch (node->type) {
329343 case RuleNodeTypeToken:
330344 {
......@@ -346,13 +360,14 @@ static void gen(Gen *g, RuleNode *node, Buf *out_field_name) {
346360 break;
347361 case RuleNodeTypeTuple:
348362 {
349 buf_init_from_str(out_field_name, "node");
363 buf_init_from_buf(out_field_name, &node->tuple.union_field_name);
350364
351365 state_add_push_node(g->cur_state);
352366
353367 bool is_root = (node == g->root);
354368 int field_name_count = node->tuple.children.length;
355 CodeGen *code = codegen_create_capture(&node->tuple.body, is_root, field_name_count);
369 CodeGen *code = codegen_create_capture(&node->tuple.body, is_root, field_name_count,
370 &node->tuple.union_field_name);
356371
357372 for (int i = 0; i < node->tuple.children.length; i += 1) {
358373 RuleNode *child = node->tuple.children.at(i);
......@@ -411,7 +426,7 @@ static void lex_error(Gen *g, const char *format, ...) {
411426
412427 va_list ap;
413428 va_start(ap, format);
414 fprintf(stderr, "Error: Line %d, column %d: ", line, column);
429 fprintf(stderr, "Grammar Error: Line %d, column %d: ", line, column);
415430 vfprintf(stderr, format, ap);
416431 fprintf(stderr, "\n");
417432 va_end(ap);
......@@ -428,10 +443,16 @@ static void lex_pop_stack(Gen *g) {
428443 g->lex_stack.pop();
429444}
430445
446static RuleNode *create_rule_node(Gen *g) {
447 RuleNode *node = allocate<RuleNode>(1);
448 node->lex_line = g->lex_line;
449 node->lex_column = g->lex_column;
450 return node;
451}
431452
432453static void begin_rule(Gen *g) {
433454 assert(!g->lex_cur_rule);
434 g->lex_cur_rule = allocate<RuleNode>(1);
455 g->lex_cur_rule = create_rule_node(g);
435456 g->lex_cur_rule->type = RuleNodeTypeTuple;
436457 g->lex_cur_rule_begin = g->lex_pos;
437458
......@@ -452,6 +473,18 @@ static void end_rule_name(Gen *g) {
452473 buf_init_from_mem(&g->lex_cur_rule->tuple.name, ptr, len);
453474}
454475
476static void begin_rule_field_name(Gen *g) {
477 assert(g->lex_cur_rule);
478 g->lex_field_name_begin = g->lex_pos;
479}
480
481static void end_rule_field_name(Gen *g) {
482 assert(g->lex_cur_rule);
483 char *ptr = &buf_ptr(g->in_buf)[g->lex_field_name_begin];
484 int len = g->lex_pos - g->lex_field_name_begin;
485 buf_init_from_mem(&g->lex_cur_rule->tuple.union_field_name, ptr, len);
486}
487
455488static void begin_fn_name(Gen *g) {
456489 g->lex_fn_name_begin = g->lex_pos;
457490 lex_push_stack(g);
......@@ -478,15 +511,13 @@ static void end_token_name(Gen *g) {
478511 buf_init_from_mem(&token_name, ptr, len);
479512
480513 Token *token = find_or_create_token(g, &token_name);
481 RuleNode *node = allocate<RuleNode>(1);
514 RuleNode *node = create_rule_node(g);
482515 node->type = RuleNodeTypeToken;
483516 node->token.token = token;
484517
485518 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
486519 g->lex_cur_rule->tuple.children.append(node);
487520
488 g->biggest_tuple_len = max(g->biggest_tuple_len, g->lex_cur_rule->tuple.children.length);
489
490521
491522 lex_pop_stack(g);
492523}
......@@ -504,6 +535,36 @@ static void end_tuple_body(Gen *g) {
504535 buf_init_from_mem(&g->lex_cur_rule->tuple.body, ptr, len);
505536}
506537
538static void begin_sub_tuple(Gen *g) {
539 g->lex_sub_tuple_begin = g->lex_pos;
540 lex_push_stack(g);
541}
542
543static void end_sub_tuple(Gen *g) {
544 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
545 char *ptr = &buf_ptr(g->in_buf)[g->lex_sub_tuple_begin];
546 int len = g->lex_pos - g->lex_sub_tuple_begin;
547
548 RuleNode *node = create_rule_node(g);
549 node->type = RuleNodeTypeSubRule;
550 buf_init_from_mem(&node->sub_rule.name, ptr, len);
551
552 g->lex_cur_rule->tuple.children.append(node);
553
554 lex_pop_stack(g);
555}
556
557static RuleNode *find_rule_node(Gen *g, Buf *name) {
558 for (int i = 0; i < g->rules.length; i += 1) {
559 RuleNode *node = g->rules.at(i);
560 assert(node->type == RuleNodeTypeTuple);
561 if (buf_eql_buf(&node->tuple.name, name)) {
562 return node;
563 }
564 }
565 return nullptr;
566}
567
507568static void initialize_rules(Gen *g) {
508569 g->lex_state = LexStateStart;
509570 for (g->lex_pos = 0; g->lex_pos < buf_len(g->in_buf); g->lex_pos += 1) {
......@@ -524,18 +585,36 @@ static void initialize_rules(Gen *g) {
524585 break;
525586 case LexStateRuleName:
526587 switch (c) {
527 case WHITESPACE:
588 case '<':
528589 end_rule_name(g);
529 g->lex_state = LexStateWaitForColon;
590 g->lex_state = LexStateRuleFieldNameStart;
530591 break;
531 case ':':
532 end_rule_name(g);
533 g->lex_state = LexStateTupleRule;
592 case SYMBOL_CHAR:
593 // ok
534594 break;
595 default:
596 lex_error(g, "expected '<', not '%c'", c);
597 }
598 break;
599 case LexStateRuleFieldNameStart:
600 switch (c) {
535601 case SYMBOL_CHAR:
602 begin_rule_field_name(g);
603 g->lex_state = LexStateRuleFieldName;
536604 break;
537605 default:
538 lex_error(g, "invalid char: '%c'", c);
606 lex_error(g, "expected field name, not '%c'", c);
607 }
608 break;
609 case LexStateRuleFieldName:
610 switch (c) {
611 case SYMBOL_CHAR:
612 // ok
613 break;
614 case '>':
615 end_rule_field_name(g);
616 g->lex_state = LexStateWaitForColon;
617 break;
539618 }
540619 break;
541620 case LexStateWaitForColon:
......@@ -559,12 +638,16 @@ static void initialize_rules(Gen *g) {
559638 begin_fn_name(g);
560639 g->lex_state = LexStateFnName;
561640 break;
641 case UPPER_ALPHA:
642 begin_sub_tuple(g);
643 g->lex_state = LexStateSubTupleName;
644 break;
562645 case '{':
563646 begin_tuple_body(g);
564647 g->lex_state = LexStateBody;
565648 break;
566649 default:
567 lex_error(g, "invalid char: '%c'", c);
650 lex_error(g, "expected rule, not '%c'", c);
568651 }
569652 break;
570653 case LexStateFnName:
......@@ -589,7 +672,7 @@ static void initialize_rules(Gen *g) {
589672 g->lex_state = LexStateToken;
590673 break;
591674 default:
592 lex_error(g, "invalid char '%c'", c);
675 lex_error(g, "expected token name, not '%c'", c);
593676 }
594677 break;
595678 case LexStateToken:
......@@ -601,7 +684,7 @@ static void initialize_rules(Gen *g) {
601684 end_token_name(g);
602685 break;
603686 default:
604 lex_error(g, "invalid char '%c'", c);
687 lex_error(g, "expected token name or ')', not '%c'", c);
605688 }
606689 break;
607690 case LexStateBody:
......@@ -627,6 +710,20 @@ static void initialize_rules(Gen *g) {
627710 default:
628711 lex_error(g, "expected ';' or '|'");
629712 }
713 break;
714 case LexStateSubTupleName:
715 switch (c) {
716 case ALPHA:
717 // ignore
718 break;
719 case WHITESPACE:
720 end_sub_tuple(g);
721 assert(g->lex_state == LexStateTupleRule);
722 break;
723 default:
724 lex_error(g, "expected rule name, not '%c'", c);
725 }
726 break;
630727 }
631728 if (c == '\n') {
632729 g->lex_line += 1;
......@@ -647,9 +744,40 @@ static void initialize_rules(Gen *g) {
647744 case LexStateTokenStart:
648745 case LexStateToken:
649746 case LexStateBody:
747 case LexStateSubTupleName:
748 case LexStateRuleFieldNameStart:
749 case LexStateRuleFieldName:
650750 lex_error(g, "unexpected EOF");
651751 break;
652752 }
753
754 // Resolve child references into pointers
755 for (int tuple_i = 0; tuple_i < g->rules.length; tuple_i += 1) {
756 RuleNode *node = g->rules.at(tuple_i);
757 assert(node->type == RuleNodeTypeTuple);
758
759 for (int child_i = 0; child_i < node->tuple.children.length; child_i += 1) {
760 RuleNode *child = node->tuple.children.at(child_i);
761 if (child->type == RuleNodeTypeSubRule) {
762 int line = child->lex_line + 1;
763 int column = child->lex_column + 1;
764 RuleNode *referenced_node = find_rule_node(g, &child->sub_rule.name);
765 if (!referenced_node) {
766 fprintf(stderr, "Grammar Error: Line %d, column %d: Rule not defined: '%s'\n",
767 line, column, buf_ptr(&child->sub_rule.name));
768 }
769 child->sub_rule.child = referenced_node;
770 }
771 }
772 }
773
774
775 // calculate the biggest tuple len
776 for (int i = 0; i < g->rules.length; i += 1) {
777 RuleNode *node = g->rules.at(i);
778 assert(node->type == RuleNodeTypeTuple);
779 g->biggest_tuple_len = max(g->biggest_tuple_len, node->tuple.children.length);
780 }
653781}
654782
655783enum TemplateState {
......@@ -828,6 +956,8 @@ int main(int argc, char **argv) {
828956 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);
829957 break;
830958 case CodeGenTypeError:
959 fprintf(out_f, " token_index -= 1;\n");
960 fprintf(out_f, " token = &tokens->at(token_index);\n");
831961 fprintf(out_f, " ast_error(token, \"%s\");\n", buf_ptr(code->error.msg));
832962 break;
833963 case CodeGenTypeSave:
......@@ -843,7 +973,12 @@ int main(int argc, char **argv) {
843973 fprintf(out_f, "%s\n", buf_ptr(code_text));
844974 fprintf(out_f, " return root;\n");
845975 } else {
846 zig_panic("TODO capture non-root");
976 fprintf(out_f, " ParserGenNode *parent_node = stack.at(stack.length - 2);\n");
977 Buf *dest = buf_sprintf("parent_node->data[parent_node->next_index++].%s",
978 buf_ptr(code->capture.union_field_name));
979 Buf *code_text = fill_template(code->capture.body, buf_ptr(dest),
980 code->capture.field_names);
981 fprintf(out_f, "%s\n", buf_ptr(code_text));
847982 }
848983 break;
849984 case CodeGenTypePopNode:
src/tokenizer.hpp+3-3
......@@ -36,15 +36,15 @@ enum TokenId {
3636
3737// TODO: debug delete this
3838enum TokenId {
39 TokenIdStar = 0,
40 TokenIdLParen = 1,
39 TokenIdLParen = 0,
40 TokenIdRParen = 1,
4141 TokenIdEof = 2,
42 TokenIdStar = 3,
4243 TokenIdSymbol,
4344 TokenIdKeywordFn,
4445 TokenIdKeywordReturn,
4546 TokenIdKeywordMut,
4647 TokenIdKeywordConst,
47 TokenIdRParen,
4848 TokenIdComma,
4949 TokenIdLBrace,
5050 TokenIdRBrace,