authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-07 04:50:48-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-07 04:50:48-07:00
log4236b85c722b36cb31595634bae0ff28d6082761
tree37ad80afbb6e28e46eaf1c5808c35018a538aa54
parentae0e9685c690e9ddb31a28e5a9f827ea8da111ca

parser generator supports a simple OR

5 files changed, 214 insertions(+), 75 deletions(-)

src/parser.cpp+6
......@@ -82,3 +82,9 @@ void ast_print(AstNode *node, int indent) {
8282AstNode *ast_create_root(Token *token) {
8383 return nullptr;
8484}
85
86void ast_invalid_token_error(Buf *buf, Token *token) {
87 Buf token_value = {0};
88 buf_init_from_mem(&token_value, buf_ptr(buf) + token->start_pos, token->end_pos - token->start_pos);
89 ast_error(token, "invalid token: '%s'", buf_ptr(&token_value));
90}
src/parser.hpp+2
......@@ -81,6 +81,8 @@ struct AstNode {
8181
8282__attribute__ ((format (printf, 2, 3)))
8383void ast_error(Token *token, const char *format, ...);
84void ast_invalid_token_error(Buf *buf, Token *token);
85
8486
8587// This function is provided by generated code, generated by parsergen.cpp
8688AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens);
src/parsergen.cpp+202-74
......@@ -133,10 +133,8 @@ struct Token {
133133struct RuleNode;
134134
135135struct RuleTuple {
136 Buf name;
137136 ZigList<RuleNode *> children;
138137 Buf body;
139 Buf union_field_name;
140138};
141139
142140struct RuleMany {
......@@ -144,11 +142,13 @@ struct RuleMany {
144142};
145143
146144struct RuleOption {
147 ZigList<RuleNode *> child;
145 RuleNode *child;
148146};
149147
150148struct RuleOr {
151 ZigList<RuleTuple *> children;
149 Buf name;
150 Buf union_field_name;
151 ZigList<RuleNode *> children;
152152};
153153
154154struct RuleToken {
......@@ -227,6 +227,7 @@ struct ParserState {
227227 // One for each token ID.
228228 ParserState **transition;
229229 int index;
230 bool is_error;
230231};
231232
232233enum LexState {
......@@ -250,8 +251,9 @@ struct LexStack {
250251
251252struct Gen {
252253 ZigList<RuleNode *> rules;
253 ParserState *cur_state;
254
254255 ZigList<ParserState *> transition_table;
256 ParserState *start_state;
255257 ZigList<Token *> tokens;
256258 RuleNode *root;
257259 int biggest_tuple_len;
......@@ -260,7 +262,8 @@ struct Gen {
260262 LexState lex_state;
261263 int lex_line;
262264 int lex_column;
263 RuleNode *lex_cur_rule;
265 RuleNode *lex_cur_or_rule;
266 RuleNode *lex_cur_tuple_rule;;
264267 int lex_cur_rule_begin;
265268 int lex_fn_name_begin;
266269 int lex_pos;
......@@ -274,9 +277,8 @@ struct Gen {
274277
275278static ParserState *create_state(Gen *g) {
276279 ParserState *state = allocate<ParserState>(1);
277 state->index = g->transition_table.length;
280 state->index = -1;
278281 state->transition = allocate<ParserState*>(g->tokens.length);
279 g->transition_table.append(state);
280282 return state;
281283}
282284
......@@ -301,6 +303,7 @@ static void state_add_error(ParserState *state, Buf *msg) {
301303 code->type = CodeGenTypeError;
302304 code->error.msg = msg;
303305 state_add_code(state, code);
306 state->is_error = true;
304307}
305308
306309static void state_add_transition(ParserState *state) {
......@@ -337,45 +340,61 @@ static void state_add_eat_token(ParserState *state) {
337340 state_add_code(state, code);
338341}
339342
340static void gen(Gen *g, RuleNode *node, Buf *out_field_name) {
343
344static void gen(Gen *g, RuleNode *node, Buf *out_field_name, ParserState *cur_state,
345 ZigList<ParserState *> *end_states, bool is_root)
346{
347 struct PossibleState {
348 ParserState *test_state;
349 ZigList<ParserState *> end_states;
350 };
341351 assert(node);
342352 switch (node->type) {
343353 case RuleNodeTypeToken:
344354 {
345355 buf_init_from_str(out_field_name, "token");
346356
347 state_add_save_token(g->cur_state);
357 state_add_save_token(cur_state);
348358
349359 ParserState *ok_state = create_state(g);
350360 ParserState *err_state = create_state(g);
351361 state_add_error(err_state, buf_sprintf("expected token '%s'", buf_ptr(&node->token.token->name)));
352362
353 fill_state_with_transition(g, g->cur_state, err_state);
354 g->cur_state->transition[node->token.token->id] = ok_state;
355 state_add_transition(g->cur_state);
356 state_add_eat_token(g->cur_state);
363 fill_state_with_transition(g, cur_state, err_state);
364 cur_state->transition[node->token.token->id] = ok_state;
365 state_add_transition(cur_state);
366 state_add_eat_token(cur_state);
357367
358 g->cur_state = ok_state;
368 end_states->append(ok_state);
359369 }
360370 break;
361371 case RuleNodeTypeTuple:
362372 {
363 buf_init_from_buf(out_field_name, &node->tuple.union_field_name);
364
365 state_add_push_node(g->cur_state);
366373
367 bool is_root = (node == g->root);
368374 int field_name_count = node->tuple.children.length;
369375 CodeGen *code = codegen_create_capture(&node->tuple.body, is_root, field_name_count,
370 &node->tuple.union_field_name);
376 out_field_name);
371377
372 for (int i = 0; i < node->tuple.children.length; i += 1) {
373 RuleNode *child = node->tuple.children.at(i);
374 gen(g, child, &code->capture.field_names[i]);
378 ZigList<ParserState *> *my_end_states = allocate<ZigList<ParserState *>>(1);
379 my_end_states->append(cur_state);
380 for (int child_index = 0; child_index < node->tuple.children.length; child_index += 1) {
381 RuleNode *child = node->tuple.children.at(child_index);
382
383 ZigList<ParserState *> *more_end_states = allocate<ZigList<ParserState *>>(1);
384 for (int i = 0; i < my_end_states->length; i += 1) {
385 ParserState *use_state = my_end_states->at(i);
386 gen(g, child, &code->capture.field_names[i], use_state, more_end_states, false);
387 }
388
389 my_end_states = more_end_states;
375390 }
376 state_add_code(g->cur_state, code);
377391
378 state_add_pop_node(g->cur_state);
392 for (int i = 0; i < my_end_states->length; i += 1) {
393 ParserState *use_state = my_end_states->at(i);
394 state_add_code(use_state, code);
395
396 end_states->append(use_state);
397 }
379398 }
380399 break;
381400 case RuleNodeTypeMany:
......@@ -388,12 +407,73 @@ static void gen(Gen *g, RuleNode *node, Buf *out_field_name) {
388407 zig_panic("TODO");
389408 break;
390409 case RuleNodeTypeOr:
391 zig_panic("TODO");
410 {
411 buf_init_from_buf(out_field_name, &node->_or.union_field_name);
412
413 state_add_push_node(cur_state);
414
415 // TODO this probably need to get moved when or can handle conflicts
416 state_add_save_token(cur_state);
417 state_add_transition(cur_state);
418 state_add_eat_token(cur_state);
419
420
421 int possible_state_count = node->_or.children.length;
422 PossibleState *possible_states = allocate<PossibleState>(possible_state_count);
423 for (int i = 0; i < possible_state_count; i += 1) {
424 RuleNode *child = node->_or.children.at(i);
425 assert(child->type == RuleNodeTypeTuple);
426
427 PossibleState *possible_state = &possible_states[i];
428 possible_state->test_state = create_state(g);
429 gen(g, child, &node->_or.union_field_name, possible_state->test_state,
430 &possible_state->end_states, is_root);
431 }
432
433 // try to merge all the possible states into new state.
434 ParserState *err_state = create_state(g);
435 state_add_error(err_state, buf_create_from_str("unexpected token"));
436 for (int token_i = 0; token_i < g->tokens.length; token_i += 1) {
437 bool any_called_it = false;
438 bool conflict = false;
439 for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
440 PossibleState *possible_state = &possible_states[state_i];
441 if (!possible_state->test_state->transition[token_i]->is_error) {
442 if (any_called_it) {
443 conflict = true;
444 } else {
445 any_called_it = true;
446 }
447 }
448 }
449 if (conflict) {
450 zig_panic("TODO state transition conflict");
451 } else {
452 cur_state->transition[token_i] = err_state;
453 for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
454 PossibleState *possible_state = &possible_states[state_i];
455 if (!possible_state->test_state->transition[token_i]->is_error) {
456 cur_state->transition[token_i] = possible_state->test_state->transition[token_i];
457 }
458 }
459 }
460 }
461
462 for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
463 PossibleState *possible_state = &possible_states[state_i];
464 for (int end_i = 0; end_i < possible_state->end_states.length; end_i += 1) {
465 ParserState *state = possible_state->end_states.at(end_i);
466 state_add_pop_node(state);
467
468 end_states->append(state);
469 }
470 }
471 }
392472 break;
393473 case RuleNodeTypeSubRule:
394474 {
395475 RuleNode *child = node->sub_rule.child;
396 gen(g, child, out_field_name);
476 gen(g, child, out_field_name, cur_state, end_states, false);
397477 }
398478 break;
399479 }
......@@ -451,38 +531,51 @@ static RuleNode *create_rule_node(Gen *g) {
451531}
452532
453533static void begin_rule(Gen *g) {
454 assert(!g->lex_cur_rule);
455 g->lex_cur_rule = create_rule_node(g);
456 g->lex_cur_rule->type = RuleNodeTypeTuple;
457 g->lex_cur_rule_begin = g->lex_pos;
534 assert(!g->lex_cur_or_rule);
535 assert(!g->lex_cur_tuple_rule);
458536
459 g->lex_state = LexStateEndOrOr;
460 lex_push_stack(g);
537 g->lex_cur_or_rule = create_rule_node(g);
538 g->lex_cur_or_rule->type = RuleNodeTypeOr;
539
540 g->lex_cur_tuple_rule = create_rule_node(g);
541 g->lex_cur_tuple_rule->type = RuleNodeTypeTuple;
542 g->lex_cur_rule_begin = g->lex_pos;
461543}
462544
463545static void end_rule(Gen *g) {
464 assert(g->lex_cur_rule);
465 g->rules.append(g->lex_cur_rule);
466 g->lex_cur_rule = nullptr;
546 assert(g->lex_cur_or_rule);
547 assert(!g->lex_cur_tuple_rule);
548
549 g->rules.append(g->lex_cur_or_rule);
550 g->lex_cur_or_rule = nullptr;
551}
552
553static void perform_or(Gen *g) {
554 assert(g->lex_cur_or_rule);
555 assert(!g->lex_cur_tuple_rule);
556
557 g->lex_cur_tuple_rule = create_rule_node(g);
558 g->lex_cur_tuple_rule->type = RuleNodeTypeTuple;
559 g->lex_cur_rule_begin = g->lex_pos;
467560}
468561
469562static void end_rule_name(Gen *g) {
470 assert(g->lex_cur_rule);
563 assert(g->lex_cur_or_rule);
471564 char *ptr = &buf_ptr(g->in_buf)[g->lex_cur_rule_begin];
472565 int len = g->lex_pos - g->lex_cur_rule_begin;
473 buf_init_from_mem(&g->lex_cur_rule->tuple.name, ptr, len);
566 buf_init_from_mem(&g->lex_cur_or_rule->_or.name, ptr, len);
474567}
475568
476569static void begin_rule_field_name(Gen *g) {
477 assert(g->lex_cur_rule);
570 assert(g->lex_cur_or_rule);
478571 g->lex_field_name_begin = g->lex_pos;
479572}
480573
481574static void end_rule_field_name(Gen *g) {
482 assert(g->lex_cur_rule);
575 assert(g->lex_cur_or_rule);
483576 char *ptr = &buf_ptr(g->in_buf)[g->lex_field_name_begin];
484577 int len = g->lex_pos - g->lex_field_name_begin;
485 buf_init_from_mem(&g->lex_cur_rule->tuple.union_field_name, ptr, len);
578 buf_init_from_mem(&g->lex_cur_or_rule->_or.union_field_name, ptr, len);
486579}
487580
488581static void begin_fn_name(Gen *g) {
......@@ -505,6 +598,9 @@ static void begin_token_name(Gen *g) {
505598}
506599
507600static void end_token_name(Gen *g) {
601 assert(g->lex_cur_tuple_rule);
602 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
603
508604 char *ptr = &buf_ptr(g->in_buf)[g->lex_token_name_begin];
509605 int len = g->lex_pos - g->lex_token_name_begin;
510606 Buf token_name = {0};
......@@ -515,24 +611,27 @@ static void end_token_name(Gen *g) {
515611 node->type = RuleNodeTypeToken;
516612 node->token.token = token;
517613
518 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
519 g->lex_cur_rule->tuple.children.append(node);
614 g->lex_cur_tuple_rule->tuple.children.append(node);
520615
521616
522617 lex_pop_stack(g);
523618}
524619
525620static void begin_tuple_body(Gen *g) {
526 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
621 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
527622 g->lex_body_begin = g->lex_pos;
528623}
529624
530625static void end_tuple_body(Gen *g) {
531 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
626 assert(g->lex_cur_or_rule);
627 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
532628 int end_pos = g->lex_pos + 1;
533629 char *ptr = &buf_ptr(g->in_buf)[g->lex_body_begin];
534630 int len = end_pos - g->lex_body_begin;
535 buf_init_from_mem(&g->lex_cur_rule->tuple.body, ptr, len);
631 buf_init_from_mem(&g->lex_cur_tuple_rule->tuple.body, ptr, len);
632
633 g->lex_cur_or_rule->_or.children.append(g->lex_cur_tuple_rule);
634 g->lex_cur_tuple_rule = nullptr;
536635}
537636
538637static void begin_sub_tuple(Gen *g) {
......@@ -541,7 +640,7 @@ static void begin_sub_tuple(Gen *g) {
541640}
542641
543642static void end_sub_tuple(Gen *g) {
544 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
643 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
545644 char *ptr = &buf_ptr(g->in_buf)[g->lex_sub_tuple_begin];
546645 int len = g->lex_pos - g->lex_sub_tuple_begin;
547646
......@@ -549,7 +648,7 @@ static void end_sub_tuple(Gen *g) {
549648 node->type = RuleNodeTypeSubRule;
550649 buf_init_from_mem(&node->sub_rule.name, ptr, len);
551650
552 g->lex_cur_rule->tuple.children.append(node);
651 g->lex_cur_tuple_rule->tuple.children.append(node);
553652
554653 lex_pop_stack(g);
555654}
......@@ -557,8 +656,8 @@ static void end_sub_tuple(Gen *g) {
557656static RuleNode *find_rule_node(Gen *g, Buf *name) {
558657 for (int i = 0; i < g->rules.length; i += 1) {
559658 RuleNode *node = g->rules.at(i);
560 assert(node->type == RuleNodeTypeTuple);
561 if (buf_eql_buf(&node->tuple.name, name)) {
659 assert(node->type == RuleNodeTypeOr);
660 if (buf_eql_buf(&node->_or.name, name)) {
562661 return node;
563662 }
564663 }
......@@ -691,7 +790,7 @@ static void initialize_rules(Gen *g) {
691790 switch (c) {
692791 case '}':
693792 end_tuple_body(g);
694 lex_pop_stack(g);
793 g->lex_state = LexStateEndOrOr;
695794 break;
696795 default:
697796 // ignore
......@@ -707,6 +806,10 @@ static void initialize_rules(Gen *g) {
707806 end_rule(g);
708807 g->lex_state = LexStateStart;
709808 break;
809 case '|':
810 perform_or(g);
811 g->lex_state = LexStateTupleRule;
812 break;
710813 default:
711814 lex_error(g, "expected ';' or '|'");
712815 }
......@@ -751,32 +854,39 @@ static void initialize_rules(Gen *g) {
751854 break;
752855 }
753856
754 // Resolve child references into pointers
755 for (int tuple_i = 0; tuple_i < g->rules.length; tuple_i += 1) {
756 RuleNode *node = g->rules.at(tuple_i);
757 assert(node->type == RuleNodeTypeTuple);
758
759 for (int child_i = 0; child_i < node->tuple.children.length; child_i += 1) {
760 RuleNode *child = node->tuple.children.at(child_i);
761 if (child->type == RuleNodeTypeSubRule) {
762 int line = child->lex_line + 1;
763 int column = child->lex_column + 1;
764 RuleNode *referenced_node = find_rule_node(g, &child->sub_rule.name);
765 if (!referenced_node) {
766 fprintf(stderr, "Grammar Error: Line %d, column %d: Rule not defined: '%s'\n",
767 line, column, buf_ptr(&child->sub_rule.name));
857 // Iterate over the rules and
858 // * resolve child references into pointers
859 // * calculate the biggest tuple len
860 bool any_errors = false;
861 for (int or_i = 0; or_i < g->rules.length; or_i += 1) {
862 RuleNode *or_node = g->rules.at(or_i);
863 assert(or_node->type == RuleNodeTypeOr);
864
865 for (int tuple_i = 0; tuple_i < or_node->_or.children.length; tuple_i += 1) {
866 RuleNode *tuple_node = or_node->_or.children.at(tuple_i);
867 assert(tuple_node->type == RuleNodeTypeTuple);
868 g->biggest_tuple_len = max(g->biggest_tuple_len, tuple_node->tuple.children.length);
869
870 for (int child_i = 0; child_i < tuple_node->tuple.children.length; child_i += 1) {
871 RuleNode *child = tuple_node->tuple.children.at(child_i);
872
873 if (child->type == RuleNodeTypeSubRule) {
874 int line = child->lex_line + 1;
875 int column = child->lex_column + 1;
876 RuleNode *referenced_node = find_rule_node(g, &child->sub_rule.name);
877 if (!referenced_node) {
878 fprintf(stderr, "Grammar Error: Line %d, column %d: Rule not defined: '%s'\n",
879 line, column, buf_ptr(&child->sub_rule.name));
880 any_errors = true;
881 }
882 child->sub_rule.child = referenced_node;
768883 }
769 child->sub_rule.child = referenced_node;
770884 }
771885 }
772886 }
773887
774
775 // calculate the biggest tuple len
776 for (int i = 0; i < g->rules.length; i += 1) {
777 RuleNode *node = g->rules.at(i);
778 assert(node->type == RuleNodeTypeTuple);
779 g->biggest_tuple_len = max(g->biggest_tuple_len, node->tuple.children.length);
888 if (any_errors) {
889 exit(EXIT_FAILURE);
780890 }
781891}
782892
......@@ -851,6 +961,19 @@ static Buf *fill_template(Buf *body, const char *result_name, Buf *field_names)
851961 return result;
852962}
853963
964static void build_transition_table(Gen *g, ParserState *state) {
965 if (!state)
966 return;
967 if (state->index >= 0)
968 return;
969 state->index = g->transition_table.length;
970 g->transition_table.append(state);
971 for (int i = 0; i < g->tokens.length; i += 1) {
972 ParserState *other_state = state->transition[i];
973 build_transition_table(g, other_state);
974 }
975}
976
854977int main(int argc, char **argv) {
855978 const char *in_filename = argv[1];
856979 const char *out_filename = argv[2];
......@@ -882,9 +1005,11 @@ int main(int argc, char **argv) {
8821005
8831006 g.root = g.rules.at(0);
8841007
885 g.cur_state = create_state(&g);
1008 g.start_state = create_state(&g);
8861009 Buf root_field_name = {0};
887 gen(&g, g.root, &root_field_name);
1010 ZigList<ParserState *> end_states = {0};
1011 gen(&g, g.root, &root_field_name, g.start_state, &end_states, true);
1012 build_transition_table(&g, g.start_state);
8881013
8891014 fprintf(out_f, "/* This file is generated by parsergen.cpp */\n");
8901015 fprintf(out_f, "\n");
......@@ -950,6 +1075,9 @@ int main(int argc, char **argv) {
9501075 CodeGen *code = state->code_gen_list.at(code_i);
9511076 switch (code->type) {
9521077 case CodeGenTypeTransition:
1078 fprintf(out_f, " if (token->id < 0 || token->id >= %d) {\n", g.tokens.length);
1079 fprintf(out_f, " ast_invalid_token_error(buf, token);\n");
1080 fprintf(out_f, " }\n");
9531081 fprintf(out_f, " assert(transition[%d][token->id] >= 0);\n", state->index);
9541082 fprintf(out_f, " assert(transition[%d][token->id] < %d);\n",
9551083 state->index, g.transition_table.length);
src/tokenizer.hpp+1-1
......@@ -40,6 +40,7 @@ enum TokenId {
4040 TokenIdRParen = 1,
4141 TokenIdEof = 2,
4242 TokenIdStar = 3,
43 TokenIdPlus = 4,
4344 TokenIdSymbol,
4445 TokenIdKeywordFn,
4546 TokenIdKeywordReturn,
......@@ -51,7 +52,6 @@ enum TokenId {
5152 TokenIdStringLiteral,
5253 TokenIdSemicolon,
5354 TokenIdNumberLiteral,
54 TokenIdPlus,
5555 TokenIdColon,
5656 TokenIdArrow,
5757 TokenIdDash,
src/util.hpp+3
......@@ -12,6 +12,9 @@
1212#include <string.h>
1313#include <assert.h>
1414
15
16#define BREAKPOINT __asm("int $0x03")
17
1518void zig_panic(const char *format, ...)
1619 __attribute__((cold))
1720 __attribute__ ((noreturn))