authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-07 04:50:48-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-07 04:50:48-07:00
log4236b85c722b36cb31595634bae0ff28d6082761
tree37ad80afbb6e28e46eaf1c5808c35018a538aa54
parentae0e9685c690e9ddb31a28e5a9f827ea8da111ca

parser generator supports a simple OR

5 files changed, 214 insertions(+), 75 deletions(-)

src/parser.cpp+6
...@@ -82,3 +82,9 @@ void ast_print(AstNode *node, int indent) {...@@ -82,3 +82,9 @@ void ast_print(AstNode *node, int indent) {
82AstNode *ast_create_root(Token *token) {82AstNode *ast_create_root(Token *token) {
83 return nullptr;83 return nullptr;
84}84}
85
86void ast_invalid_token_error(Buf *buf, Token *token) {
87 Buf token_value = {0};
88 buf_init_from_mem(&token_value, buf_ptr(buf) + token->start_pos, token->end_pos - token->start_pos);
89 ast_error(token, "invalid token: '%s'", buf_ptr(&token_value));
90}
src/parser.hpp+2
...@@ -81,6 +81,8 @@ struct AstNode {...@@ -81,6 +81,8 @@ struct AstNode {
8181
82__attribute__ ((format (printf, 2, 3)))82__attribute__ ((format (printf, 2, 3)))
83void ast_error(Token *token, const char *format, ...);83void ast_error(Token *token, const char *format, ...);
84void ast_invalid_token_error(Buf *buf, Token *token);
85
8486
85// This function is provided by generated code, generated by parsergen.cpp87// This function is provided by generated code, generated by parsergen.cpp
86AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens);88AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens);
src/parsergen.cpp+202-74
...@@ -133,10 +133,8 @@ struct Token {...@@ -133,10 +133,8 @@ struct Token {
133struct RuleNode;133struct RuleNode;
134134
135struct RuleTuple {135struct RuleTuple {
136 Buf name;
137 ZigList<RuleNode *> children;136 ZigList<RuleNode *> children;
138 Buf body;137 Buf body;
139 Buf union_field_name;
140};138};
141139
142struct RuleMany {140struct RuleMany {
...@@ -144,11 +142,13 @@ struct RuleMany {...@@ -144,11 +142,13 @@ struct RuleMany {
144};142};
145143
146struct RuleOption {144struct RuleOption {
147 ZigList<RuleNode *> child;145 RuleNode *child;
148};146};
149147
150struct RuleOr {148struct RuleOr {
151 ZigList<RuleTuple *> children;149 Buf name;
150 Buf union_field_name;
151 ZigList<RuleNode *> children;
152};152};
153153
154struct RuleToken {154struct RuleToken {
...@@ -227,6 +227,7 @@ struct ParserState {...@@ -227,6 +227,7 @@ struct ParserState {
227 // One for each token ID.227 // One for each token ID.
228 ParserState **transition;228 ParserState **transition;
229 int index;229 int index;
230 bool is_error;
230};231};
231232
232enum LexState {233enum LexState {
...@@ -250,8 +251,9 @@ struct LexStack {...@@ -250,8 +251,9 @@ struct LexStack {
250251
251struct Gen {252struct Gen {
252 ZigList<RuleNode *> rules;253 ZigList<RuleNode *> rules;
253 ParserState *cur_state;254
254 ZigList<ParserState *> transition_table;255 ZigList<ParserState *> transition_table;
256 ParserState *start_state;
255 ZigList<Token *> tokens;257 ZigList<Token *> tokens;
256 RuleNode *root;258 RuleNode *root;
257 int biggest_tuple_len;259 int biggest_tuple_len;
...@@ -260,7 +262,8 @@ struct Gen {...@@ -260,7 +262,8 @@ struct Gen {
260 LexState lex_state;262 LexState lex_state;
261 int lex_line;263 int lex_line;
262 int lex_column;264 int lex_column;
263 RuleNode *lex_cur_rule;265 RuleNode *lex_cur_or_rule;
266 RuleNode *lex_cur_tuple_rule;;
264 int lex_cur_rule_begin;267 int lex_cur_rule_begin;
265 int lex_fn_name_begin;268 int lex_fn_name_begin;
266 int lex_pos;269 int lex_pos;
...@@ -274,9 +277,8 @@ struct Gen {...@@ -274,9 +277,8 @@ struct Gen {
274277
275static ParserState *create_state(Gen *g) {278static ParserState *create_state(Gen *g) {
276 ParserState *state = allocate<ParserState>(1);279 ParserState *state = allocate<ParserState>(1);
277 state->index = g->transition_table.length;280 state->index = -1;
278 state->transition = allocate<ParserState*>(g->tokens.length);281 state->transition = allocate<ParserState*>(g->tokens.length);
279 g->transition_table.append(state);
280 return state;282 return state;
281}283}
282284
...@@ -301,6 +303,7 @@ static void state_add_error(ParserState *state, Buf *msg) {...@@ -301,6 +303,7 @@ static void state_add_error(ParserState *state, Buf *msg) {
301 code->type = CodeGenTypeError;303 code->type = CodeGenTypeError;
302 code->error.msg = msg;304 code->error.msg = msg;
303 state_add_code(state, code);305 state_add_code(state, code);
306 state->is_error = true;
304}307}
305308
306static void state_add_transition(ParserState *state) {309static void state_add_transition(ParserState *state) {
...@@ -337,45 +340,61 @@ static void state_add_eat_token(ParserState *state) {...@@ -337,45 +340,61 @@ static void state_add_eat_token(ParserState *state) {
337 state_add_code(state, code);340 state_add_code(state, code);
338}341}
339342
340static void gen(Gen *g, RuleNode *node, Buf *out_field_name) {343
344static void gen(Gen *g, RuleNode *node, Buf *out_field_name, ParserState *cur_state,
345 ZigList<ParserState *> *end_states, bool is_root)
346{
347 struct PossibleState {
348 ParserState *test_state;
349 ZigList<ParserState *> end_states;
350 };
341 assert(node);351 assert(node);
342 switch (node->type) {352 switch (node->type) {
343 case RuleNodeTypeToken:353 case RuleNodeTypeToken:
344 {354 {
345 buf_init_from_str(out_field_name, "token");355 buf_init_from_str(out_field_name, "token");
346356
347 state_add_save_token(g->cur_state);357 state_add_save_token(cur_state);
348358
349 ParserState *ok_state = create_state(g);359 ParserState *ok_state = create_state(g);
350 ParserState *err_state = create_state(g);360 ParserState *err_state = create_state(g);
351 state_add_error(err_state, buf_sprintf("expected token '%s'", buf_ptr(&node->token.token->name)));361 state_add_error(err_state, buf_sprintf("expected token '%s'", buf_ptr(&node->token.token->name)));
352362
353 fill_state_with_transition(g, g->cur_state, err_state);363 fill_state_with_transition(g, cur_state, err_state);
354 g->cur_state->transition[node->token.token->id] = ok_state;364 cur_state->transition[node->token.token->id] = ok_state;
355 state_add_transition(g->cur_state);365 state_add_transition(cur_state);
356 state_add_eat_token(g->cur_state);366 state_add_eat_token(cur_state);
357367
358 g->cur_state = ok_state;368 end_states->append(ok_state);
359 }369 }
360 break;370 break;
361 case RuleNodeTypeTuple:371 case RuleNodeTypeTuple:
362 {372 {
363 buf_init_from_buf(out_field_name, &node->tuple.union_field_name);
364
365 state_add_push_node(g->cur_state);
366373
367 bool is_root = (node == g->root);
368 int field_name_count = node->tuple.children.length;374 int field_name_count = node->tuple.children.length;
369 CodeGen *code = codegen_create_capture(&node->tuple.body, is_root, field_name_count,375 CodeGen *code = codegen_create_capture(&node->tuple.body, is_root, field_name_count,
370 &node->tuple.union_field_name);376 out_field_name);
371377
372 for (int i = 0; i < node->tuple.children.length; i += 1) {378 ZigList<ParserState *> *my_end_states = allocate<ZigList<ParserState *>>(1);
373 RuleNode *child = node->tuple.children.at(i);379 my_end_states->append(cur_state);
374 gen(g, child, &code->capture.field_names[i]);380 for (int child_index = 0; child_index < node->tuple.children.length; child_index += 1) {
381 RuleNode *child = node->tuple.children.at(child_index);
382
383 ZigList<ParserState *> *more_end_states = allocate<ZigList<ParserState *>>(1);
384 for (int i = 0; i < my_end_states->length; i += 1) {
385 ParserState *use_state = my_end_states->at(i);
386 gen(g, child, &code->capture.field_names[i], use_state, more_end_states, false);
387 }
388
389 my_end_states = more_end_states;
375 }390 }
376 state_add_code(g->cur_state, code);
377391
378 state_add_pop_node(g->cur_state);392 for (int i = 0; i < my_end_states->length; i += 1) {
393 ParserState *use_state = my_end_states->at(i);
394 state_add_code(use_state, code);
395
396 end_states->append(use_state);
397 }
379 }398 }
380 break;399 break;
381 case RuleNodeTypeMany:400 case RuleNodeTypeMany:
...@@ -388,12 +407,73 @@ static void gen(Gen *g, RuleNode *node, Buf *out_field_name) {...@@ -388,12 +407,73 @@ static void gen(Gen *g, RuleNode *node, Buf *out_field_name) {
388 zig_panic("TODO");407 zig_panic("TODO");
389 break;408 break;
390 case RuleNodeTypeOr:409 case RuleNodeTypeOr:
391 zig_panic("TODO");410 {
411 buf_init_from_buf(out_field_name, &node->_or.union_field_name);
412
413 state_add_push_node(cur_state);
414
415 // TODO this probably need to get moved when or can handle conflicts
416 state_add_save_token(cur_state);
417 state_add_transition(cur_state);
418 state_add_eat_token(cur_state);
419
420
421 int possible_state_count = node->_or.children.length;
422 PossibleState *possible_states = allocate<PossibleState>(possible_state_count);
423 for (int i = 0; i < possible_state_count; i += 1) {
424 RuleNode *child = node->_or.children.at(i);
425 assert(child->type == RuleNodeTypeTuple);
426
427 PossibleState *possible_state = &possible_states[i];
428 possible_state->test_state = create_state(g);
429 gen(g, child, &node->_or.union_field_name, possible_state->test_state,
430 &possible_state->end_states, is_root);
431 }
432
433 // try to merge all the possible states into new state.
434 ParserState *err_state = create_state(g);
435 state_add_error(err_state, buf_create_from_str("unexpected token"));
436 for (int token_i = 0; token_i < g->tokens.length; token_i += 1) {
437 bool any_called_it = false;
438 bool conflict = false;
439 for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
440 PossibleState *possible_state = &possible_states[state_i];
441 if (!possible_state->test_state->transition[token_i]->is_error) {
442 if (any_called_it) {
443 conflict = true;
444 } else {
445 any_called_it = true;
446 }
447 }
448 }
449 if (conflict) {
450 zig_panic("TODO state transition conflict");
451 } else {
452 cur_state->transition[token_i] = err_state;
453 for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
454 PossibleState *possible_state = &possible_states[state_i];
455 if (!possible_state->test_state->transition[token_i]->is_error) {
456 cur_state->transition[token_i] = possible_state->test_state->transition[token_i];
457 }
458 }
459 }
460 }
461
462 for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
463 PossibleState *possible_state = &possible_states[state_i];
464 for (int end_i = 0; end_i < possible_state->end_states.length; end_i += 1) {
465 ParserState *state = possible_state->end_states.at(end_i);
466 state_add_pop_node(state);
467
468 end_states->append(state);
469 }
470 }
471 }
392 break;472 break;
393 case RuleNodeTypeSubRule:473 case RuleNodeTypeSubRule:
394 {474 {
395 RuleNode *child = node->sub_rule.child;475 RuleNode *child = node->sub_rule.child;
396 gen(g, child, out_field_name);476 gen(g, child, out_field_name, cur_state, end_states, false);
397 }477 }
398 break;478 break;
399 }479 }
...@@ -451,38 +531,51 @@ static RuleNode *create_rule_node(Gen *g) {...@@ -451,38 +531,51 @@ static RuleNode *create_rule_node(Gen *g) {
451}531}
452532
453static void begin_rule(Gen *g) {533static void begin_rule(Gen *g) {
454 assert(!g->lex_cur_rule);534 assert(!g->lex_cur_or_rule);
455 g->lex_cur_rule = create_rule_node(g);535 assert(!g->lex_cur_tuple_rule);
456 g->lex_cur_rule->type = RuleNodeTypeTuple;
457 g->lex_cur_rule_begin = g->lex_pos;
458536
459 g->lex_state = LexStateEndOrOr;537 g->lex_cur_or_rule = create_rule_node(g);
460 lex_push_stack(g);538 g->lex_cur_or_rule->type = RuleNodeTypeOr;
539
540 g->lex_cur_tuple_rule = create_rule_node(g);
541 g->lex_cur_tuple_rule->type = RuleNodeTypeTuple;
542 g->lex_cur_rule_begin = g->lex_pos;
461}543}
462544
463static void end_rule(Gen *g) {545static void end_rule(Gen *g) {
464 assert(g->lex_cur_rule);546 assert(g->lex_cur_or_rule);
465 g->rules.append(g->lex_cur_rule);547 assert(!g->lex_cur_tuple_rule);
466 g->lex_cur_rule = nullptr;548
549 g->rules.append(g->lex_cur_or_rule);
550 g->lex_cur_or_rule = nullptr;
551}
552
553static void perform_or(Gen *g) {
554 assert(g->lex_cur_or_rule);
555 assert(!g->lex_cur_tuple_rule);
556
557 g->lex_cur_tuple_rule = create_rule_node(g);
558 g->lex_cur_tuple_rule->type = RuleNodeTypeTuple;
559 g->lex_cur_rule_begin = g->lex_pos;
467}560}
468561
469static void end_rule_name(Gen *g) {562static void end_rule_name(Gen *g) {
470 assert(g->lex_cur_rule);563 assert(g->lex_cur_or_rule);
471 char *ptr = &buf_ptr(g->in_buf)[g->lex_cur_rule_begin];564 char *ptr = &buf_ptr(g->in_buf)[g->lex_cur_rule_begin];
472 int len = g->lex_pos - g->lex_cur_rule_begin;565 int len = g->lex_pos - g->lex_cur_rule_begin;
473 buf_init_from_mem(&g->lex_cur_rule->tuple.name, ptr, len);566 buf_init_from_mem(&g->lex_cur_or_rule->_or.name, ptr, len);
474}567}
475568
476static void begin_rule_field_name(Gen *g) {569static void begin_rule_field_name(Gen *g) {
477 assert(g->lex_cur_rule);570 assert(g->lex_cur_or_rule);
478 g->lex_field_name_begin = g->lex_pos;571 g->lex_field_name_begin = g->lex_pos;
479}572}
480573
481static void end_rule_field_name(Gen *g) {574static void end_rule_field_name(Gen *g) {
482 assert(g->lex_cur_rule);575 assert(g->lex_cur_or_rule);
483 char *ptr = &buf_ptr(g->in_buf)[g->lex_field_name_begin];576 char *ptr = &buf_ptr(g->in_buf)[g->lex_field_name_begin];
484 int len = g->lex_pos - g->lex_field_name_begin;577 int len = g->lex_pos - g->lex_field_name_begin;
485 buf_init_from_mem(&g->lex_cur_rule->tuple.union_field_name, ptr, len);578 buf_init_from_mem(&g->lex_cur_or_rule->_or.union_field_name, ptr, len);
486}579}
487580
488static void begin_fn_name(Gen *g) {581static void begin_fn_name(Gen *g) {
...@@ -505,6 +598,9 @@ static void begin_token_name(Gen *g) {...@@ -505,6 +598,9 @@ static void begin_token_name(Gen *g) {
505}598}
506599
507static void end_token_name(Gen *g) {600static void end_token_name(Gen *g) {
601 assert(g->lex_cur_tuple_rule);
602 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
603
508 char *ptr = &buf_ptr(g->in_buf)[g->lex_token_name_begin];604 char *ptr = &buf_ptr(g->in_buf)[g->lex_token_name_begin];
509 int len = g->lex_pos - g->lex_token_name_begin;605 int len = g->lex_pos - g->lex_token_name_begin;
510 Buf token_name = {0};606 Buf token_name = {0};
...@@ -515,24 +611,27 @@ static void end_token_name(Gen *g) {...@@ -515,24 +611,27 @@ static void end_token_name(Gen *g) {
515 node->type = RuleNodeTypeToken;611 node->type = RuleNodeTypeToken;
516 node->token.token = token;612 node->token.token = token;
517613
518 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);614 g->lex_cur_tuple_rule->tuple.children.append(node);
519 g->lex_cur_rule->tuple.children.append(node);
520615
521616
522 lex_pop_stack(g);617 lex_pop_stack(g);
523}618}
524619
525static void begin_tuple_body(Gen *g) {620static void begin_tuple_body(Gen *g) {
526 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);621 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
527 g->lex_body_begin = g->lex_pos;622 g->lex_body_begin = g->lex_pos;
528}623}
529624
530static void end_tuple_body(Gen *g) {625static void end_tuple_body(Gen *g) {
531 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);626 assert(g->lex_cur_or_rule);
627 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
532 int end_pos = g->lex_pos + 1;628 int end_pos = g->lex_pos + 1;
533 char *ptr = &buf_ptr(g->in_buf)[g->lex_body_begin];629 char *ptr = &buf_ptr(g->in_buf)[g->lex_body_begin];
534 int len = end_pos - g->lex_body_begin;630 int len = end_pos - g->lex_body_begin;
535 buf_init_from_mem(&g->lex_cur_rule->tuple.body, ptr, len);631 buf_init_from_mem(&g->lex_cur_tuple_rule->tuple.body, ptr, len);
632
633 g->lex_cur_or_rule->_or.children.append(g->lex_cur_tuple_rule);
634 g->lex_cur_tuple_rule = nullptr;
536}635}
537636
538static void begin_sub_tuple(Gen *g) {637static void begin_sub_tuple(Gen *g) {
...@@ -541,7 +640,7 @@ static void begin_sub_tuple(Gen *g) {...@@ -541,7 +640,7 @@ static void begin_sub_tuple(Gen *g) {
541}640}
542641
543static void end_sub_tuple(Gen *g) {642static void end_sub_tuple(Gen *g) {
544 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);643 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
545 char *ptr = &buf_ptr(g->in_buf)[g->lex_sub_tuple_begin];644 char *ptr = &buf_ptr(g->in_buf)[g->lex_sub_tuple_begin];
546 int len = g->lex_pos - g->lex_sub_tuple_begin;645 int len = g->lex_pos - g->lex_sub_tuple_begin;
547646
...@@ -549,7 +648,7 @@ static void end_sub_tuple(Gen *g) {...@@ -549,7 +648,7 @@ static void end_sub_tuple(Gen *g) {
549 node->type = RuleNodeTypeSubRule;648 node->type = RuleNodeTypeSubRule;
550 buf_init_from_mem(&node->sub_rule.name, ptr, len);649 buf_init_from_mem(&node->sub_rule.name, ptr, len);
551650
552 g->lex_cur_rule->tuple.children.append(node);651 g->lex_cur_tuple_rule->tuple.children.append(node);
553652
554 lex_pop_stack(g);653 lex_pop_stack(g);
555}654}
...@@ -557,8 +656,8 @@ static void end_sub_tuple(Gen *g) {...@@ -557,8 +656,8 @@ static void end_sub_tuple(Gen *g) {
557static RuleNode *find_rule_node(Gen *g, Buf *name) {656static RuleNode *find_rule_node(Gen *g, Buf *name) {
558 for (int i = 0; i < g->rules.length; i += 1) {657 for (int i = 0; i < g->rules.length; i += 1) {
559 RuleNode *node = g->rules.at(i);658 RuleNode *node = g->rules.at(i);
560 assert(node->type == RuleNodeTypeTuple);659 assert(node->type == RuleNodeTypeOr);
561 if (buf_eql_buf(&node->tuple.name, name)) {660 if (buf_eql_buf(&node->_or.name, name)) {
562 return node;661 return node;
563 }662 }
564 }663 }
...@@ -691,7 +790,7 @@ static void initialize_rules(Gen *g) {...@@ -691,7 +790,7 @@ static void initialize_rules(Gen *g) {
691 switch (c) {790 switch (c) {
692 case '}':791 case '}':
693 end_tuple_body(g);792 end_tuple_body(g);
694 lex_pop_stack(g);793 g->lex_state = LexStateEndOrOr;
695 break;794 break;
696 default:795 default:
697 // ignore796 // ignore
...@@ -707,6 +806,10 @@ static void initialize_rules(Gen *g) {...@@ -707,6 +806,10 @@ static void initialize_rules(Gen *g) {
707 end_rule(g);806 end_rule(g);
708 g->lex_state = LexStateStart;807 g->lex_state = LexStateStart;
709 break;808 break;
809 case '|':
810 perform_or(g);
811 g->lex_state = LexStateTupleRule;
812 break;
710 default:813 default:
711 lex_error(g, "expected ';' or '|'");814 lex_error(g, "expected ';' or '|'");
712 }815 }
...@@ -751,32 +854,39 @@ static void initialize_rules(Gen *g) {...@@ -751,32 +854,39 @@ static void initialize_rules(Gen *g) {
751 break;854 break;
752 }855 }
753856
754 // Resolve child references into pointers857 // Iterate over the rules and
755 for (int tuple_i = 0; tuple_i < g->rules.length; tuple_i += 1) {858 // * resolve child references into pointers
756 RuleNode *node = g->rules.at(tuple_i);859 // * calculate the biggest tuple len
757 assert(node->type == RuleNodeTypeTuple);860 bool any_errors = false;
758861 for (int or_i = 0; or_i < g->rules.length; or_i += 1) {
759 for (int child_i = 0; child_i < node->tuple.children.length; child_i += 1) {862 RuleNode *or_node = g->rules.at(or_i);
760 RuleNode *child = node->tuple.children.at(child_i);863 assert(or_node->type == RuleNodeTypeOr);
761 if (child->type == RuleNodeTypeSubRule) {864
762 int line = child->lex_line + 1;865 for (int tuple_i = 0; tuple_i < or_node->_or.children.length; tuple_i += 1) {
763 int column = child->lex_column + 1;866 RuleNode *tuple_node = or_node->_or.children.at(tuple_i);
764 RuleNode *referenced_node = find_rule_node(g, &child->sub_rule.name);867 assert(tuple_node->type == RuleNodeTypeTuple);
765 if (!referenced_node) {868 g->biggest_tuple_len = max(g->biggest_tuple_len, tuple_node->tuple.children.length);
766 fprintf(stderr, "Grammar Error: Line %d, column %d: Rule not defined: '%s'\n",869
767 line, column, buf_ptr(&child->sub_rule.name));870 for (int child_i = 0; child_i < tuple_node->tuple.children.length; child_i += 1) {
871 RuleNode *child = tuple_node->tuple.children.at(child_i);
872
873 if (child->type == RuleNodeTypeSubRule) {
874 int line = child->lex_line + 1;
875 int column = child->lex_column + 1;
876 RuleNode *referenced_node = find_rule_node(g, &child->sub_rule.name);
877 if (!referenced_node) {
878 fprintf(stderr, "Grammar Error: Line %d, column %d: Rule not defined: '%s'\n",
879 line, column, buf_ptr(&child->sub_rule.name));
880 any_errors = true;
881 }
882 child->sub_rule.child = referenced_node;
768 }883 }
769 child->sub_rule.child = referenced_node;
770 }884 }
771 }885 }
772 }886 }
773887
774888 if (any_errors) {
775 // calculate the biggest tuple len889 exit(EXIT_FAILURE);
776 for (int i = 0; i < g->rules.length; i += 1) {
777 RuleNode *node = g->rules.at(i);
778 assert(node->type == RuleNodeTypeTuple);
779 g->biggest_tuple_len = max(g->biggest_tuple_len, node->tuple.children.length);
780 }890 }
781}891}
782892
...@@ -851,6 +961,19 @@ static Buf *fill_template(Buf *body, const char *result_name, Buf *field_names)...@@ -851,6 +961,19 @@ static Buf *fill_template(Buf *body, const char *result_name, Buf *field_names)
851 return result;961 return result;
852}962}
853963
964static void build_transition_table(Gen *g, ParserState *state) {
965 if (!state)
966 return;
967 if (state->index >= 0)
968 return;
969 state->index = g->transition_table.length;
970 g->transition_table.append(state);
971 for (int i = 0; i < g->tokens.length; i += 1) {
972 ParserState *other_state = state->transition[i];
973 build_transition_table(g, other_state);
974 }
975}
976
854int main(int argc, char **argv) {977int main(int argc, char **argv) {
855 const char *in_filename = argv[1];978 const char *in_filename = argv[1];
856 const char *out_filename = argv[2];979 const char *out_filename = argv[2];
...@@ -882,9 +1005,11 @@ int main(int argc, char **argv) {...@@ -882,9 +1005,11 @@ int main(int argc, char **argv) {
8821005
883 g.root = g.rules.at(0);1006 g.root = g.rules.at(0);
8841007
885 g.cur_state = create_state(&g);1008 g.start_state = create_state(&g);
886 Buf root_field_name = {0};1009 Buf root_field_name = {0};
887 gen(&g, g.root, &root_field_name);1010 ZigList<ParserState *> end_states = {0};
1011 gen(&g, g.root, &root_field_name, g.start_state, &end_states, true);
1012 build_transition_table(&g, g.start_state);
8881013
889 fprintf(out_f, "/* This file is generated by parsergen.cpp */\n");1014 fprintf(out_f, "/* This file is generated by parsergen.cpp */\n");
890 fprintf(out_f, "\n");1015 fprintf(out_f, "\n");
...@@ -950,6 +1075,9 @@ int main(int argc, char **argv) {...@@ -950,6 +1075,9 @@ int main(int argc, char **argv) {
950 CodeGen *code = state->code_gen_list.at(code_i);1075 CodeGen *code = state->code_gen_list.at(code_i);
951 switch (code->type) {1076 switch (code->type) {
952 case CodeGenTypeTransition:1077 case CodeGenTypeTransition:
1078 fprintf(out_f, " if (token->id < 0 || token->id >= %d) {\n", g.tokens.length);
1079 fprintf(out_f, " ast_invalid_token_error(buf, token);\n");
1080 fprintf(out_f, " }\n");
953 fprintf(out_f, " assert(transition[%d][token->id] >= 0);\n", state->index);1081 fprintf(out_f, " assert(transition[%d][token->id] >= 0);\n", state->index);
954 fprintf(out_f, " assert(transition[%d][token->id] < %d);\n",1082 fprintf(out_f, " assert(transition[%d][token->id] < %d);\n",
955 state->index, g.transition_table.length);1083 state->index, g.transition_table.length);
src/tokenizer.hpp+1-1
...@@ -40,6 +40,7 @@ enum TokenId {...@@ -40,6 +40,7 @@ enum TokenId {
40 TokenIdRParen = 1,40 TokenIdRParen = 1,
41 TokenIdEof = 2,41 TokenIdEof = 2,
42 TokenIdStar = 3,42 TokenIdStar = 3,
43 TokenIdPlus = 4,
43 TokenIdSymbol,44 TokenIdSymbol,
44 TokenIdKeywordFn,45 TokenIdKeywordFn,
45 TokenIdKeywordReturn,46 TokenIdKeywordReturn,
...@@ -51,7 +52,6 @@ enum TokenId {...@@ -51,7 +52,6 @@ enum TokenId {
51 TokenIdStringLiteral,52 TokenIdStringLiteral,
52 TokenIdSemicolon,53 TokenIdSemicolon,
53 TokenIdNumberLiteral,54 TokenIdNumberLiteral,
54 TokenIdPlus,
55 TokenIdColon,55 TokenIdColon,
56 TokenIdArrow,56 TokenIdArrow,
57 TokenIdDash,57 TokenIdDash,
src/util.hpp+3
...@@ -12,6 +12,9 @@...@@ -12,6 +12,9 @@
12#include <string.h>12#include <string.h>
13#include <assert.h>13#include <assert.h>
1414
15
16#define BREAKPOINT __asm("int $0x03")
17
15void zig_panic(const char *format, ...)18void zig_panic(const char *format, ...)
16 __attribute__((cold))19 __attribute__((cold))
17 __attribute__ ((noreturn))20 __attribute__ ((noreturn))