authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-06 22:11:47-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-06 22:11:47-07:00
log72be61fc0a0cb552c8eec1d9d4b8ccacf64f491c
tree9ed920d629cb02894f3f134144d7b2f1083b9f29
parent4ecb37a8a4ab552705445a7299bac266c3e06720

generated parser understands tuples

7 files changed, 244 insertions(+), 51 deletions(-)

README.md+2
......@@ -26,6 +26,8 @@ readable, safe, optimal, and concise code to solve any computing problem.
2626 * Resilient to parsing errors to make IDE integration work well.
2727 * Source code is UTF-8.
2828 * Shebang line OK so language can be used for "scripting" as well.
29 * Ability to mark functions as test and automatically run them in test mode.
30 * Memory zeroed by default, unless you initialize with "uninitialized".
2931
3032## Roadmap
3133
src/buffer.cpp+1-1
......@@ -36,7 +36,7 @@ void buf_appendf(Buf *buf, const char *format, ...) {
3636
3737 int orig_len = buf_len(buf);
3838
39 buf_resize(buf, orig_len + required_size);
39 buf_resize(buf, orig_len + len1);
4040
4141 int len2 = vsnprintf(buf_ptr(buf) + orig_len, required_size, format, ap2);
4242 assert(len2 == len1);
src/buffer.hpp+8
......@@ -55,6 +55,14 @@ static inline void buf_init_from_mem(Buf *buf, const char *ptr, int len) {
5555 buf->list.at(buf_len(buf)) = 0;
5656}
5757
58static inline void buf_init_from_str(Buf *buf, const char *str) {
59 buf_init_from_mem(buf, str, strlen(str));
60}
61
62static inline void buf_init_from_buf(Buf *buf, Buf *other) {
63 buf_init_from_mem(buf, buf_ptr(other), buf_len(other));
64}
65
5866static inline Buf *buf_create_from_mem(const char *ptr, int len) {
5967 Buf *buf = allocate<Buf>(1);
6068 buf_init_from_mem(buf, ptr, len);
src/main.cpp-1
......@@ -28,7 +28,6 @@ static int usage(const char *arg0) {
2828 fprintf(stderr, "Usage: %s [command] [options] target\n"
2929 "Commands:\n"
3030 " build create an executable from target\n"
31 " link turn a .o file into an executable\n"
3231 "Options:\n"
3332 " --output output file\n"
3433 " --version print version number and exit\n"
src/parser.cpp+4
......@@ -78,3 +78,7 @@ void ast_print(AstNode *node, int indent) {
7878 break;
7979 }
8080}
81
82AstNode *ast_create_root(Token *token) {
83 return nullptr;
84}
src/parser.hpp+2
......@@ -89,4 +89,6 @@ const char *node_type_str(NodeType node_type);
8989
9090void ast_print(AstNode *node, int indent);
9191
92AstNode *ast_create_root(Token *token);
93
9294#endif
src/parsergen.cpp+227-49
......@@ -187,29 +187,39 @@ struct RuleNode {
187187};
188188
189189
190enum ParserStateType {
191 ParserStateTypeError,
192 ParserStateTypeOk,
193 ParserStateTypeCapture,
190enum CodeGenType {
191 CodeGenTypeTransition,
192 CodeGenTypeError,
193 CodeGenTypeSave,
194 CodeGenTypePushNode,
195 CodeGenTypeCapture,
196 CodeGenTypePopNode,
197 CodeGenTypeEatToken,
194198};
195199
196struct ParserStateError {
200struct CodeGenError {
197201 Buf *msg;
198202};
199203
200struct ParserStateCapture {
204struct CodeGenCapture {
201205 Buf *body;
206 bool is_root;
207 Buf *field_names;
208};
209
210struct CodeGen {
211 CodeGenType type;
212 union {
213 CodeGenError error;
214 CodeGenCapture capture;
215 };
202216};
203217
204218struct ParserState {
205 ParserStateType type;
219 ZigList<CodeGen *> code_gen_list;
206220 // One for each token ID.
207221 ParserState **transition;
208222 int index;
209 union {
210 ParserStateError error;
211 ParserStateCapture capture;
212 };
213223};
214224
215225enum LexState {
......@@ -234,6 +244,7 @@ struct Gen {
234244 ZigList<ParserState *> transition_table;
235245 ZigList<Token *> tokens;
236246 RuleNode *root;
247 int biggest_tuple_len;
237248
238249 Buf *in_buf;
239250 LexState lex_state;
......@@ -249,9 +260,8 @@ struct Gen {
249260 int lex_body_end;
250261};
251262
252static ParserState *create_state(Gen *g, ParserStateType type) {
263static ParserState *create_state(Gen *g) {
253264 ParserState *state = allocate<ParserState>(1);
254 state->type = type;
255265 state->index = g->transition_table.length;
256266 state->transition = allocate<ParserState*>(g->tokens.length);
257267 g->transition_table.append(state);
......@@ -264,28 +274,93 @@ static void fill_state_with_transition(Gen *g, ParserState *source, ParserState
264274 }
265275}
266276
267static void gen(Gen *g, RuleNode *node) {
277static void state_add_code(ParserState *state, CodeGen *code) {
278 state->code_gen_list.append(code);
279}
280
281static void state_add_save_token(ParserState *state) {
282 CodeGen *code = allocate<CodeGen>(1);
283 code->type = CodeGenTypeSave;
284 state_add_code(state, code);
285}
286
287static void state_add_error(ParserState *state, Buf *msg) {
288 CodeGen *code = allocate<CodeGen>(1);
289 code->type = CodeGenTypeError;
290 code->error.msg = msg;
291 state_add_code(state, code);
292}
293
294static void state_add_transition(ParserState *state) {
295 CodeGen *code = allocate<CodeGen>(1);
296 code->type = CodeGenTypeTransition;
297 state_add_code(state, code);
298}
299
300static void state_add_push_node(ParserState *state) {
301 CodeGen *code = allocate<CodeGen>(1);
302 code->type = CodeGenTypePushNode;
303 state_add_code(state, code);
304}
305
306static CodeGen *codegen_create_capture(Buf *body, bool is_root, int field_name_count) {
307 CodeGen *code = allocate<CodeGen>(1);
308 code->type = CodeGenTypeCapture;
309 code->capture.body = body;
310 code->capture.is_root = is_root;
311 code->capture.field_names = allocate<Buf>(field_name_count);
312 return code;
313}
314
315static void state_add_pop_node(ParserState *state) {
316 CodeGen *code = allocate<CodeGen>(1);
317 code->type = CodeGenTypePopNode;
318 state_add_code(state, code);
319}
320
321static void state_add_eat_token(ParserState *state) {
322 CodeGen *code = allocate<CodeGen>(1);
323 code->type = CodeGenTypeEatToken;
324 state_add_code(state, code);
325}
326
327static void gen(Gen *g, RuleNode *node, Buf *out_field_name) {
268328 switch (node->type) {
269329 case RuleNodeTypeToken:
270330 {
271 ParserState *ok_state = create_state(g, ParserStateTypeOk);
272 ParserState *err_state = create_state(g, ParserStateTypeError);
331 buf_init_from_str(out_field_name, "token");
332
333 state_add_save_token(g->cur_state);
273334
274 err_state->error.msg = buf_sprintf("expected token '%s'", buf_ptr(&node->token.token->name));
335 ParserState *ok_state = create_state(g);
336 ParserState *err_state = create_state(g);
337 state_add_error(err_state, buf_sprintf("expected token '%s'", buf_ptr(&node->token.token->name)));
275338
276339 fill_state_with_transition(g, g->cur_state, err_state);
277340 g->cur_state->transition[node->token.token->id] = ok_state;
341 state_add_transition(g->cur_state);
342 state_add_eat_token(g->cur_state);
343
278344 g->cur_state = ok_state;
279345 }
280346 break;
281347 case RuleNodeTypeTuple:
282348 {
349 buf_init_from_str(out_field_name, "node");
350
351 state_add_push_node(g->cur_state);
352
353 bool is_root = (node == g->root);
354 int field_name_count = node->tuple.children.length;
355 CodeGen *code = codegen_create_capture(&node->tuple.body, is_root, field_name_count);
356
283357 for (int i = 0; i < node->tuple.children.length; i += 1) {
284358 RuleNode *child = node->tuple.children.at(i);
285 gen(g, child);
359 gen(g, child, &code->capture.field_names[i]);
286360 }
287 g->cur_state->type = ParserStateTypeCapture;
288 g->cur_state->capture.body = &node->tuple.body;
361 state_add_code(g->cur_state, code);
362
363 state_add_pop_node(g->cur_state);
289364 }
290365 break;
291366 case RuleNodeTypeMany:
......@@ -301,7 +376,10 @@ static void gen(Gen *g, RuleNode *node) {
301376 zig_panic("TODO");
302377 break;
303378 case RuleNodeTypeSubRule:
304 zig_panic("TODO");
379 {
380 RuleNode *child = node->sub_rule.child;
381 gen(g, child, out_field_name);
382 }
305383 break;
306384 }
307385}
......@@ -407,6 +485,9 @@ static void end_token_name(Gen *g) {
407485 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
408486 g->lex_cur_rule->tuple.children.append(node);
409487
488 g->biggest_tuple_len = max(g->biggest_tuple_len, g->lex_cur_rule->tuple.children.length);
489
490
410491 lex_pop_stack(g);
411492}
412493
......@@ -571,6 +652,76 @@ static void initialize_rules(Gen *g) {
571652 }
572653}
573654
655enum TemplateState {
656 TemplateStateStart,
657 TemplateStateDollar,
658 TemplateStateNumber,
659};
660
661static Buf *fill_template(Buf *body, const char *result_name, Buf *field_names) {
662 //fprintf(stderr, "fill template input:\n%s\n", buf_ptr(body));
663 Buf *result = buf_alloc();
664 TemplateState state = TemplateStateStart;
665 int digit_start;
666 for (int i = 0; i < buf_len(body); i += 1) {
667 uint8_t c = buf_ptr(body)[i];
668 switch (state) {
669 case TemplateStateStart:
670 switch (c) {
671 case '$':
672 state = TemplateStateDollar;
673 break;
674 default:
675 buf_append_char(result, c);
676 break;
677 }
678 break;
679 case TemplateStateDollar:
680 switch (c) {
681 case '$':
682 buf_append_str(result, result_name);
683 state = TemplateStateStart;
684 break;
685 case DIGIT:
686 digit_start = i;
687 state = TemplateStateNumber;
688 break;
689 default:
690 buf_append_char(result, '$');
691 buf_append_char(result, c);
692 state = TemplateStateStart;
693 break;
694 }
695 break;
696 case TemplateStateNumber:
697 switch (c) {
698 case DIGIT:
699 // nothing
700 break;
701 default:
702 {
703 Buf *num_buf = buf_create_from_mem(&buf_ptr(body)[digit_start], i - digit_start);
704 int index = atoi(buf_ptr(num_buf)) - 1;
705 buf_appendf(result, "(top_node->data[%d].%s)%c",
706 index, buf_ptr(&field_names[index]), c);
707
708 state = TemplateStateStart;
709 }
710 break;
711 }
712 break;
713 }
714 }
715 switch (state) {
716 case TemplateStateStart:
717 // OK
718 break;
719 default:
720 zig_panic("unable to fill grammar template");
721 }
722 //fprintf(stderr, "fill template output:\n%s\n", buf_ptr(result));
723 return result;
724}
574725
575726int main(int argc, char **argv) {
576727 const char *in_filename = argv[1];
......@@ -603,8 +754,9 @@ int main(int argc, char **argv) {
603754
604755 g.root = g.rules.at(0);
605756
606 g.cur_state = create_state(&g, ParserStateTypeOk);
607 gen(&g, g.root);
757 g.cur_state = create_state(&g);
758 Buf root_field_name = {0};
759 gen(&g, g.root, &root_field_name);
608760
609761 fprintf(out_f, "/* This file is generated by parsergen.cpp */\n");
610762 fprintf(out_f, "\n");
......@@ -627,15 +779,14 @@ int main(int argc, char **argv) {
627779 }
628780 fprintf(out_f, "\n");
629781
630 /* TODO
631 fprintf(out_f, "struct ParserGenNode{\n");
782 fprintf(out_f, "struct ParserGenNode {\n");
783 fprintf(out_f, " int next_index;\n");
632784 fprintf(out_f, " union {\n");
633 fprintf(out_f, " [%d];\n", biggest_tuple_len);
634785 fprintf(out_f, " Token *token;\n");
635 fprintf(out_f, " };\n");
786 fprintf(out_f, " AstNode *node;\n");
787 fprintf(out_f, " } data[%d];\n", g.biggest_tuple_len);
636788 fprintf(out_f, "};\n");
637789 fprintf(out_f, "\n");
638 */
639790
640791 fprintf(out_f, "AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens) {\n");
641792
......@@ -655,39 +806,66 @@ int main(int argc, char **argv) {
655806
656807
657808 fprintf(out_f, " int state = 0;\n");
809 fprintf(out_f, " int token_index = 0;\n");
810 fprintf(out_f, " Token *token = &tokens->at(token_index);\n");
658811 fprintf(out_f, " AstNode *root = nullptr;\n");
812 fprintf(out_f, " ZigList<ParserGenNode *> stack = {0};\n");
813 fprintf(out_f, " ParserGenNode *top_node = nullptr;\n");
659814
660 fprintf(out_f, " for (int i = 0; i < tokens->length; i += 1) {\n");
661 fprintf(out_f, " Token *token = &tokens->at(i);\n");
815 fprintf(out_f, " for (;;) {\n");
662816 fprintf(out_f, " switch (state) {\n");
663817
664 for (int i = 0; i < g.transition_table.length; i += 1) {
665 ParserState *state = g.transition_table.at(i);
666 fprintf(out_f, " case %d:\n", i);
667 switch (state->type) {
668 case ParserStateTypeError:
669 fprintf(out_f, " ast_error(token, \"%s\");\n", buf_ptr(state->error.msg));
670 break;
671 case ParserStateTypeOk:
672 fprintf(out_f, " assert(transition[%d][token->id] >= 0);\n", state->index);
673 fprintf(out_f, " assert(transition[%d][token->id] < %d);\n",
674 state->index, g.transition_table.length);
675 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);
676 break;
677 case ParserStateTypeCapture:
678 // TODO fprintf(out_f, " %s\n", buf_ptr(state->capture.body));
679 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);
680 break;
818 for (int state_i = 0; state_i < g.transition_table.length; state_i += 1) {
819 ParserState *state = g.transition_table.at(state_i);
820 fprintf(out_f, " case %d: {\n", state_i);
821 for (int code_i = 0; code_i < state->code_gen_list.length; code_i += 1) {
822 CodeGen *code = state->code_gen_list.at(code_i);
823 switch (code->type) {
824 case CodeGenTypeTransition:
825 fprintf(out_f, " assert(transition[%d][token->id] >= 0);\n", state->index);
826 fprintf(out_f, " assert(transition[%d][token->id] < %d);\n",
827 state->index, g.transition_table.length);
828 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);
829 break;
830 case CodeGenTypeError:
831 fprintf(out_f, " ast_error(token, \"%s\");\n", buf_ptr(code->error.msg));
832 break;
833 case CodeGenTypeSave:
834 fprintf(out_f, " top_node->data[top_node->next_index++].token = token;\n");
835 break;
836 case CodeGenTypePushNode:
837 fprintf(out_f, " top_node = allocate<ParserGenNode>(1);\n");
838 fprintf(out_f, " stack.append(top_node);\n");
839 break;
840 case CodeGenTypeCapture:
841 if (code->capture.is_root) {
842 Buf *code_text = fill_template(code->capture.body, "root", code->capture.field_names);
843 fprintf(out_f, "%s\n", buf_ptr(code_text));
844 fprintf(out_f, " return root;\n");
845 } else {
846 zig_panic("TODO capture non-root");
847 }
848 break;
849 case CodeGenTypePopNode:
850 fprintf(out_f, " stack.pop();\n");
851 fprintf(out_f, " top_node = stack.length ? stack.last() : nullptr;\n");
852 break;
853 case CodeGenTypeEatToken:
854 fprintf(out_f, " token_index += 1;\n");
855 fprintf(out_f, " token = (token_index < tokens->length) ? &tokens->at(token_index) : nullptr;\n");
856 break;
857 }
681858 }
682859 fprintf(out_f, " break;\n");
860 fprintf(out_f, " }\n");
683861 }
684862 fprintf(out_f, " default:\n");
685863 fprintf(out_f, " zig_panic(\"unreachable\");\n");
686864
687865 fprintf(out_f, " }\n");
688866 fprintf(out_f, " }\n");
689 fprintf(out_f, " return root;\n");
867 fprintf(out_f, " zig_panic(\"unreachable\");\n");
690868 fprintf(out_f, "}\n");
691869
692
870 return 0;
693871}