authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-06 22:11:47-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-06 22:11:47-07:00
log72be61fc0a0cb552c8eec1d9d4b8ccacf64f491c
tree9ed920d629cb02894f3f134144d7b2f1083b9f29
parent4ecb37a8a4ab552705445a7299bac266c3e06720

generated parser understands tuples

7 files changed, 244 insertions(+), 51 deletions(-)

README.md+2
...@@ -26,6 +26,8 @@ readable, safe, optimal, and concise code to solve any computing problem....@@ -26,6 +26,8 @@ readable, safe, optimal, and concise code to solve any computing problem.
26 * Resilient to parsing errors to make IDE integration work well.26 * Resilient to parsing errors to make IDE integration work well.
27 * Source code is UTF-8.27 * Source code is UTF-8.
28 * Shebang line OK so language can be used for "scripting" as well.28 * Shebang line OK so language can be used for "scripting" as well.
29 * Ability to mark functions as test and automatically run them in test mode.
30 * Memory zeroed by default, unless you initialize with "uninitialized".
2931
30## Roadmap32## Roadmap
3133
src/buffer.cpp+1-1
...@@ -36,7 +36,7 @@ void buf_appendf(Buf *buf, const char *format, ...) {...@@ -36,7 +36,7 @@ void buf_appendf(Buf *buf, const char *format, ...) {
3636
37 int orig_len = buf_len(buf);37 int orig_len = buf_len(buf);
3838
39 buf_resize(buf, orig_len + required_size);39 buf_resize(buf, orig_len + len1);
4040
41 int len2 = vsnprintf(buf_ptr(buf) + orig_len, required_size, format, ap2);41 int len2 = vsnprintf(buf_ptr(buf) + orig_len, required_size, format, ap2);
42 assert(len2 == len1);42 assert(len2 == len1);
src/buffer.hpp+8
...@@ -55,6 +55,14 @@ static inline void buf_init_from_mem(Buf *buf, const char *ptr, int len) {...@@ -55,6 +55,14 @@ static inline void buf_init_from_mem(Buf *buf, const char *ptr, int len) {
55 buf->list.at(buf_len(buf)) = 0;55 buf->list.at(buf_len(buf)) = 0;
56}56}
5757
58static inline void buf_init_from_str(Buf *buf, const char *str) {
59 buf_init_from_mem(buf, str, strlen(str));
60}
61
62static inline void buf_init_from_buf(Buf *buf, Buf *other) {
63 buf_init_from_mem(buf, buf_ptr(other), buf_len(other));
64}
65
58static inline Buf *buf_create_from_mem(const char *ptr, int len) {66static inline Buf *buf_create_from_mem(const char *ptr, int len) {
59 Buf *buf = allocate<Buf>(1);67 Buf *buf = allocate<Buf>(1);
60 buf_init_from_mem(buf, ptr, len);68 buf_init_from_mem(buf, ptr, len);
src/main.cpp-1
...@@ -28,7 +28,6 @@ static int usage(const char *arg0) {...@@ -28,7 +28,6 @@ static int usage(const char *arg0) {
28 fprintf(stderr, "Usage: %s [command] [options] target\n"28 fprintf(stderr, "Usage: %s [command] [options] target\n"
29 "Commands:\n"29 "Commands:\n"
30 " build create an executable from target\n"30 " build create an executable from target\n"
31 " link turn a .o file into an executable\n"
32 "Options:\n"31 "Options:\n"
33 " --output output file\n"32 " --output output file\n"
34 " --version print version number and exit\n"33 " --version print version number and exit\n"
src/parser.cpp+4
...@@ -78,3 +78,7 @@ void ast_print(AstNode *node, int indent) {...@@ -78,3 +78,7 @@ void ast_print(AstNode *node, int indent) {
78 break;78 break;
79 }79 }
80}80}
81
82AstNode *ast_create_root(Token *token) {
83 return nullptr;
84}
src/parser.hpp+2
...@@ -89,4 +89,6 @@ const char *node_type_str(NodeType node_type);...@@ -89,4 +89,6 @@ const char *node_type_str(NodeType node_type);
8989
90void ast_print(AstNode *node, int indent);90void ast_print(AstNode *node, int indent);
9191
92AstNode *ast_create_root(Token *token);
93
92#endif94#endif
src/parsergen.cpp+227-49
...@@ -187,29 +187,39 @@ struct RuleNode {...@@ -187,29 +187,39 @@ struct RuleNode {
187};187};
188188
189189
190enum ParserStateType {190enum CodeGenType {
191 ParserStateTypeError,191 CodeGenTypeTransition,
192 ParserStateTypeOk,192 CodeGenTypeError,
193 ParserStateTypeCapture,193 CodeGenTypeSave,
194 CodeGenTypePushNode,
195 CodeGenTypeCapture,
196 CodeGenTypePopNode,
197 CodeGenTypeEatToken,
194};198};
195199
196struct ParserStateError {200struct CodeGenError {
197 Buf *msg;201 Buf *msg;
198};202};
199203
200struct ParserStateCapture {204struct CodeGenCapture {
201 Buf *body;205 Buf *body;
206 bool is_root;
207 Buf *field_names;
208};
209
210struct CodeGen {
211 CodeGenType type;
212 union {
213 CodeGenError error;
214 CodeGenCapture capture;
215 };
202};216};
203217
204struct ParserState {218struct ParserState {
205 ParserStateType type;219 ZigList<CodeGen *> code_gen_list;
206 // One for each token ID.220 // One for each token ID.
207 ParserState **transition;221 ParserState **transition;
208 int index;222 int index;
209 union {
210 ParserStateError error;
211 ParserStateCapture capture;
212 };
213};223};
214224
215enum LexState {225enum LexState {
...@@ -234,6 +244,7 @@ struct Gen {...@@ -234,6 +244,7 @@ struct Gen {
234 ZigList<ParserState *> transition_table;244 ZigList<ParserState *> transition_table;
235 ZigList<Token *> tokens;245 ZigList<Token *> tokens;
236 RuleNode *root;246 RuleNode *root;
247 int biggest_tuple_len;
237248
238 Buf *in_buf;249 Buf *in_buf;
239 LexState lex_state;250 LexState lex_state;
...@@ -249,9 +260,8 @@ struct Gen {...@@ -249,9 +260,8 @@ struct Gen {
249 int lex_body_end;260 int lex_body_end;
250};261};
251262
252static ParserState *create_state(Gen *g, ParserStateType type) {263static ParserState *create_state(Gen *g) {
253 ParserState *state = allocate<ParserState>(1);264 ParserState *state = allocate<ParserState>(1);
254 state->type = type;
255 state->index = g->transition_table.length;265 state->index = g->transition_table.length;
256 state->transition = allocate<ParserState*>(g->tokens.length);266 state->transition = allocate<ParserState*>(g->tokens.length);
257 g->transition_table.append(state);267 g->transition_table.append(state);
...@@ -264,28 +274,93 @@ static void fill_state_with_transition(Gen *g, ParserState *source, ParserState...@@ -264,28 +274,93 @@ static void fill_state_with_transition(Gen *g, ParserState *source, ParserState
264 }274 }
265}275}
266276
267static void gen(Gen *g, RuleNode *node) {277static void state_add_code(ParserState *state, CodeGen *code) {
278 state->code_gen_list.append(code);
279}
280
281static void state_add_save_token(ParserState *state) {
282 CodeGen *code = allocate<CodeGen>(1);
283 code->type = CodeGenTypeSave;
284 state_add_code(state, code);
285}
286
287static void state_add_error(ParserState *state, Buf *msg) {
288 CodeGen *code = allocate<CodeGen>(1);
289 code->type = CodeGenTypeError;
290 code->error.msg = msg;
291 state_add_code(state, code);
292}
293
294static void state_add_transition(ParserState *state) {
295 CodeGen *code = allocate<CodeGen>(1);
296 code->type = CodeGenTypeTransition;
297 state_add_code(state, code);
298}
299
300static void state_add_push_node(ParserState *state) {
301 CodeGen *code = allocate<CodeGen>(1);
302 code->type = CodeGenTypePushNode;
303 state_add_code(state, code);
304}
305
306static CodeGen *codegen_create_capture(Buf *body, bool is_root, int field_name_count) {
307 CodeGen *code = allocate<CodeGen>(1);
308 code->type = CodeGenTypeCapture;
309 code->capture.body = body;
310 code->capture.is_root = is_root;
311 code->capture.field_names = allocate<Buf>(field_name_count);
312 return code;
313}
314
315static void state_add_pop_node(ParserState *state) {
316 CodeGen *code = allocate<CodeGen>(1);
317 code->type = CodeGenTypePopNode;
318 state_add_code(state, code);
319}
320
321static void state_add_eat_token(ParserState *state) {
322 CodeGen *code = allocate<CodeGen>(1);
323 code->type = CodeGenTypeEatToken;
324 state_add_code(state, code);
325}
326
327static void gen(Gen *g, RuleNode *node, Buf *out_field_name) {
268 switch (node->type) {328 switch (node->type) {
269 case RuleNodeTypeToken:329 case RuleNodeTypeToken:
270 {330 {
271 ParserState *ok_state = create_state(g, ParserStateTypeOk);331 buf_init_from_str(out_field_name, "token");
272 ParserState *err_state = create_state(g, ParserStateTypeError);332
333 state_add_save_token(g->cur_state);
273334
274 err_state->error.msg = buf_sprintf("expected token '%s'", buf_ptr(&node->token.token->name));335 ParserState *ok_state = create_state(g);
336 ParserState *err_state = create_state(g);
337 state_add_error(err_state, buf_sprintf("expected token '%s'", buf_ptr(&node->token.token->name)));
275338
276 fill_state_with_transition(g, g->cur_state, err_state);339 fill_state_with_transition(g, g->cur_state, err_state);
277 g->cur_state->transition[node->token.token->id] = ok_state;340 g->cur_state->transition[node->token.token->id] = ok_state;
341 state_add_transition(g->cur_state);
342 state_add_eat_token(g->cur_state);
343
278 g->cur_state = ok_state;344 g->cur_state = ok_state;
279 }345 }
280 break;346 break;
281 case RuleNodeTypeTuple:347 case RuleNodeTypeTuple:
282 {348 {
349 buf_init_from_str(out_field_name, "node");
350
351 state_add_push_node(g->cur_state);
352
353 bool is_root = (node == g->root);
354 int field_name_count = node->tuple.children.length;
355 CodeGen *code = codegen_create_capture(&node->tuple.body, is_root, field_name_count);
356
283 for (int i = 0; i < node->tuple.children.length; i += 1) {357 for (int i = 0; i < node->tuple.children.length; i += 1) {
284 RuleNode *child = node->tuple.children.at(i);358 RuleNode *child = node->tuple.children.at(i);
285 gen(g, child);359 gen(g, child, &code->capture.field_names[i]);
286 }360 }
287 g->cur_state->type = ParserStateTypeCapture;361 state_add_code(g->cur_state, code);
288 g->cur_state->capture.body = &node->tuple.body;362
363 state_add_pop_node(g->cur_state);
289 }364 }
290 break;365 break;
291 case RuleNodeTypeMany:366 case RuleNodeTypeMany:
...@@ -301,7 +376,10 @@ static void gen(Gen *g, RuleNode *node) {...@@ -301,7 +376,10 @@ static void gen(Gen *g, RuleNode *node) {
301 zig_panic("TODO");376 zig_panic("TODO");
302 break;377 break;
303 case RuleNodeTypeSubRule:378 case RuleNodeTypeSubRule:
304 zig_panic("TODO");379 {
380 RuleNode *child = node->sub_rule.child;
381 gen(g, child, out_field_name);
382 }
305 break;383 break;
306 }384 }
307}385}
...@@ -407,6 +485,9 @@ static void end_token_name(Gen *g) {...@@ -407,6 +485,9 @@ static void end_token_name(Gen *g) {
407 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);485 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
408 g->lex_cur_rule->tuple.children.append(node);486 g->lex_cur_rule->tuple.children.append(node);
409487
488 g->biggest_tuple_len = max(g->biggest_tuple_len, g->lex_cur_rule->tuple.children.length);
489
490
410 lex_pop_stack(g);491 lex_pop_stack(g);
411}492}
412493
...@@ -571,6 +652,76 @@ static void initialize_rules(Gen *g) {...@@ -571,6 +652,76 @@ static void initialize_rules(Gen *g) {
571 }652 }
572}653}
573654
655enum TemplateState {
656 TemplateStateStart,
657 TemplateStateDollar,
658 TemplateStateNumber,
659};
660
661static Buf *fill_template(Buf *body, const char *result_name, Buf *field_names) {
662 //fprintf(stderr, "fill template input:\n%s\n", buf_ptr(body));
663 Buf *result = buf_alloc();
664 TemplateState state = TemplateStateStart;
665 int digit_start;
666 for (int i = 0; i < buf_len(body); i += 1) {
667 uint8_t c = buf_ptr(body)[i];
668 switch (state) {
669 case TemplateStateStart:
670 switch (c) {
671 case '$':
672 state = TemplateStateDollar;
673 break;
674 default:
675 buf_append_char(result, c);
676 break;
677 }
678 break;
679 case TemplateStateDollar:
680 switch (c) {
681 case '$':
682 buf_append_str(result, result_name);
683 state = TemplateStateStart;
684 break;
685 case DIGIT:
686 digit_start = i;
687 state = TemplateStateNumber;
688 break;
689 default:
690 buf_append_char(result, '$');
691 buf_append_char(result, c);
692 state = TemplateStateStart;
693 break;
694 }
695 break;
696 case TemplateStateNumber:
697 switch (c) {
698 case DIGIT:
699 // nothing
700 break;
701 default:
702 {
703 Buf *num_buf = buf_create_from_mem(&buf_ptr(body)[digit_start], i - digit_start);
704 int index = atoi(buf_ptr(num_buf)) - 1;
705 buf_appendf(result, "(top_node->data[%d].%s)%c",
706 index, buf_ptr(&field_names[index]), c);
707
708 state = TemplateStateStart;
709 }
710 break;
711 }
712 break;
713 }
714 }
715 switch (state) {
716 case TemplateStateStart:
717 // OK
718 break;
719 default:
720 zig_panic("unable to fill grammar template");
721 }
722 //fprintf(stderr, "fill template output:\n%s\n", buf_ptr(result));
723 return result;
724}
574725
575int main(int argc, char **argv) {726int main(int argc, char **argv) {
576 const char *in_filename = argv[1];727 const char *in_filename = argv[1];
...@@ -603,8 +754,9 @@ int main(int argc, char **argv) {...@@ -603,8 +754,9 @@ int main(int argc, char **argv) {
603754
604 g.root = g.rules.at(0);755 g.root = g.rules.at(0);
605756
606 g.cur_state = create_state(&g, ParserStateTypeOk);757 g.cur_state = create_state(&g);
607 gen(&g, g.root);758 Buf root_field_name = {0};
759 gen(&g, g.root, &root_field_name);
608760
609 fprintf(out_f, "/* This file is generated by parsergen.cpp */\n");761 fprintf(out_f, "/* This file is generated by parsergen.cpp */\n");
610 fprintf(out_f, "\n");762 fprintf(out_f, "\n");
...@@ -627,15 +779,14 @@ int main(int argc, char **argv) {...@@ -627,15 +779,14 @@ int main(int argc, char **argv) {
627 }779 }
628 fprintf(out_f, "\n");780 fprintf(out_f, "\n");
629781
630 /* TODO782 fprintf(out_f, "struct ParserGenNode {\n");
631 fprintf(out_f, "struct ParserGenNode{\n");783 fprintf(out_f, " int next_index;\n");
632 fprintf(out_f, " union {\n");784 fprintf(out_f, " union {\n");
633 fprintf(out_f, " [%d];\n", biggest_tuple_len);
634 fprintf(out_f, " Token *token;\n");785 fprintf(out_f, " Token *token;\n");
635 fprintf(out_f, " };\n");786 fprintf(out_f, " AstNode *node;\n");
787 fprintf(out_f, " } data[%d];\n", g.biggest_tuple_len);
636 fprintf(out_f, "};\n");788 fprintf(out_f, "};\n");
637 fprintf(out_f, "\n");789 fprintf(out_f, "\n");
638 */
639790
640 fprintf(out_f, "AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens) {\n");791 fprintf(out_f, "AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens) {\n");
641792
...@@ -655,39 +806,66 @@ int main(int argc, char **argv) {...@@ -655,39 +806,66 @@ int main(int argc, char **argv) {
655806
656807
657 fprintf(out_f, " int state = 0;\n");808 fprintf(out_f, " int state = 0;\n");
809 fprintf(out_f, " int token_index = 0;\n");
810 fprintf(out_f, " Token *token = &tokens->at(token_index);\n");
658 fprintf(out_f, " AstNode *root = nullptr;\n");811 fprintf(out_f, " AstNode *root = nullptr;\n");
812 fprintf(out_f, " ZigList<ParserGenNode *> stack = {0};\n");
813 fprintf(out_f, " ParserGenNode *top_node = nullptr;\n");
659814
660 fprintf(out_f, " for (int i = 0; i < tokens->length; i += 1) {\n");815 fprintf(out_f, " for (;;) {\n");
661 fprintf(out_f, " Token *token = &tokens->at(i);\n");
662 fprintf(out_f, " switch (state) {\n");816 fprintf(out_f, " switch (state) {\n");
663817
664 for (int i = 0; i < g.transition_table.length; i += 1) {818 for (int state_i = 0; state_i < g.transition_table.length; state_i += 1) {
665 ParserState *state = g.transition_table.at(i);819 ParserState *state = g.transition_table.at(state_i);
666 fprintf(out_f, " case %d:\n", i);820 fprintf(out_f, " case %d: {\n", state_i);
667 switch (state->type) {821 for (int code_i = 0; code_i < state->code_gen_list.length; code_i += 1) {
668 case ParserStateTypeError:822 CodeGen *code = state->code_gen_list.at(code_i);
669 fprintf(out_f, " ast_error(token, \"%s\");\n", buf_ptr(state->error.msg));823 switch (code->type) {
670 break;824 case CodeGenTypeTransition:
671 case ParserStateTypeOk:825 fprintf(out_f, " assert(transition[%d][token->id] >= 0);\n", state->index);
672 fprintf(out_f, " assert(transition[%d][token->id] >= 0);\n", state->index);826 fprintf(out_f, " assert(transition[%d][token->id] < %d);\n",
673 fprintf(out_f, " assert(transition[%d][token->id] < %d);\n",827 state->index, g.transition_table.length);
674 state->index, g.transition_table.length);828 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);
675 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);829 break;
676 break;830 case CodeGenTypeError:
677 case ParserStateTypeCapture:831 fprintf(out_f, " ast_error(token, \"%s\");\n", buf_ptr(code->error.msg));
678 // TODO fprintf(out_f, " %s\n", buf_ptr(state->capture.body));832 break;
679 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);833 case CodeGenTypeSave:
680 break;834 fprintf(out_f, " top_node->data[top_node->next_index++].token = token;\n");
835 break;
836 case CodeGenTypePushNode:
837 fprintf(out_f, " top_node = allocate<ParserGenNode>(1);\n");
838 fprintf(out_f, " stack.append(top_node);\n");
839 break;
840 case CodeGenTypeCapture:
841 if (code->capture.is_root) {
842 Buf *code_text = fill_template(code->capture.body, "root", code->capture.field_names);
843 fprintf(out_f, "%s\n", buf_ptr(code_text));
844 fprintf(out_f, " return root;\n");
845 } else {
846 zig_panic("TODO capture non-root");
847 }
848 break;
849 case CodeGenTypePopNode:
850 fprintf(out_f, " stack.pop();\n");
851 fprintf(out_f, " top_node = stack.length ? stack.last() : nullptr;\n");
852 break;
853 case CodeGenTypeEatToken:
854 fprintf(out_f, " token_index += 1;\n");
855 fprintf(out_f, " token = (token_index < tokens->length) ? &tokens->at(token_index) : nullptr;\n");
856 break;
857 }
681 }858 }
682 fprintf(out_f, " break;\n");859 fprintf(out_f, " break;\n");
860 fprintf(out_f, " }\n");
683 }861 }
684 fprintf(out_f, " default:\n");862 fprintf(out_f, " default:\n");
685 fprintf(out_f, " zig_panic(\"unreachable\");\n");863 fprintf(out_f, " zig_panic(\"unreachable\");\n");
686864
687 fprintf(out_f, " }\n");865 fprintf(out_f, " }\n");
688 fprintf(out_f, " }\n");866 fprintf(out_f, " }\n");
689 fprintf(out_f, " return root;\n");867 fprintf(out_f, " zig_panic(\"unreachable\");\n");
690 fprintf(out_f, "}\n");868 fprintf(out_f, "}\n");
691869
692870 return 0;
693}871}