authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-04 00:07:24-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-04 00:07:24-07:00
logc36cd9d313d5069898233188f72b0d2b5fe8f349
tree0b8f65c114edbf24e595e31acd862d834a42e8df
parent7cfceeca2d5965baf6dcd45f07ebeaedbe48856f

parsergen parsing a simple grammar

2 files changed, 379 insertions(+), 49 deletions(-)

src/grammar.txt+11-11
...@@ -1,44 +1,44 @@...@@ -1,44 +1,44 @@
1Root : many(FnDecl) token(EOF) {1Root : many(FnDecl) token(EOF) {
2 $$ = ast_create_root($1);2 $$ = ast_create_root($1);
3}3};
44
5FnDecl : token(Fn) token(Symbol) token(LParen) list(ParamDecl, token(Comma)) token(RParen) option(token(Arrow) Type) Block {5FnDecl : token(Fn) token(Symbol) token(LParen) list(ParamDecl, token(Comma)) token(RParen) option(token(Arrow) Type) Block {
6 $$ = ast_create_fn_decl($2, $4, $6, $7);6 $$ = ast_create_fn_decl($2, $4, $6, $7);
7}7};
88
9ParamDecl : token(Symbol) token(Colon) Type {9ParamDecl : token(Symbol) token(Colon) Type {
10 $$ = ast_create_param_decl($1, $2);10 $$ = ast_create_param_decl($1, $2);
11}11};
1212
13Type : token(Symbol) {13Type : token(Symbol) {
14 $$ = ast_create_symbol_type($1);14 $$ = ast_create_symbol_type($1);
15} | PointerType {15} | PointerType {
16 $$ = $1;16 $$ = $1;
17}17};
1818
19PointerType : token(Star) token(Const) Type {19PointerType : token(Star) token(Const) Type {
20 $$ = ast_create_pointer_type($2, $3);20 $$ = ast_create_pointer_type($2, $3);
21} | token(Star) token(Mut) Type {21} | token(Star) token(Mut) Type {
22 $$ = ast_create_pointer_type($2, $3);22 $$ = ast_create_pointer_type($2, $3);
23}23};
2424
25Block : token(LBrace) many(Statement) option(Expression) token(RBrace) {25Block : token(LBrace) many(Statement) option(Expression) token(RBrace) {
26 $$ = ast_create_block($2, $3);26 $$ = ast_create_block($2, $3);
27}27};
2828
29Statement : ExpressionStatement {29Statement : ExpressionStatement {
30 $$ = $1;30 $$ = $1;
31} | ReturnStatement {31} | ReturnStatement {
32 $$ = $1;32 $$ = $1;
33}33};
3434
35ExpressionStatement : Expression token(Semicolon) {35ExpressionStatement : Expression token(Semicolon) {
36 $$ = ast_create_expression_statement($1);36 $$ = ast_create_expression_statement($1);
37}37};
3838
39ReturnStatement : token(Return) Expression token(Semicolon) {39ReturnStatement : token(Return) Expression token(Semicolon) {
40 $$ = ast_create_return_statement($2);40 $$ = ast_create_return_statement($2);
41}41};
4242
43Expression : token(Number) {43Expression : token(Number) {
44 $$ = ast_create_number($1);44 $$ = ast_create_number($1);
...@@ -46,8 +46,8 @@ Expression : token(Number) {...@@ -46,8 +46,8 @@ Expression : token(Number) {
46 $$ = ast_create_string($1);46 $$ = ast_create_string($1);
47} | FnCall {47} | FnCall {
48 $$ = $1;48 $$ = $1;
49}49};
5050
51FnCall : token(Symbol) token(LParen) list(Expression, token(Comma)) token(RParen) {51FnCall : token(Symbol) token(LParen) list(Expression, token(Comma)) token(RParen) {
52 $$ = ast_create_fn_call($1, $3);52 $$ = ast_create_fn_call($1, $3);
53}53};
src/parsergen.cpp+368-38
...@@ -10,12 +10,98 @@...@@ -10,12 +10,98 @@
10#include "list.hpp"10#include "list.hpp"
1111
12#include <stdio.h>12#include <stdio.h>
13#include <stdarg.h>
13#include <sys/types.h>14#include <sys/types.h>
14#include <sys/stat.h>15#include <sys/stat.h>
15#include <unistd.h>16#include <unistd.h>
16#include <errno.h>17#include <errno.h>
17#include <limits.h>18#include <limits.h>
1819
20#define WHITESPACE \
21 ' ': \
22 case '\t': \
23 case '\n': \
24 case '\f': \
25 case '\r': \
26 case 0xb
27
28#define DIGIT \
29 '0': \
30 case '1': \
31 case '2': \
32 case '3': \
33 case '4': \
34 case '5': \
35 case '6': \
36 case '7': \
37 case '8': \
38 case '9'
39
40#define LOWER_ALPHA \
41 'a': \
42 case 'b': \
43 case 'c': \
44 case 'd': \
45 case 'e': \
46 case 'f': \
47 case 'g': \
48 case 'h': \
49 case 'i': \
50 case 'j': \
51 case 'k': \
52 case 'l': \
53 case 'm': \
54 case 'n': \
55 case 'o': \
56 case 'p': \
57 case 'q': \
58 case 'r': \
59 case 's': \
60 case 't': \
61 case 'u': \
62 case 'v': \
63 case 'w': \
64 case 'x': \
65 case 'y': \
66 case 'z'
67
68#define UPPER_ALPHA \
69 'A': \
70 case 'B': \
71 case 'C': \
72 case 'D': \
73 case 'E': \
74 case 'F': \
75 case 'G': \
76 case 'H': \
77 case 'I': \
78 case 'J': \
79 case 'K': \
80 case 'L': \
81 case 'M': \
82 case 'N': \
83 case 'O': \
84 case 'P': \
85 case 'Q': \
86 case 'R': \
87 case 'S': \
88 case 'T': \
89 case 'U': \
90 case 'V': \
91 case 'W': \
92 case 'X': \
93 case 'Y': \
94 case 'Z'
95
96#define ALPHA \
97 LOWER_ALPHA: \
98 case UPPER_ALPHA
99
100#define SYMBOL_CHAR \
101 ALPHA: \
102 case DIGIT: \
103 case '_'
104
19static Buf *fetch_file(FILE *f) {105static Buf *fetch_file(FILE *f) {
20 int fd = fileno(f);106 int fd = fileno(f);
21 struct stat st;107 struct stat st;
...@@ -47,7 +133,9 @@ struct Token {...@@ -47,7 +133,9 @@ struct Token {
47struct RuleNode;133struct RuleNode;
48134
49struct RuleTuple {135struct RuleTuple {
136 Buf name;
50 ZigList<RuleNode *> children;137 ZigList<RuleNode *> children;
138 Buf body;
51};139};
52140
53struct RuleMany {141struct RuleMany {
...@@ -66,10 +154,6 @@ struct RuleToken {...@@ -66,10 +154,6 @@ struct RuleToken {
66 Token *token;154 Token *token;
67};155};
68156
69struct RuleBlock {
70 Buf *body;
71};
72
73struct RuleList {157struct RuleList {
74 RuleNode *rule;158 RuleNode *rule;
75 RuleToken *separator;159 RuleToken *separator;
...@@ -102,6 +186,7 @@ struct RuleNode {...@@ -102,6 +186,7 @@ struct RuleNode {
102 };186 };
103};187};
104188
189
105enum ParserStateType {190enum ParserStateType {
106 ParserStateTypeError,191 ParserStateTypeError,
107 ParserStateTypeOk,192 ParserStateTypeOk,
...@@ -121,11 +206,41 @@ struct ParserState {...@@ -121,11 +206,41 @@ struct ParserState {
121 };206 };
122};207};
123208
209enum LexState {
210 LexStateStart,
211 LexStateRuleName,
212 LexStateWaitForColon,
213 LexStateTupleRule,
214 LexStateFnName,
215 LexStateTokenStart,
216 LexStateToken,
217 LexStateBody,
218 LexStateEndOrOr,
219};
220
221struct LexStack {
222 LexState state;
223};
224
124struct Gen {225struct Gen {
226 ZigList<RuleNode *> rules;
125 ParserState *cur_state;227 ParserState *cur_state;
126 ZigList<ParserState *> transition_table;228 ZigList<ParserState *> transition_table;
127 ZigList<Token *> tokens;229 ZigList<Token *> tokens;
128 RuleNode *root;230 RuleNode *root;
231
232 Buf *in_buf;
233 LexState lex_state;
234 int lex_line;
235 int lex_column;
236 RuleNode *lex_cur_rule;
237 int lex_cur_rule_begin;
238 int lex_fn_name_begin;
239 int lex_pos;
240 ZigList<LexStack> lex_stack;
241 int lex_token_name_begin;
242 int lex_body_begin;
243 int lex_body_end;
129};244};
130245
131static ParserState *create_state(Gen *g, ParserStateType type) {246static ParserState *create_state(Gen *g, ParserStateType type) {
...@@ -203,6 +318,252 @@ static Token *find_or_create_token(Gen *g, Buf *name) {...@@ -203,6 +318,252 @@ static Token *find_or_create_token(Gen *g, Buf *name) {
203 return token;318 return token;
204}319}
205320
321__attribute__ ((format (printf, 2, 3)))
322static void lex_error(Gen *g, const char *format, ...) {
323 int line = g->lex_line + 1;
324 int column = g->lex_column + 1;
325
326 va_list ap;
327 va_start(ap, format);
328 fprintf(stderr, "Error: Line %d, column %d: ", line, column);
329 vfprintf(stderr, format, ap);
330 fprintf(stderr, "\n");
331 va_end(ap);
332 exit(EXIT_FAILURE);
333}
334
335static void lex_push_stack(Gen *g) {
336 g->lex_stack.append({g->lex_state});
337}
338
339static void lex_pop_stack(Gen *g) {
340 LexStack *entry = &g->lex_stack.last();
341 g->lex_state = entry->state;
342 g->lex_stack.pop();
343}
344
345
346static void begin_rule(Gen *g) {
347 assert(!g->lex_cur_rule);
348 g->lex_cur_rule = allocate<RuleNode>(1);
349 g->lex_cur_rule->type = RuleNodeTypeTuple;
350 g->lex_cur_rule_begin = g->lex_pos;
351
352 g->lex_state = LexStateEndOrOr;
353 lex_push_stack(g);
354}
355
356static void end_rule(Gen *g) {
357 assert(g->lex_cur_rule);
358 g->rules.append(g->lex_cur_rule);
359 g->lex_cur_rule = nullptr;
360}
361
362static void end_rule_name(Gen *g) {
363 assert(g->lex_cur_rule);
364 char *ptr = &buf_ptr(g->in_buf)[g->lex_cur_rule_begin];
365 int len = g->lex_pos - g->lex_cur_rule_begin;
366 buf_init_from_mem(&g->lex_cur_rule->tuple.name, ptr, len);
367}
368
369static void begin_fn_name(Gen *g) {
370 g->lex_fn_name_begin = g->lex_pos;
371 lex_push_stack(g);
372}
373
374static void end_fn_name(Gen *g) {
375 char *ptr = &buf_ptr(g->in_buf)[g->lex_fn_name_begin];
376 int len = g->lex_pos - g->lex_fn_name_begin;
377 if (mem_eql_str(ptr, len, "token")) {
378 g->lex_state = LexStateTokenStart;
379 } else {
380 lex_error(g, "invalid function name: '%s'", buf_ptr(buf_create_from_mem(ptr, len)));
381 }
382}
383
384static void begin_token_name(Gen *g) {
385 g->lex_token_name_begin = g->lex_pos;
386}
387
388static void end_token_name(Gen *g) {
389 char *ptr = &buf_ptr(g->in_buf)[g->lex_token_name_begin];
390 int len = g->lex_pos - g->lex_token_name_begin;
391 Buf token_name = {0};
392 buf_init_from_mem(&token_name, ptr, len);
393
394 Token *token = find_or_create_token(g, &token_name);
395 RuleNode *node = allocate<RuleNode>(1);
396 node->type = RuleNodeTypeToken;
397 node->token.token = token;
398
399 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
400 g->lex_cur_rule->tuple.children.append(node);
401
402 lex_pop_stack(g);
403}
404
405static void begin_tuple_body(Gen *g) {
406 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
407 g->lex_body_begin = g->lex_pos;
408}
409
410static void end_tuple_body(Gen *g) {
411 assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
412 int end_pos = g->lex_pos + 1;
413 char *ptr = &buf_ptr(g->in_buf)[g->lex_body_begin];
414 int len = end_pos - g->lex_body_begin;
415 buf_init_from_mem(&g->lex_cur_rule->tuple.body, ptr, len);
416}
417
418static void initialize_rules(Gen *g) {
419 g->lex_state = LexStateStart;
420 for (g->lex_pos = 0; g->lex_pos < buf_len(g->in_buf); g->lex_pos += 1) {
421 uint8_t c = buf_ptr(g->in_buf)[g->lex_pos];
422 switch (g->lex_state) {
423 case LexStateStart:
424 switch (c) {
425 case WHITESPACE:
426 // ignore
427 break;
428 case UPPER_ALPHA:
429 begin_rule(g);
430 g->lex_state = LexStateRuleName;
431 break;
432 default:
433 lex_error(g, "invalid char: '%c'", c);
434 }
435 break;
436 case LexStateRuleName:
437 switch (c) {
438 case WHITESPACE:
439 end_rule_name(g);
440 g->lex_state = LexStateWaitForColon;
441 break;
442 case ':':
443 end_rule_name(g);
444 g->lex_state = LexStateTupleRule;
445 break;
446 case SYMBOL_CHAR:
447 break;
448 default:
449 lex_error(g, "invalid char: '%c'", c);
450 }
451 break;
452 case LexStateWaitForColon:
453 switch (c) {
454 case WHITESPACE:
455 // ignore
456 break;
457 case ':':
458 g->lex_state = LexStateTupleRule;
459 break;
460 default:
461 lex_error(g, "invalid char: '%c'", c);
462 }
463 break;
464 case LexStateTupleRule:
465 switch (c) {
466 case WHITESPACE:
467 // ignore
468 break;
469 case LOWER_ALPHA:
470 begin_fn_name(g);
471 g->lex_state = LexStateFnName;
472 break;
473 case '{':
474 begin_tuple_body(g);
475 g->lex_state = LexStateBody;
476 break;
477 default:
478 lex_error(g, "invalid char: '%c'", c);
479 }
480 break;
481 case LexStateFnName:
482 switch (c) {
483 case LOWER_ALPHA:
484 // ignore
485 break;
486 case '(':
487 end_fn_name(g);
488 break;
489 default:
490 lex_error(g, "expected '('");
491 }
492 break;
493 case LexStateTokenStart:
494 switch (c) {
495 case WHITESPACE:
496 // ignore
497 break;
498 case ALPHA:
499 begin_token_name(g);
500 g->lex_state = LexStateToken;
501 break;
502 default:
503 lex_error(g, "invalid char '%c'", c);
504 }
505 break;
506 case LexStateToken:
507 switch (c) {
508 case ALPHA:
509 // ignore
510 break;
511 case ')':
512 end_token_name(g);
513 break;
514 default:
515 lex_error(g, "invalid char '%c'", c);
516 }
517 break;
518 case LexStateBody:
519 switch (c) {
520 case '}':
521 end_tuple_body(g);
522 lex_pop_stack(g);
523 break;
524 default:
525 // ignore
526 break;
527 }
528 break;
529 case LexStateEndOrOr:
530 switch (c) {
531 case WHITESPACE:
532 // ignore
533 break;
534 case ';':
535 end_rule(g);
536 g->lex_state = LexStateStart;
537 break;
538 default:
539 lex_error(g, "expected ';' or '|'");
540 }
541 }
542 if (c == '\n') {
543 g->lex_line += 1;
544 g->lex_column = 0;
545 } else {
546 g->lex_column += 1;
547 }
548 }
549 switch (g->lex_state) {
550 case LexStateStart:
551 // ok
552 break;
553 case LexStateEndOrOr:
554 case LexStateRuleName:
555 case LexStateWaitForColon:
556 case LexStateTupleRule:
557 case LexStateFnName:
558 case LexStateTokenStart:
559 case LexStateToken:
560 case LexStateBody:
561 lex_error(g, "unexpected EOF");
562 break;
563 }
564}
565
566
206int main(int argc, char **argv) {567int main(int argc, char **argv) {
207 const char *in_filename = argv[1];568 const char *in_filename = argv[1];
208 const char *out_filename = argv[2];569 const char *out_filename = argv[2];
...@@ -227,47 +588,16 @@ int main(int argc, char **argv) {...@@ -227,47 +588,16 @@ int main(int argc, char **argv) {
227 if (!in_f || !out_f)588 if (!in_f || !out_f)
228 zig_panic("unable to open file(s)");589 zig_panic("unable to open file(s)");
229590
230 Buf *in_buf = fetch_file(in_f);
231
232 ZigList<RuleNode *> rules = {0};
233 Gen g = {0};591 Gen g = {0};
234592
235 //zig_panic("TODO initialize rules");593 g.in_buf = fetch_file(in_f);
236 {594 initialize_rules(&g);
237 Token *star_token = find_or_create_token(&g, buf_create_from_str("Star"));
238 Token *lparen_token = find_or_create_token(&g, buf_create_from_str("LParen"));
239 Token *eof_token = find_or_create_token(&g, buf_create_from_str("Eof"));
240595
241 RuleNode *root = allocate<RuleNode>(1);596 g.root = g.rules.at(0);
242 root->type = RuleNodeTypeTuple;
243
244 RuleNode *star_node = allocate<RuleNode>(1);
245 star_node->type = RuleNodeTypeToken;
246 star_node->token.token = star_token;
247 root->tuple.children.append(star_node);
248
249 RuleNode *lparen_node = allocate<RuleNode>(1);
250 lparen_node->type = RuleNodeTypeToken;
251 lparen_node->token.token = lparen_token;
252 root->tuple.children.append(lparen_node);
253
254 RuleNode *eof_node = allocate<RuleNode>(1);
255 eof_node->type = RuleNodeTypeToken;
256 eof_node->token.token = eof_token;
257 root->tuple.children.append(eof_node);
258
259 rules.append(root);
260 }
261
262 g.root = rules.at(0);
263597
264 g.cur_state = create_state(&g, ParserStateTypeOk);598 g.cur_state = create_state(&g, ParserStateTypeOk);
265 gen(&g, g.root);599 gen(&g, g.root);
266600
267
268
269 (void)in_buf;
270
271 fprintf(out_f, "/* This file is auto-generated by parsergen.cpp */\n");601 fprintf(out_f, "/* This file is auto-generated by parsergen.cpp */\n");
272 fprintf(out_f, "#include \"src/parser.hpp\"\n");602 fprintf(out_f, "#include \"src/parser.hpp\"\n");
273 fprintf(out_f, "#include <stdio.h>\n");603 fprintf(out_f, "#include <stdio.h>\n");