authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-23 19:21:52-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-23 19:21:52-07:00
log6b911f1e94b77730b78493183aa85296bb0fdd76
tree4a19d2e98f2a7c59f43490cc89a0c5d044b86e58
parent5b663ddbb2152b77ba71b18c7be2b6e379ae632a

delete parser generator. let's try recursive descent

4 files changed, 22 insertions(+), 1148 deletions(-)

CMakeLists.txt-20
...@@ -21,7 +21,6 @@ include_directories(...@@ -21,7 +21,6 @@ include_directories(
21)21)
2222
23set(GRAMMAR_TXT "${CMAKE_BINARY_DIR}/simple.txt")23set(GRAMMAR_TXT "${CMAKE_BINARY_DIR}/simple.txt")
24set(PARSER_GENERATED_CPP "${CMAKE_BINARY_DIR}/parser_generated.cpp")
2524
26set(ZIG_SOURCES25set(ZIG_SOURCES
27 "${CMAKE_SOURCE_DIR}/src/buffer.cpp"26 "${CMAKE_SOURCE_DIR}/src/buffer.cpp"
...@@ -30,13 +29,6 @@ set(ZIG_SOURCES...@@ -30,13 +29,6 @@ set(ZIG_SOURCES
30 "${CMAKE_SOURCE_DIR}/src/parser.cpp"29 "${CMAKE_SOURCE_DIR}/src/parser.cpp"
31 "${CMAKE_SOURCE_DIR}/src/tokenizer.cpp"30 "${CMAKE_SOURCE_DIR}/src/tokenizer.cpp"
32 "${CMAKE_SOURCE_DIR}/src/util.cpp"31 "${CMAKE_SOURCE_DIR}/src/util.cpp"
33 ${PARSER_GENERATED_CPP}
34)
35
36set(PARSERGEN_SOURCES
37 "${CMAKE_SOURCE_DIR}/src/parsergen.cpp"
38 "${CMAKE_SOURCE_DIR}/src/util.cpp"
39 "${CMAKE_SOURCE_DIR}/src/buffer.cpp"
40)32)
4133
42set(CONFIGURE_OUT_FILE "${CMAKE_BINARY_DIR}/config.h")34set(CONFIGURE_OUT_FILE "${CMAKE_BINARY_DIR}/config.h")
...@@ -63,15 +55,3 @@ target_link_libraries(zig LINK_PUBLIC...@@ -63,15 +55,3 @@ target_link_libraries(zig LINK_PUBLIC
63)55)
64install(TARGETS zig DESTINATION bin)56install(TARGETS zig DESTINATION bin)
6557
66add_executable(parsergen ${PARSERGEN_SOURCES})
67set_target_properties(parsergen PROPERTIES
68 LINKER_LANGUAGE C
69 COMPILE_FLAGS ${EXE_CFLAGS})
70
71
72add_custom_command(
73 OUTPUT ${PARSER_GENERATED_CPP}
74 COMMAND parsergen ARGS ${GRAMMAR_TXT} ${PARSER_GENERATED_CPP}
75 DEPENDS ${GRAMMAR_TXT} ${PARSERGEN_SOURCES}
76 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
77)
src/parser.cpp+21-2
...@@ -79,8 +79,13 @@ void ast_print(AstNode *node, int indent) {...@@ -79,8 +79,13 @@ void ast_print(AstNode *node, int indent) {
79 }79 }
80}80}
8181
82AstNode *ast_create_root(Token *token) {82struct ParseContext {
83 return nullptr;83 Buf *buf;
84 AstNode *root;
85};
86
87AstNode *ast_create_root(void) {
88 zig_panic("TODO create root");
84}89}
8590
86void ast_invalid_token_error(Buf *buf, Token *token) {91void ast_invalid_token_error(Buf *buf, Token *token) {
...@@ -88,3 +93,17 @@ void ast_invalid_token_error(Buf *buf, Token *token) {...@@ -88,3 +93,17 @@ void ast_invalid_token_error(Buf *buf, Token *token) {
88 buf_init_from_mem(&token_value, buf_ptr(buf) + token->start_pos, token->end_pos - token->start_pos);93 buf_init_from_mem(&token_value, buf_ptr(buf) + token->start_pos, token->end_pos - token->start_pos);
89 ast_error(token, "invalid token: '%s'", buf_ptr(&token_value));94 ast_error(token, "invalid token: '%s'", buf_ptr(&token_value));
90}95}
96
97void ast_parse_fn_decls(ParseContext *pc, ZigList<AstNode *> *fn_decls) {
98 zig_panic("TODO parse fn decls");
99}
100
101AstNode *ast_parse(Buf *buf, ZigList<Token> *tokens) {
102 ParseContext pc = {0};
103 pc.buf = buf;
104 pc.root = ast_create_root();
105
106 ast_parse_fn_decls(&pc, &pc.root->data.root.fn_decls);
107
108 return pc.root;
109}
src/parser.hpp+1-1
...@@ -91,6 +91,6 @@ const char *node_type_str(NodeType node_type);...@@ -91,6 +91,6 @@ const char *node_type_str(NodeType node_type);
9191
92void ast_print(AstNode *node, int indent);92void ast_print(AstNode *node, int indent);
9393
94AstNode *ast_create_root(Token *token);94AstNode *ast_create_root(void);
9595
96#endif96#endif
src/parsergen.cpp deleted-1125
...@@ -1,1125 +0,0 @@
1/*
2 * Copyright (c) 2015 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
8#include "util.hpp"
9#include "buffer.hpp"
10#include "list.hpp"
11
12#include <stdio.h>
13#include <stdarg.h>
14#include <sys/types.h>
15#include <sys/stat.h>
16#include <unistd.h>
17#include <errno.h>
18#include <limits.h>
19
20#define WHITESPACE \
21 ' ': \
22 case '\t': \
23 case '\n': \
24 case '\f': \
25 case '\r': \
26 case 0xb
27
28#define DIGIT \
29 '0': \
30 case '1': \
31 case '2': \
32 case '3': \
33 case '4': \
34 case '5': \
35 case '6': \
36 case '7': \
37 case '8': \
38 case '9'
39
40#define LOWER_ALPHA \
41 'a': \
42 case 'b': \
43 case 'c': \
44 case 'd': \
45 case 'e': \
46 case 'f': \
47 case 'g': \
48 case 'h': \
49 case 'i': \
50 case 'j': \
51 case 'k': \
52 case 'l': \
53 case 'm': \
54 case 'n': \
55 case 'o': \
56 case 'p': \
57 case 'q': \
58 case 'r': \
59 case 's': \
60 case 't': \
61 case 'u': \
62 case 'v': \
63 case 'w': \
64 case 'x': \
65 case 'y': \
66 case 'z'
67
68#define UPPER_ALPHA \
69 'A': \
70 case 'B': \
71 case 'C': \
72 case 'D': \
73 case 'E': \
74 case 'F': \
75 case 'G': \
76 case 'H': \
77 case 'I': \
78 case 'J': \
79 case 'K': \
80 case 'L': \
81 case 'M': \
82 case 'N': \
83 case 'O': \
84 case 'P': \
85 case 'Q': \
86 case 'R': \
87 case 'S': \
88 case 'T': \
89 case 'U': \
90 case 'V': \
91 case 'W': \
92 case 'X': \
93 case 'Y': \
94 case 'Z'
95
96#define ALPHA \
97 LOWER_ALPHA: \
98 case UPPER_ALPHA
99
100#define SYMBOL_CHAR \
101 ALPHA: \
102 case DIGIT: \
103 case '_'
104
105static Buf *fetch_file(FILE *f) {
106 int fd = fileno(f);
107 struct stat st;
108 if (fstat(fd, &st))
109 zig_panic("unable to stat file: %s", strerror(errno));
110 off_t big_size = st.st_size;
111 if (big_size > INT_MAX)
112 zig_panic("file too big");
113 int size = (int)big_size;
114
115 Buf *buf = buf_alloc_fixed(size);
116 size_t amt_read = fread(buf_ptr(buf), 1, buf_len(buf), f);
117 if (amt_read != (size_t)buf_len(buf))
118 zig_panic("error reading: %s", strerror(errno));
119
120 return buf;
121}
122
123static int usage(const char *arg0) {
124 fprintf(stderr, "Usage: %s in-grammar.txt out-parser.c\n", arg0);
125 return 1;
126}
127
128struct Token {
129 Buf name;
130 int id;
131};
132
133struct RuleNode;
134
135struct RuleTuple {
136 ZigList<RuleNode *> children;
137 Buf body;
138};
139
140struct RuleMany {
141 RuleNode *child;
142};
143
144struct RuleOr {
145 Buf name;
146 Buf union_field_name;
147 ZigList<RuleNode *> children;
148};
149
150struct RuleToken {
151 Token *token;
152};
153
154struct RuleList {
155 RuleNode *rule;
156 RuleToken *separator;
157};
158
159struct RuleSubRule {
160 RuleNode *child;
161
162 // for lexer use only
163 Buf name;
164};
165
166enum RuleNodeType {
167 RuleNodeTypeTuple,
168 RuleNodeTypeMany,
169 RuleNodeTypeList,
170 RuleNodeTypeOr,
171 RuleNodeTypeToken,
172 RuleNodeTypeSubRule,
173};
174
175struct RuleNode {
176 RuleNodeType type;
177 int lex_line;
178 int lex_column;
179 union {
180 RuleTuple tuple;
181 RuleMany many;
182 RuleList list;
183 RuleOr _or;
184 RuleToken token;
185 RuleSubRule sub_rule;
186 };
187};
188
189
190enum CodeGenType {
191 CodeGenTypeTransition,
192 CodeGenTypeError,
193 CodeGenTypeSave,
194 CodeGenTypePushNode,
195 CodeGenTypeCapture,
196 CodeGenTypePopNode,
197 CodeGenTypeEatToken,
198};
199
200struct CodeGenError {
201 Buf *msg;
202};
203
204struct CodeGenCapture {
205 Buf *body;
206 bool is_root;
207 Buf *field_names;
208 Buf *union_field_name;
209};
210
211struct CodeGen {
212 CodeGenType type;
213 union {
214 CodeGenError error;
215 CodeGenCapture capture;
216 };
217};
218
219struct ParserState {
220 ZigList<CodeGen *> code_gen_list;
221 // One for each token ID.
222 ParserState **transition;
223 int index;
224 bool is_error;
225};
226
227enum LexState {
228 LexStateStart,
229 LexStateRuleName,
230 LexStateRuleFieldNameStart,
231 LexStateRuleFieldName,
232 LexStateWaitForColon,
233 LexStateTupleRule,
234 LexStateFnName,
235 LexStateTokenStart,
236 LexStateToken,
237 LexStateBody,
238 LexStateEndOrOr,
239 LexStateSubTupleName,
240};
241
242struct LexStack {
243 LexState state;
244};
245
246struct Gen {
247 ZigList<RuleNode *> rules;
248
249 ZigList<ParserState *> transition_table;
250 ParserState *start_state;
251 ZigList<Token *> tokens;
252 RuleNode *root;
253 int biggest_tuple_len;
254
255 Buf *in_buf;
256 LexState lex_state;
257 int lex_line;
258 int lex_column;
259 RuleNode *lex_cur_or_rule;
260 RuleNode *lex_cur_tuple_rule;;
261 int lex_cur_rule_begin;
262 int lex_fn_name_begin;
263 int lex_pos;
264 ZigList<LexStack> lex_stack;
265 int lex_token_name_begin;
266 int lex_body_begin;
267 int lex_body_end;
268 int lex_sub_tuple_begin;
269 int lex_field_name_begin;
270};
271
272static ParserState *create_state(Gen *g) {
273 ParserState *state = allocate<ParserState>(1);
274 state->index = -1;
275 state->transition = allocate<ParserState*>(g->tokens.length);
276 return state;
277}
278
279static void fill_state_with_transition(Gen *g, ParserState *source, ParserState *dest) {
280 for (int i = 0; i < g->tokens.length; i += 1) {
281 source->transition[i] = dest;
282 }
283}
284
285static void state_add_code(ParserState *state, CodeGen *code) {
286 state->code_gen_list.append(code);
287}
288
289static void state_add_save_token(ParserState *state) {
290 CodeGen *code = allocate<CodeGen>(1);
291 code->type = CodeGenTypeSave;
292 state_add_code(state, code);
293}
294
295static void state_add_error(ParserState *state, Buf *msg) {
296 CodeGen *code = allocate<CodeGen>(1);
297 code->type = CodeGenTypeError;
298 code->error.msg = msg;
299 state_add_code(state, code);
300 state->is_error = true;
301}
302
303static void state_add_transition(ParserState *state) {
304 CodeGen *code = allocate<CodeGen>(1);
305 code->type = CodeGenTypeTransition;
306 state_add_code(state, code);
307}
308
309static void state_add_push_node(ParserState *state) {
310 CodeGen *code = allocate<CodeGen>(1);
311 code->type = CodeGenTypePushNode;
312 state_add_code(state, code);
313}
314
315static CodeGen *codegen_create_capture(Buf *body, bool is_root, int field_name_count, Buf *union_field_name) {
316 CodeGen *code = allocate<CodeGen>(1);
317 code->type = CodeGenTypeCapture;
318 code->capture.body = body;
319 code->capture.is_root = is_root;
320 code->capture.field_names = allocate<Buf>(field_name_count);
321 code->capture.union_field_name = union_field_name;
322 return code;
323}
324
325static void state_add_pop_node(ParserState *state) {
326 CodeGen *code = allocate<CodeGen>(1);
327 code->type = CodeGenTypePopNode;
328 state_add_code(state, code);
329}
330
331static void state_add_eat_token(ParserState *state) {
332 CodeGen *code = allocate<CodeGen>(1);
333 code->type = CodeGenTypeEatToken;
334 state_add_code(state, code);
335}
336
337
338static void gen(Gen *g, RuleNode *node, Buf *out_field_name, ParserState *cur_state,
339 ZigList<ParserState *> *end_states, bool is_root)
340{
341 struct PossibleState {
342 ParserState *test_state;
343 ZigList<ParserState *> end_states;
344 };
345 assert(node);
346 switch (node->type) {
347 case RuleNodeTypeToken:
348 {
349 buf_init_from_str(out_field_name, "token");
350
351 state_add_save_token(cur_state);
352
353 ParserState *ok_state = create_state(g);
354 ParserState *err_state = create_state(g);
355 state_add_error(err_state, buf_sprintf("expected token '%s'", buf_ptr(&node->token.token->name)));
356
357 fill_state_with_transition(g, cur_state, err_state);
358 cur_state->transition[node->token.token->id] = ok_state;
359 state_add_transition(cur_state);
360 state_add_eat_token(cur_state);
361
362 end_states->append(ok_state);
363 }
364 break;
365 case RuleNodeTypeTuple:
366 {
367
368 int field_name_count = node->tuple.children.length;
369 CodeGen *code = codegen_create_capture(&node->tuple.body, is_root, field_name_count,
370 out_field_name);
371
372 ZigList<ParserState *> *my_end_states = allocate<ZigList<ParserState *>>(1);
373 my_end_states->append(cur_state);
374 for (int child_index = 0; child_index < node->tuple.children.length; child_index += 1) {
375 RuleNode *child = node->tuple.children.at(child_index);
376
377 ZigList<ParserState *> *more_end_states = allocate<ZigList<ParserState *>>(1);
378 for (int i = 0; i < my_end_states->length; i += 1) {
379 ParserState *use_state = my_end_states->at(i);
380 gen(g, child, &code->capture.field_names[i], use_state, more_end_states, false);
381 }
382
383 my_end_states = more_end_states;
384 }
385
386 for (int i = 0; i < my_end_states->length; i += 1) {
387 ParserState *use_state = my_end_states->at(i);
388 state_add_code(use_state, code);
389
390 end_states->append(use_state);
391 }
392 }
393 break;
394 case RuleNodeTypeMany:
395 zig_panic("TODO");
396 break;
397 case RuleNodeTypeList:
398 zig_panic("TODO");
399 break;
400 case RuleNodeTypeOr:
401 {
402 buf_init_from_buf(out_field_name, &node->_or.union_field_name);
403
404 state_add_push_node(cur_state);
405
406 // TODO this probably need to get moved when or can handle conflicts
407 state_add_save_token(cur_state);
408 state_add_transition(cur_state);
409 state_add_eat_token(cur_state);
410
411
412 int possible_state_count = node->_or.children.length;
413 PossibleState *possible_states = allocate<PossibleState>(possible_state_count);
414 for (int i = 0; i < possible_state_count; i += 1) {
415 RuleNode *child = node->_or.children.at(i);
416 assert(child->type == RuleNodeTypeTuple);
417
418 PossibleState *possible_state = &possible_states[i];
419 possible_state->test_state = create_state(g);
420 gen(g, child, &node->_or.union_field_name, possible_state->test_state,
421 &possible_state->end_states, is_root);
422 }
423
424 // try to merge all the possible states into new state.
425 ParserState *err_state = create_state(g);
426 state_add_error(err_state, buf_create_from_str("unexpected token"));
427 for (int token_i = 0; token_i < g->tokens.length; token_i += 1) {
428 bool any_called_it = false;
429 bool conflict = false;
430 for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
431 PossibleState *possible_state = &possible_states[state_i];
432 if (!possible_state->test_state->transition[token_i]->is_error) {
433 if (any_called_it) {
434 conflict = true;
435 } else {
436 any_called_it = true;
437 }
438 }
439 }
440 if (conflict) {
441 zig_panic("TODO state transition conflict");
442 } else {
443 cur_state->transition[token_i] = err_state;
444 for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
445 PossibleState *possible_state = &possible_states[state_i];
446 if (!possible_state->test_state->transition[token_i]->is_error) {
447 cur_state->transition[token_i] = possible_state->test_state->transition[token_i];
448 }
449 }
450 }
451 }
452
453 for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
454 PossibleState *possible_state = &possible_states[state_i];
455 for (int end_i = 0; end_i < possible_state->end_states.length; end_i += 1) {
456 ParserState *state = possible_state->end_states.at(end_i);
457 state_add_pop_node(state);
458
459 end_states->append(state);
460 }
461 }
462 }
463 break;
464 case RuleNodeTypeSubRule:
465 {
466 RuleNode *child = node->sub_rule.child;
467 gen(g, child, out_field_name, cur_state, end_states, false);
468 }
469 break;
470 }
471}
472
473static Token *find_token_by_name(Gen *g, Buf *name) {
474 for (int i = 0; i < g->tokens.length; i += 1) {
475 Token *token = g->tokens.at(i);
476 if (buf_eql_buf(name, &token->name))
477 return token;
478 }
479 return nullptr;
480}
481
482static Token *find_or_create_token(Gen *g, Buf *name) {
483 Token *token = find_token_by_name(g, name);
484 if (!token) {
485 token = allocate<Token>(1);
486 token->id = g->tokens.length;
487 buf_init_from_mem(&token->name, buf_ptr(name), buf_len(name));
488 g->tokens.append(token);
489 }
490 return token;
491}
492
493__attribute__ ((format (printf, 2, 3)))
494static void lex_error(Gen *g, const char *format, ...) {
495 int line = g->lex_line + 1;
496 int column = g->lex_column + 1;
497
498 va_list ap;
499 va_start(ap, format);
500 fprintf(stderr, "Grammar Error: Line %d, column %d: ", line, column);
501 vfprintf(stderr, format, ap);
502 fprintf(stderr, "\n");
503 va_end(ap);
504 exit(EXIT_FAILURE);
505}
506
507static void lex_push_stack(Gen *g) {
508 g->lex_stack.append({g->lex_state});
509}
510
511static void lex_pop_stack(Gen *g) {
512 LexStack *entry = &g->lex_stack.last();
513 g->lex_state = entry->state;
514 g->lex_stack.pop();
515}
516
517static RuleNode *create_rule_node(Gen *g) {
518 RuleNode *node = allocate<RuleNode>(1);
519 node->lex_line = g->lex_line;
520 node->lex_column = g->lex_column;
521 return node;
522}
523
524static void begin_rule(Gen *g) {
525 assert(!g->lex_cur_or_rule);
526 assert(!g->lex_cur_tuple_rule);
527
528 g->lex_cur_or_rule = create_rule_node(g);
529 g->lex_cur_or_rule->type = RuleNodeTypeOr;
530
531 g->lex_cur_tuple_rule = create_rule_node(g);
532 g->lex_cur_tuple_rule->type = RuleNodeTypeTuple;
533 g->lex_cur_rule_begin = g->lex_pos;
534}
535
536static void end_rule(Gen *g) {
537 assert(g->lex_cur_or_rule);
538 assert(!g->lex_cur_tuple_rule);
539
540 g->rules.append(g->lex_cur_or_rule);
541 g->lex_cur_or_rule = nullptr;
542}
543
544static void perform_or(Gen *g) {
545 assert(g->lex_cur_or_rule);
546 assert(!g->lex_cur_tuple_rule);
547
548 g->lex_cur_tuple_rule = create_rule_node(g);
549 g->lex_cur_tuple_rule->type = RuleNodeTypeTuple;
550 g->lex_cur_rule_begin = g->lex_pos;
551}
552
553static void end_rule_name(Gen *g) {
554 assert(g->lex_cur_or_rule);
555 char *ptr = &buf_ptr(g->in_buf)[g->lex_cur_rule_begin];
556 int len = g->lex_pos - g->lex_cur_rule_begin;
557 buf_init_from_mem(&g->lex_cur_or_rule->_or.name, ptr, len);
558}
559
560static void begin_rule_field_name(Gen *g) {
561 assert(g->lex_cur_or_rule);
562 g->lex_field_name_begin = g->lex_pos;
563}
564
565static void end_rule_field_name(Gen *g) {
566 assert(g->lex_cur_or_rule);
567 char *ptr = &buf_ptr(g->in_buf)[g->lex_field_name_begin];
568 int len = g->lex_pos - g->lex_field_name_begin;
569 buf_init_from_mem(&g->lex_cur_or_rule->_or.union_field_name, ptr, len);
570}
571
572static void begin_fn_name(Gen *g) {
573 g->lex_fn_name_begin = g->lex_pos;
574 lex_push_stack(g);
575}
576
577static void end_fn_name(Gen *g) {
578 char *ptr = &buf_ptr(g->in_buf)[g->lex_fn_name_begin];
579 int len = g->lex_pos - g->lex_fn_name_begin;
580 if (mem_eql_str(ptr, len, "token")) {
581 g->lex_state = LexStateTokenStart;
582 } else {
583 lex_error(g, "invalid function name: '%s'", buf_ptr(buf_create_from_mem(ptr, len)));
584 }
585}
586
587static void begin_token_name(Gen *g) {
588 g->lex_token_name_begin = g->lex_pos;
589}
590
591static void end_token_name(Gen *g) {
592 assert(g->lex_cur_tuple_rule);
593 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
594
595 char *ptr = &buf_ptr(g->in_buf)[g->lex_token_name_begin];
596 int len = g->lex_pos - g->lex_token_name_begin;
597 Buf token_name = {0};
598 buf_init_from_mem(&token_name, ptr, len);
599
600 Token *token = find_or_create_token(g, &token_name);
601 RuleNode *node = create_rule_node(g);
602 node->type = RuleNodeTypeToken;
603 node->token.token = token;
604
605 g->lex_cur_tuple_rule->tuple.children.append(node);
606
607
608 lex_pop_stack(g);
609}
610
611static void begin_tuple_body(Gen *g) {
612 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
613 g->lex_body_begin = g->lex_pos;
614}
615
616static void end_tuple_body(Gen *g) {
617 assert(g->lex_cur_or_rule);
618 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
619 int end_pos = g->lex_pos + 1;
620 char *ptr = &buf_ptr(g->in_buf)[g->lex_body_begin];
621 int len = end_pos - g->lex_body_begin;
622 buf_init_from_mem(&g->lex_cur_tuple_rule->tuple.body, ptr, len);
623
624 g->lex_cur_or_rule->_or.children.append(g->lex_cur_tuple_rule);
625 g->lex_cur_tuple_rule = nullptr;
626}
627
628static void begin_sub_tuple(Gen *g) {
629 g->lex_sub_tuple_begin = g->lex_pos;
630 lex_push_stack(g);
631}
632
633static void end_sub_tuple(Gen *g) {
634 assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
635 char *ptr = &buf_ptr(g->in_buf)[g->lex_sub_tuple_begin];
636 int len = g->lex_pos - g->lex_sub_tuple_begin;
637
638 RuleNode *node = create_rule_node(g);
639 node->type = RuleNodeTypeSubRule;
640 buf_init_from_mem(&node->sub_rule.name, ptr, len);
641
642 g->lex_cur_tuple_rule->tuple.children.append(node);
643
644 lex_pop_stack(g);
645}
646
647static RuleNode *find_rule_node(Gen *g, Buf *name) {
648 for (int i = 0; i < g->rules.length; i += 1) {
649 RuleNode *node = g->rules.at(i);
650 assert(node->type == RuleNodeTypeOr);
651 if (buf_eql_buf(&node->_or.name, name)) {
652 return node;
653 }
654 }
655 return nullptr;
656}
657
658static void initialize_rules(Gen *g) {
659 g->lex_state = LexStateStart;
660 for (g->lex_pos = 0; g->lex_pos < buf_len(g->in_buf); g->lex_pos += 1) {
661 uint8_t c = buf_ptr(g->in_buf)[g->lex_pos];
662 switch (g->lex_state) {
663 case LexStateStart:
664 switch (c) {
665 case WHITESPACE:
666 // ignore
667 break;
668 case UPPER_ALPHA:
669 begin_rule(g);
670 g->lex_state = LexStateRuleName;
671 break;
672 default:
673 lex_error(g, "invalid char: '%c'", c);
674 }
675 break;
676 case LexStateRuleName:
677 switch (c) {
678 case '<':
679 end_rule_name(g);
680 g->lex_state = LexStateRuleFieldNameStart;
681 break;
682 case SYMBOL_CHAR:
683 // ok
684 break;
685 default:
686 lex_error(g, "expected '<', not '%c'", c);
687 }
688 break;
689 case LexStateRuleFieldNameStart:
690 switch (c) {
691 case SYMBOL_CHAR:
692 begin_rule_field_name(g);
693 g->lex_state = LexStateRuleFieldName;
694 break;
695 default:
696 lex_error(g, "expected field name, not '%c'", c);
697 }
698 break;
699 case LexStateRuleFieldName:
700 switch (c) {
701 case SYMBOL_CHAR:
702 // ok
703 break;
704 case '>':
705 end_rule_field_name(g);
706 g->lex_state = LexStateWaitForColon;
707 break;
708 }
709 break;
710 case LexStateWaitForColon:
711 switch (c) {
712 case WHITESPACE:
713 // ignore
714 break;
715 case ':':
716 g->lex_state = LexStateTupleRule;
717 break;
718 default:
719 lex_error(g, "invalid char: '%c'", c);
720 }
721 break;
722 case LexStateTupleRule:
723 switch (c) {
724 case WHITESPACE:
725 // ignore
726 break;
727 case LOWER_ALPHA:
728 begin_fn_name(g);
729 g->lex_state = LexStateFnName;
730 break;
731 case UPPER_ALPHA:
732 begin_sub_tuple(g);
733 g->lex_state = LexStateSubTupleName;
734 break;
735 case '{':
736 begin_tuple_body(g);
737 g->lex_state = LexStateBody;
738 break;
739 default:
740 lex_error(g, "expected rule, not '%c'", c);
741 }
742 break;
743 case LexStateFnName:
744 switch (c) {
745 case LOWER_ALPHA:
746 // ignore
747 break;
748 case '(':
749 end_fn_name(g);
750 break;
751 default:
752 lex_error(g, "expected '('");
753 }
754 break;
755 case LexStateTokenStart:
756 switch (c) {
757 case WHITESPACE:
758 // ignore
759 break;
760 case ALPHA:
761 begin_token_name(g);
762 g->lex_state = LexStateToken;
763 break;
764 default:
765 lex_error(g, "expected token name, not '%c'", c);
766 }
767 break;
768 case LexStateToken:
769 switch (c) {
770 case ALPHA:
771 // ignore
772 break;
773 case ')':
774 end_token_name(g);
775 break;
776 default:
777 lex_error(g, "expected token name or ')', not '%c'", c);
778 }
779 break;
780 case LexStateBody:
781 switch (c) {
782 case '}':
783 end_tuple_body(g);
784 g->lex_state = LexStateEndOrOr;
785 break;
786 default:
787 // ignore
788 break;
789 }
790 break;
791 case LexStateEndOrOr:
792 switch (c) {
793 case WHITESPACE:
794 // ignore
795 break;
796 case ';':
797 end_rule(g);
798 g->lex_state = LexStateStart;
799 break;
800 case '|':
801 perform_or(g);
802 g->lex_state = LexStateTupleRule;
803 break;
804 default:
805 lex_error(g, "expected ';' or '|'");
806 }
807 break;
808 case LexStateSubTupleName:
809 switch (c) {
810 case ALPHA:
811 // ignore
812 break;
813 case WHITESPACE:
814 end_sub_tuple(g);
815 assert(g->lex_state == LexStateTupleRule);
816 break;
817 default:
818 lex_error(g, "expected rule name, not '%c'", c);
819 }
820 break;
821 }
822 if (c == '\n') {
823 g->lex_line += 1;
824 g->lex_column = 0;
825 } else {
826 g->lex_column += 1;
827 }
828 }
829 switch (g->lex_state) {
830 case LexStateStart:
831 // ok
832 break;
833 case LexStateEndOrOr:
834 case LexStateRuleName:
835 case LexStateWaitForColon:
836 case LexStateTupleRule:
837 case LexStateFnName:
838 case LexStateTokenStart:
839 case LexStateToken:
840 case LexStateBody:
841 case LexStateSubTupleName:
842 case LexStateRuleFieldNameStart:
843 case LexStateRuleFieldName:
844 lex_error(g, "unexpected EOF");
845 break;
846 }
847
848 // Iterate over the rules and
849 // * resolve child references into pointers
850 // * calculate the biggest tuple len
851 bool any_errors = false;
852 for (int or_i = 0; or_i < g->rules.length; or_i += 1) {
853 RuleNode *or_node = g->rules.at(or_i);
854 assert(or_node->type == RuleNodeTypeOr);
855
856 for (int tuple_i = 0; tuple_i < or_node->_or.children.length; tuple_i += 1) {
857 RuleNode *tuple_node = or_node->_or.children.at(tuple_i);
858 assert(tuple_node->type == RuleNodeTypeTuple);
859 g->biggest_tuple_len = max(g->biggest_tuple_len, tuple_node->tuple.children.length);
860
861 for (int child_i = 0; child_i < tuple_node->tuple.children.length; child_i += 1) {
862 RuleNode *child = tuple_node->tuple.children.at(child_i);
863
864 if (child->type == RuleNodeTypeSubRule) {
865 int line = child->lex_line + 1;
866 int column = child->lex_column + 1;
867 RuleNode *referenced_node = find_rule_node(g, &child->sub_rule.name);
868 if (!referenced_node) {
869 fprintf(stderr, "Grammar Error: Line %d, column %d: Rule not defined: '%s'\n",
870 line, column, buf_ptr(&child->sub_rule.name));
871 any_errors = true;
872 }
873 child->sub_rule.child = referenced_node;
874 }
875 }
876 }
877 }
878
879 if (any_errors) {
880 exit(EXIT_FAILURE);
881 }
882}
883
884enum TemplateState {
885 TemplateStateStart,
886 TemplateStateDollar,
887 TemplateStateNumber,
888};
889
890static Buf *fill_template(Buf *body, const char *result_name, Buf *field_names) {
891 //fprintf(stderr, "fill template input:\n%s\n", buf_ptr(body));
892 Buf *result = buf_alloc();
893 TemplateState state = TemplateStateStart;
894 int digit_start;
895 for (int i = 0; i < buf_len(body); i += 1) {
896 uint8_t c = buf_ptr(body)[i];
897 switch (state) {
898 case TemplateStateStart:
899 switch (c) {
900 case '$':
901 state = TemplateStateDollar;
902 break;
903 default:
904 buf_append_char(result, c);
905 break;
906 }
907 break;
908 case TemplateStateDollar:
909 switch (c) {
910 case '$':
911 buf_append_str(result, result_name);
912 state = TemplateStateStart;
913 break;
914 case DIGIT:
915 digit_start = i;
916 state = TemplateStateNumber;
917 break;
918 default:
919 buf_append_char(result, '$');
920 buf_append_char(result, c);
921 state = TemplateStateStart;
922 break;
923 }
924 break;
925 case TemplateStateNumber:
926 switch (c) {
927 case DIGIT:
928 // nothing
929 break;
930 default:
931 {
932 Buf *num_buf = buf_create_from_mem(&buf_ptr(body)[digit_start], i - digit_start);
933 int index = atoi(buf_ptr(num_buf)) - 1;
934 buf_appendf(result, "(top_node->data[%d].%s)%c",
935 index, buf_ptr(&field_names[index]), c);
936
937 state = TemplateStateStart;
938 }
939 break;
940 }
941 break;
942 }
943 }
944 switch (state) {
945 case TemplateStateStart:
946 // OK
947 break;
948 default:
949 zig_panic("unable to fill grammar template");
950 }
951 //fprintf(stderr, "fill template output:\n%s\n", buf_ptr(result));
952 return result;
953}
954
955static void build_transition_table(Gen *g, ParserState *state) {
956 if (!state)
957 return;
958 if (state->index >= 0)
959 return;
960 state->index = g->transition_table.length;
961 g->transition_table.append(state);
962 for (int i = 0; i < g->tokens.length; i += 1) {
963 ParserState *other_state = state->transition[i];
964 build_transition_table(g, other_state);
965 }
966}
967
968int main(int argc, char **argv) {
969 const char *in_filename = argv[1];
970 const char *out_filename = argv[2];
971
972 if (!in_filename || !out_filename)
973 return usage(argv[0]);
974
975 FILE *in_f;
976 if (strcmp(in_filename, "-") == 0) {
977 in_f = stdin;
978 } else {
979 in_f = fopen(in_filename, "rb");
980 }
981
982 FILE *out_f;
983 if (strcmp(out_filename, "-") == 0) {
984 out_f = stdout;
985 } else {
986 out_f = fopen(out_filename, "wb");
987 }
988
989 if (!in_f || !out_f)
990 zig_panic("unable to open file(s)");
991
992 Gen g = {0};
993
994 g.in_buf = fetch_file(in_f);
995 initialize_rules(&g);
996
997 g.root = g.rules.at(0);
998
999 g.start_state = create_state(&g);
1000 Buf root_field_name = {0};
1001 ZigList<ParserState *> end_states = {0};
1002 gen(&g, g.root, &root_field_name, g.start_state, &end_states, true);
1003 build_transition_table(&g, g.start_state);
1004
1005 fprintf(out_f, "/* This file is generated by parsergen.cpp */\n");
1006 fprintf(out_f, "\n");
1007 fprintf(out_f, "#include \"src/parser.hpp\"\n");
1008 fprintf(out_f, "#include <stdio.h>\n");
1009
1010 fprintf(out_f, "\n");
1011 fprintf(out_f, "/*\n");
1012 fprintf(out_f, "enum TokenId {\n");
1013 for (int i = 0; i < g.tokens.length; i += 1) {
1014 Token *token = g.tokens.at(i);
1015 fprintf(out_f, " TokenId%s = %d,\n", buf_ptr(&token->name), token->id);
1016 }
1017 fprintf(out_f, "};\n");
1018 fprintf(out_f, "*/\n");
1019 for (int i = 0; i < g.tokens.length; i += 1) {
1020 Token *token = g.tokens.at(i);
1021 fprintf(out_f, "static_assert(TokenId%s == %d, \"wrong token id\");\n",
1022 buf_ptr(&token->name), token->id);
1023 }
1024 fprintf(out_f, "\n");
1025
1026 fprintf(out_f, "struct ParserGenNode {\n");
1027 fprintf(out_f, " int next_index;\n");
1028 fprintf(out_f, " union {\n");
1029 fprintf(out_f, " Token *token;\n");
1030 fprintf(out_f, " AstNode *node;\n");
1031 fprintf(out_f, " } data[%d];\n", g.biggest_tuple_len);
1032 fprintf(out_f, "};\n");
1033 fprintf(out_f, "\n");
1034
1035 fprintf(out_f, "AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens) {\n");
1036
1037 fprintf(out_f, " static const int transition[%d][%d] = {\n", g.transition_table.length, g.tokens.length);
1038 for (int state_index = 0; state_index < g.transition_table.length; state_index += 1) {
1039 ParserState *state = g.transition_table.at(state_index);
1040 fprintf(out_f, " {\n");
1041
1042 for (int token_id = 0; token_id < g.tokens.length; token_id += 1) {
1043 ParserState *dest = state->transition[token_id];
1044 fprintf(out_f, " %d,\n", dest ? dest->index : -1);
1045 }
1046
1047 fprintf(out_f, " },\n");
1048 }
1049 fprintf(out_f, " };\n");
1050
1051
1052 fprintf(out_f, " int state = 0;\n");
1053 fprintf(out_f, " int token_index = 0;\n");
1054 fprintf(out_f, " Token *token = &tokens->at(token_index);\n");
1055 fprintf(out_f, " AstNode *root = nullptr;\n");
1056 fprintf(out_f, " ZigList<ParserGenNode *> stack = {0};\n");
1057 fprintf(out_f, " ParserGenNode *top_node = nullptr;\n");
1058
1059 fprintf(out_f, " for (;;) {\n");
1060 fprintf(out_f, " switch (state) {\n");
1061
1062 for (int state_i = 0; state_i < g.transition_table.length; state_i += 1) {
1063 ParserState *state = g.transition_table.at(state_i);
1064 fprintf(out_f, " case %d: {\n", state_i);
1065 for (int code_i = 0; code_i < state->code_gen_list.length; code_i += 1) {
1066 CodeGen *code = state->code_gen_list.at(code_i);
1067 switch (code->type) {
1068 case CodeGenTypeTransition:
1069 fprintf(out_f, " if (token->id < 0 || token->id >= %d) {\n", g.tokens.length);
1070 fprintf(out_f, " ast_invalid_token_error(buf, token);\n");
1071 fprintf(out_f, " }\n");
1072 fprintf(out_f, " assert(transition[%d][token->id] >= 0);\n", state->index);
1073 fprintf(out_f, " assert(transition[%d][token->id] < %d);\n",
1074 state->index, g.transition_table.length);
1075 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);
1076 break;
1077 case CodeGenTypeError:
1078 fprintf(out_f, " token_index -= 1;\n");
1079 fprintf(out_f, " token = &tokens->at(token_index);\n");
1080 fprintf(out_f, " ast_error(token, \"%s\");\n", buf_ptr(code->error.msg));
1081 break;
1082 case CodeGenTypeSave:
1083 fprintf(out_f, " top_node->data[top_node->next_index++].token = token;\n");
1084 break;
1085 case CodeGenTypePushNode:
1086 fprintf(out_f, " top_node = allocate<ParserGenNode>(1);\n");
1087 fprintf(out_f, " stack.append(top_node);\n");
1088 break;
1089 case CodeGenTypeCapture:
1090 if (code->capture.is_root) {
1091 Buf *code_text = fill_template(code->capture.body, "root", code->capture.field_names);
1092 fprintf(out_f, "%s\n", buf_ptr(code_text));
1093 fprintf(out_f, " return root;\n");
1094 } else {
1095 fprintf(out_f, " ParserGenNode *parent_node = stack.at(stack.length - 2);\n");
1096 Buf *dest = buf_sprintf("parent_node->data[parent_node->next_index++].%s",
1097 buf_ptr(code->capture.union_field_name));
1098 Buf *code_text = fill_template(code->capture.body, buf_ptr(dest),
1099 code->capture.field_names);
1100 fprintf(out_f, "%s\n", buf_ptr(code_text));
1101 }
1102 break;
1103 case CodeGenTypePopNode:
1104 fprintf(out_f, " stack.pop();\n");
1105 fprintf(out_f, " top_node = stack.length ? stack.last() : nullptr;\n");
1106 break;
1107 case CodeGenTypeEatToken:
1108 fprintf(out_f, " token_index += 1;\n");
1109 fprintf(out_f, " token = (token_index < tokens->length) ? &tokens->at(token_index) : nullptr;\n");
1110 break;
1111 }
1112 }
1113 fprintf(out_f, " break;\n");
1114 fprintf(out_f, " }\n");
1115 }
1116 fprintf(out_f, " default:\n");
1117 fprintf(out_f, " zig_panic(\"unreachable\");\n");
1118
1119 fprintf(out_f, " }\n");
1120 fprintf(out_f, " }\n");
1121 fprintf(out_f, " zig_panic(\"unreachable\");\n");
1122 fprintf(out_f, "}\n");
1123
1124 return 0;
1125}