Commit 4236b85c722b36cb31595634bae0ff28d6082761 | zig

author	Andrew Kelley <andrew@ziglang.org> 2015-11-07 04:50:48-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2015-11-07 04:50:48-07:00
log	4236b85c722b36cb31595634bae0ff28d6082761
tree	37ad80afbb6e28e46eaf1c5808c35018a538aa54
parent	ae0e9685c690e9ddb31a28e5a9f827ea8da111ca

parser generator supports a simple OR

5 files changed, 214 insertions(+), 75 deletions(-)

src/parser.cpp+6

...	...	@@ -82,3 +82,9 @@ void ast_print(AstNode *node, int indent) {
82	82	AstNode ast_create_root(Token token) {
83	83	return nullptr;
84	84	}
	85
	86	void ast_invalid_token_error(Buf buf, Token token) {
	87	Buf token_value = {0};
	88	buf_init_from_mem(&token_value, buf_ptr(buf) + token->start_pos, token->end_pos - token->start_pos);
	89	ast_error(token, "invalid token: '%s'", buf_ptr(&token_value));
	90	}

src/parser.hpp+2

...	...	@@ -81,6 +81,8 @@ struct AstNode {
81	81
82	82	__attribute__ ((format (printf, 2, 3)))
83	83	void ast_error(Token token, const char format, ...);
	84	void ast_invalid_token_error(Buf buf, Token token);
	85
84	86
85	87	// This function is provided by generated code, generated by parsergen.cpp
86	88	AstNode * ast_parse(Buf buf, ZigList<Token> tokens);

src/parsergen.cpp+202-74

...	...	@@ -133,10 +133,8 @@ struct Token {
133	133	struct RuleNode;
134	134
135	135	struct RuleTuple {
136		Buf name;
137	136	ZigList<RuleNode *> children;
138	137	Buf body;
139		Buf union_field_name;
140	138	};
141	139
142	140	struct RuleMany {
...	...	@@ -144,11 +142,13 @@ struct RuleMany {
144	142	};
145	143
146	144	struct RuleOption {
147		ZigList<RuleNode *> child;
	145	RuleNode *child;
148	146	};
149	147
150	148	struct RuleOr {
151		ZigList<RuleTuple *> children;
	149	Buf name;
	150	Buf union_field_name;
	151	ZigList<RuleNode *> children;
152	152	};
153	153
154	154	struct RuleToken {
...	...	@@ -227,6 +227,7 @@ struct ParserState {
227	227	// One for each token ID.
228	228	ParserState **transition;
229	229	int index;
	230	bool is_error;
230	231	};
231	232
232	233	enum LexState {
...	...	@@ -250,8 +251,9 @@ struct LexStack {
250	251
251	252	struct Gen {
252	253	ZigList<RuleNode *> rules;
253		ParserState *cur_state;
	254
254	255	ZigList<ParserState *> transition_table;
	256	ParserState *start_state;
255	257	ZigList<Token *> tokens;
256	258	RuleNode *root;
257	259	int biggest_tuple_len;
...	...	@@ -260,7 +262,8 @@ struct Gen {
260	262	LexState lex_state;
261	263	int lex_line;
262	264	int lex_column;
263		RuleNode *lex_cur_rule;
	265	RuleNode *lex_cur_or_rule;
	266	RuleNode *lex_cur_tuple_rule;;
264	267	int lex_cur_rule_begin;
265	268	int lex_fn_name_begin;
266	269	int lex_pos;
...	...	@@ -274,9 +277,8 @@ struct Gen {
274	277
275	278	static ParserState create_state(Gen g) {
276	279	ParserState *state = allocate<ParserState>(1);
277		state->index = g->transition_table.length;
	280	state->index = -1;
278	281	state->transition = allocate<ParserState*>(g->tokens.length);
279		g->transition_table.append(state);
280	282	return state;
281	283	}
282	284
...	...	@@ -301,6 +303,7 @@ static void state_add_error(ParserState state, Buf msg) {
301	303	code->type = CodeGenTypeError;
302	304	code->error.msg = msg;
303	305	state_add_code(state, code);
	306	state->is_error = true;
304	307	}
305	308
306	309	static void state_add_transition(ParserState *state) {
...	...	@@ -337,45 +340,61 @@ static void state_add_eat_token(ParserState *state) {
337	340	state_add_code(state, code);
338	341	}
339	342
340		static void gen(Gen g, RuleNode node, Buf *out_field_name) {
	343
	344	static void gen(Gen g, RuleNode node, Buf out_field_name, ParserState cur_state,
	345	ZigList<ParserState > end_states, bool is_root)
	346	{
	347	struct PossibleState {
	348	ParserState *test_state;
	349	ZigList<ParserState *> end_states;
	350	};
341	351	assert(node);
342	352	switch (node->type) {
343	353	case RuleNodeTypeToken:
344	354	{
345	355	buf_init_from_str(out_field_name, "token");
346	356
347		state_add_save_token(g->cur_state);
	357	state_add_save_token(cur_state);
348	358
349	359	ParserState *ok_state = create_state(g);
350	360	ParserState *err_state = create_state(g);
351	361	state_add_error(err_state, buf_sprintf("expected token '%s'", buf_ptr(&node->token.token->name)));
352	362
353		fill_state_with_transition(g, g->cur_state, err_state);
354		g->cur_state->transition[node->token.token->id] = ok_state;
355		state_add_transition(g->cur_state);
356		state_add_eat_token(g->cur_state);
	363	fill_state_with_transition(g, cur_state, err_state);
	364	cur_state->transition[node->token.token->id] = ok_state;
	365	state_add_transition(cur_state);
	366	state_add_eat_token(cur_state);
357	367
358		g->cur_state = ok_state;
	368	end_states->append(ok_state);
359	369	}
360	370	break;
361	371	case RuleNodeTypeTuple:
362	372	{
363		buf_init_from_buf(out_field_name, &node->tuple.union_field_name);
364
365		state_add_push_node(g->cur_state);
366	373
367		bool is_root = (node == g->root);
368	374	int field_name_count = node->tuple.children.length;
369	375	CodeGen *code = codegen_create_capture(&node->tuple.body, is_root, field_name_count,
370		&node->tuple.union_field_name);
	376	out_field_name);
371	377
372		for (int i = 0; i < node->tuple.children.length; i += 1) {
373		RuleNode *child = node->tuple.children.at(i);
374		gen(g, child, &code->capture.field_names[i]);
	378	ZigList<ParserState > my_end_states = allocate<ZigList<ParserState *>>(1);
	379	my_end_states->append(cur_state);
	380	for (int child_index = 0; child_index < node->tuple.children.length; child_index += 1) {
	381	RuleNode *child = node->tuple.children.at(child_index);
	382
	383	ZigList<ParserState > more_end_states = allocate<ZigList<ParserState *>>(1);
	384	for (int i = 0; i < my_end_states->length; i += 1) {
	385	ParserState *use_state = my_end_states->at(i);
	386	gen(g, child, &code->capture.field_names[i], use_state, more_end_states, false);
	387	}
	388
	389	my_end_states = more_end_states;
375	390	}
376		state_add_code(g->cur_state, code);
377	391
378		state_add_pop_node(g->cur_state);
	392	for (int i = 0; i < my_end_states->length; i += 1) {
	393	ParserState *use_state = my_end_states->at(i);
	394	state_add_code(use_state, code);
	395
	396	end_states->append(use_state);
	397	}
379	398	}
380	399	break;
381	400	case RuleNodeTypeMany:
...	...	@@ -388,12 +407,73 @@ static void gen(Gen g, RuleNode node, Buf *out_field_name) {
388	407	zig_panic("TODO");
389	408	break;
390	409	case RuleNodeTypeOr:
391		zig_panic("TODO");
	410	{
	411	buf_init_from_buf(out_field_name, &node->_or.union_field_name);
	412
	413	state_add_push_node(cur_state);
	414
	415	// TODO this probably need to get moved when or can handle conflicts
	416	state_add_save_token(cur_state);
	417	state_add_transition(cur_state);
	418	state_add_eat_token(cur_state);
	419
	420
	421	int possible_state_count = node->_or.children.length;
	422	PossibleState *possible_states = allocate<PossibleState>(possible_state_count);
	423	for (int i = 0; i < possible_state_count; i += 1) {
	424	RuleNode *child = node->_or.children.at(i);
	425	assert(child->type == RuleNodeTypeTuple);
	426
	427	PossibleState *possible_state = &possible_states[i];
	428	possible_state->test_state = create_state(g);
	429	gen(g, child, &node->_or.union_field_name, possible_state->test_state,
	430	&possible_state->end_states, is_root);
	431	}
	432
	433	// try to merge all the possible states into new state.
	434	ParserState *err_state = create_state(g);
	435	state_add_error(err_state, buf_create_from_str("unexpected token"));
	436	for (int token_i = 0; token_i < g->tokens.length; token_i += 1) {
	437	bool any_called_it = false;
	438	bool conflict = false;
	439	for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
	440	PossibleState *possible_state = &possible_states[state_i];
	441	if (!possible_state->test_state->transition[token_i]->is_error) {
	442	if (any_called_it) {
	443	conflict = true;
	444	} else {
	445	any_called_it = true;
	446	}
	447	}
	448	}
	449	if (conflict) {
	450	zig_panic("TODO state transition conflict");
	451	} else {
	452	cur_state->transition[token_i] = err_state;
	453	for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
	454	PossibleState *possible_state = &possible_states[state_i];
	455	if (!possible_state->test_state->transition[token_i]->is_error) {
	456	cur_state->transition[token_i] = possible_state->test_state->transition[token_i];
	457	}
	458	}
	459	}
	460	}
	461
	462	for (int state_i = 0; state_i < possible_state_count; state_i += 1) {
	463	PossibleState *possible_state = &possible_states[state_i];
	464	for (int end_i = 0; end_i < possible_state->end_states.length; end_i += 1) {
	465	ParserState *state = possible_state->end_states.at(end_i);
	466	state_add_pop_node(state);
	467
	468	end_states->append(state);
	469	}
	470	}
	471	}
392	472	break;
393	473	case RuleNodeTypeSubRule:
394	474	{
395	475	RuleNode *child = node->sub_rule.child;
396		gen(g, child, out_field_name);
	476	gen(g, child, out_field_name, cur_state, end_states, false);
397	477	}
398	478	break;
399	479	}
...	...	@@ -451,38 +531,51 @@ static RuleNode create_rule_node(Gen g) {
451	531	}
452	532
453	533	static void begin_rule(Gen *g) {
454		assert(!g->lex_cur_rule);
455		g->lex_cur_rule = create_rule_node(g);
456		g->lex_cur_rule->type = RuleNodeTypeTuple;
457		g->lex_cur_rule_begin = g->lex_pos;
	534	assert(!g->lex_cur_or_rule);
	535	assert(!g->lex_cur_tuple_rule);
458	536
459		g->lex_state = LexStateEndOrOr;
460		lex_push_stack(g);
	537	g->lex_cur_or_rule = create_rule_node(g);
	538	g->lex_cur_or_rule->type = RuleNodeTypeOr;
	539
	540	g->lex_cur_tuple_rule = create_rule_node(g);
	541	g->lex_cur_tuple_rule->type = RuleNodeTypeTuple;
	542	g->lex_cur_rule_begin = g->lex_pos;
461	543	}
462	544
463	545	static void end_rule(Gen *g) {
464		assert(g->lex_cur_rule);
465		g->rules.append(g->lex_cur_rule);
466		g->lex_cur_rule = nullptr;
	546	assert(g->lex_cur_or_rule);
	547	assert(!g->lex_cur_tuple_rule);
	548
	549	g->rules.append(g->lex_cur_or_rule);
	550	g->lex_cur_or_rule = nullptr;
	551	}
	552
	553	static void perform_or(Gen *g) {
	554	assert(g->lex_cur_or_rule);
	555	assert(!g->lex_cur_tuple_rule);
	556
	557	g->lex_cur_tuple_rule = create_rule_node(g);
	558	g->lex_cur_tuple_rule->type = RuleNodeTypeTuple;
	559	g->lex_cur_rule_begin = g->lex_pos;
467	560	}
468	561
469	562	static void end_rule_name(Gen *g) {
470		assert(g->lex_cur_rule);
	563	assert(g->lex_cur_or_rule);
471	564	char *ptr = &buf_ptr(g->in_buf)[g->lex_cur_rule_begin];
472	565	int len = g->lex_pos - g->lex_cur_rule_begin;
473		buf_init_from_mem(&g->lex_cur_rule->tuple.name, ptr, len);
	566	buf_init_from_mem(&g->lex_cur_or_rule->_or.name, ptr, len);
474	567	}
475	568
476	569	static void begin_rule_field_name(Gen *g) {
477		assert(g->lex_cur_rule);
	570	assert(g->lex_cur_or_rule);
478	571	g->lex_field_name_begin = g->lex_pos;
479	572	}
480	573
481	574	static void end_rule_field_name(Gen *g) {
482		assert(g->lex_cur_rule);
	575	assert(g->lex_cur_or_rule);
483	576	char *ptr = &buf_ptr(g->in_buf)[g->lex_field_name_begin];
484	577	int len = g->lex_pos - g->lex_field_name_begin;
485		buf_init_from_mem(&g->lex_cur_rule->tuple.union_field_name, ptr, len);
	578	buf_init_from_mem(&g->lex_cur_or_rule->_or.union_field_name, ptr, len);
486	579	}
487	580
488	581	static void begin_fn_name(Gen *g) {
...	...	@@ -505,6 +598,9 @@ static void begin_token_name(Gen *g) {
505	598	}
506	599
507	600	static void end_token_name(Gen *g) {
	601	assert(g->lex_cur_tuple_rule);
	602	assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
	603
508	604	char *ptr = &buf_ptr(g->in_buf)[g->lex_token_name_begin];
509	605	int len = g->lex_pos - g->lex_token_name_begin;
510	606	Buf token_name = {0};
...	...	@@ -515,24 +611,27 @@ static void end_token_name(Gen *g) {
515	611	node->type = RuleNodeTypeToken;
516	612	node->token.token = token;
517	613
518		assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
519		g->lex_cur_rule->tuple.children.append(node);
	614	g->lex_cur_tuple_rule->tuple.children.append(node);
520	615
521	616
522	617	lex_pop_stack(g);
523	618	}
524	619
525	620	static void begin_tuple_body(Gen *g) {
526		assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
	621	assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
527	622	g->lex_body_begin = g->lex_pos;
528	623	}
529	624
530	625	static void end_tuple_body(Gen *g) {
531		assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
	626	assert(g->lex_cur_or_rule);
	627	assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
532	628	int end_pos = g->lex_pos + 1;
533	629	char *ptr = &buf_ptr(g->in_buf)[g->lex_body_begin];
534	630	int len = end_pos - g->lex_body_begin;
535		buf_init_from_mem(&g->lex_cur_rule->tuple.body, ptr, len);
	631	buf_init_from_mem(&g->lex_cur_tuple_rule->tuple.body, ptr, len);
	632
	633	g->lex_cur_or_rule->_or.children.append(g->lex_cur_tuple_rule);
	634	g->lex_cur_tuple_rule = nullptr;
536	635	}
537	636
538	637	static void begin_sub_tuple(Gen *g) {
...	...	@@ -541,7 +640,7 @@ static void begin_sub_tuple(Gen *g) {
541	640	}
542	641
543	642	static void end_sub_tuple(Gen *g) {
544		assert(g->lex_cur_rule->type == RuleNodeTypeTuple);
	643	assert(g->lex_cur_tuple_rule->type == RuleNodeTypeTuple);
545	644	char *ptr = &buf_ptr(g->in_buf)[g->lex_sub_tuple_begin];
546	645	int len = g->lex_pos - g->lex_sub_tuple_begin;
547	646
...	...	@@ -549,7 +648,7 @@ static void end_sub_tuple(Gen *g) {
549	648	node->type = RuleNodeTypeSubRule;
550	649	buf_init_from_mem(&node->sub_rule.name, ptr, len);
551	650
552		g->lex_cur_rule->tuple.children.append(node);
	651	g->lex_cur_tuple_rule->tuple.children.append(node);
553	652
554	653	lex_pop_stack(g);
555	654	}
...	...	@@ -557,8 +656,8 @@ static void end_sub_tuple(Gen *g) {
557	656	static RuleNode find_rule_node(Gen g, Buf *name) {
558	657	for (int i = 0; i < g->rules.length; i += 1) {
559	658	RuleNode *node = g->rules.at(i);
560		assert(node->type == RuleNodeTypeTuple);
561		if (buf_eql_buf(&node->tuple.name, name)) {
	659	assert(node->type == RuleNodeTypeOr);
	660	if (buf_eql_buf(&node->_or.name, name)) {
562	661	return node;
563	662	}
564	663	}
...	...	@@ -691,7 +790,7 @@ static void initialize_rules(Gen *g) {
691	790	switch (c) {
692	791	case '}':
693	792	end_tuple_body(g);
694		lex_pop_stack(g);
	793	g->lex_state = LexStateEndOrOr;
695	794	break;
696	795	default:
697	796	// ignore
...	...	@@ -707,6 +806,10 @@ static void initialize_rules(Gen *g) {
707	806	end_rule(g);
708	807	g->lex_state = LexStateStart;
709	808	break;
	809	case '\|':
	810	perform_or(g);
	811	g->lex_state = LexStateTupleRule;
	812	break;
710	813	default:
711	814	lex_error(g, "expected ';' or '\|'");
712	815	}
...	...	@@ -751,32 +854,39 @@ static void initialize_rules(Gen *g) {
751	854	break;
752	855	}
753	856
754		// Resolve child references into pointers
755		for (int tuple_i = 0; tuple_i < g->rules.length; tuple_i += 1) {
756		RuleNode *node = g->rules.at(tuple_i);
757		assert(node->type == RuleNodeTypeTuple);
758
759		for (int child_i = 0; child_i < node->tuple.children.length; child_i += 1) {
760		RuleNode *child = node->tuple.children.at(child_i);
761		if (child->type == RuleNodeTypeSubRule) {
762		int line = child->lex_line + 1;
763		int column = child->lex_column + 1;
764		RuleNode *referenced_node = find_rule_node(g, &child->sub_rule.name);
765		if (!referenced_node) {
766		fprintf(stderr, "Grammar Error: Line %d, column %d: Rule not defined: '%s'\n",
767		line, column, buf_ptr(&child->sub_rule.name));
	857	// Iterate over the rules and
	858	// * resolve child references into pointers
	859	// * calculate the biggest tuple len
	860	bool any_errors = false;
	861	for (int or_i = 0; or_i < g->rules.length; or_i += 1) {
	862	RuleNode *or_node = g->rules.at(or_i);
	863	assert(or_node->type == RuleNodeTypeOr);
	864
	865	for (int tuple_i = 0; tuple_i < or_node->_or.children.length; tuple_i += 1) {
	866	RuleNode *tuple_node = or_node->_or.children.at(tuple_i);
	867	assert(tuple_node->type == RuleNodeTypeTuple);
	868	g->biggest_tuple_len = max(g->biggest_tuple_len, tuple_node->tuple.children.length);
	869
	870	for (int child_i = 0; child_i < tuple_node->tuple.children.length; child_i += 1) {
	871	RuleNode *child = tuple_node->tuple.children.at(child_i);
	872
	873	if (child->type == RuleNodeTypeSubRule) {
	874	int line = child->lex_line + 1;
	875	int column = child->lex_column + 1;
	876	RuleNode *referenced_node = find_rule_node(g, &child->sub_rule.name);
	877	if (!referenced_node) {
	878	fprintf(stderr, "Grammar Error: Line %d, column %d: Rule not defined: '%s'\n",
	879	line, column, buf_ptr(&child->sub_rule.name));
	880	any_errors = true;
	881	}
	882	child->sub_rule.child = referenced_node;
768	883	}
769		child->sub_rule.child = referenced_node;
770	884	}
771	885	}
772	886	}
773	887
774
775		// calculate the biggest tuple len
776		for (int i = 0; i < g->rules.length; i += 1) {
777		RuleNode *node = g->rules.at(i);
778		assert(node->type == RuleNodeTypeTuple);
779		g->biggest_tuple_len = max(g->biggest_tuple_len, node->tuple.children.length);
	888	if (any_errors) {
	889	exit(EXIT_FAILURE);
780	890	}
781	891	}
782	892
...	...	@@ -851,6 +961,19 @@ static Buf fill_template(Buf body, const char result_name, Buf field_names)
851	961	return result;
852	962	}
853	963
	964	static void build_transition_table(Gen g, ParserState state) {
	965	if (!state)
	966	return;
	967	if (state->index >= 0)
	968	return;
	969	state->index = g->transition_table.length;
	970	g->transition_table.append(state);
	971	for (int i = 0; i < g->tokens.length; i += 1) {
	972	ParserState *other_state = state->transition[i];
	973	build_transition_table(g, other_state);
	974	}
	975	}
	976
854	977	int main(int argc, char **argv) {
855	978	const char *in_filename = argv[1];
856	979	const char *out_filename = argv[2];
...	...	@@ -882,9 +1005,11 @@ int main(int argc, char **argv) {
882	1005
883	1006	g.root = g.rules.at(0);
884	1007
885		g.cur_state = create_state(&g);
	1008	g.start_state = create_state(&g);
886	1009	Buf root_field_name = {0};
887		gen(&g, g.root, &root_field_name);
	1010	ZigList<ParserState *> end_states = {0};
	1011	gen(&g, g.root, &root_field_name, g.start_state, &end_states, true);
	1012	build_transition_table(&g, g.start_state);
888	1013
889	1014	fprintf(out_f, "/* This file is generated by parsergen.cpp */\n");
890	1015	fprintf(out_f, "\n");
...	...	@@ -950,6 +1075,9 @@ int main(int argc, char **argv) {
950	1075	CodeGen *code = state->code_gen_list.at(code_i);
951	1076	switch (code->type) {
952	1077	case CodeGenTypeTransition:
	1078	fprintf(out_f, " if (token->id < 0 \|\| token->id >= %d) {\n", g.tokens.length);
	1079	fprintf(out_f, " ast_invalid_token_error(buf, token);\n");
	1080	fprintf(out_f, " }\n");
953	1081	fprintf(out_f, " assert(transition[%d][token->id] >= 0);\n", state->index);
954	1082	fprintf(out_f, " assert(transition[%d][token->id] < %d);\n",
955	1083	state->index, g.transition_table.length);

src/tokenizer.hpp+1-1

...	...	@@ -40,6 +40,7 @@ enum TokenId {
40	40	TokenIdRParen = 1,
41	41	TokenIdEof = 2,
42	42	TokenIdStar = 3,
	43	TokenIdPlus = 4,
43	44	TokenIdSymbol,
44	45	TokenIdKeywordFn,
45	46	TokenIdKeywordReturn,
...	...	@@ -51,7 +52,6 @@ enum TokenId {
51	52	TokenIdStringLiteral,
52	53	TokenIdSemicolon,
53	54	TokenIdNumberLiteral,
54		TokenIdPlus,
55	55	TokenIdColon,
56	56	TokenIdArrow,
57	57	TokenIdDash,

src/util.hpp+3

...	...	@@ -12,6 +12,9 @@
12	12	#include <string.h>
13	13	#include <assert.h>
14	14
	15
	16	#define BREAKPOINT __asm("int $0x03")
	17
15	18	void zig_panic(const char *format, ...)
16	19	__attribute__((cold))
17	20	__attribute__ ((noreturn))