Commit d9c36cb2506f1b8cb30c7e9e108c6005eea7cf66 | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-11-16 14:05:15-08:00
committer	GitHub <noreply@github.com> 2020-11-16 14:05:15-08:00
log	d9c36cb2506f1b8cb30c7e9e108c6005eea7cf66
tree	0da714e25f1c6663cab17306f5b284bb14ef5e62
parent	9ea4ddae97925ff3a988da8c010e98a3495c1869
parent	1d22591299445b6a28b371f8ffcd4255e7d1a364
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #6878 from frmdstryr/multiline-string-comments

Support comments in multiline string literals

2 files changed, 82 insertions(+), 4 deletions(-)

src
- stage1
  - parser.cpp
test
- stage1
  - behavior
    misc.zig

src/stage1/parser.cpp+40-4

...	@@ -493,6 +493,33 @@ static AstNode ast_parse_root(ParseContext pc) {	...	@@ -493,6 +493,33 @@ static AstNode ast_parse_root(ParseContext pc) {
493	return node;	493	return node;
494	}	494	}
495		495
		496	static Token ast_parse_multiline_string_literal(ParseContext pc, Buf *buf) {
		497	Token *first_str_token = nullptr;
		498	Token *str_token = nullptr;
		499	while ((str_token = eat_token_if(pc, TokenIdMultilineStringLiteral))) {
		500	if (first_str_token == nullptr) {
		501	first_str_token = str_token;
		502	}
		503	if (buf->list.length == 0) {
		504	buf_resize(buf, 0);
		505	}
		506	buf_append_buf(buf, token_buf(str_token));
		507
		508	// Ignore inline comments
		509	size_t cur_token = pc->current_token;
		510	while (eat_token_if(pc, TokenIdDocComment));
		511
		512	// Lookahead to see if there's another multilne string literal,
		513	// if not, we have to revert back to before the doc comment
		514	if (peek_token(pc)->id != TokenIdMultilineStringLiteral) {
		515	pc->current_token = cur_token;
		516	} else {
		517	buf_append_char(buf, '\n'); // Add a newline between comments
		518	}
		519	}
		520	return first_str_token;
		521	}
		522
496	static Token ast_parse_doc_comments(ParseContext pc, Buf *buf) {	523	static Token ast_parse_doc_comments(ParseContext pc, Buf *buf) {
497	Token *first_doc_token = nullptr;	524	Token *first_doc_token = nullptr;
498	Token *doc_token = nullptr;	525	Token *doc_token = nullptr;
...	@@ -605,7 +632,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {	...	@@ -605,7 +632,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
605	case ContainerFieldStateSeen:	632	case ContainerFieldStateSeen:
606	break;	633	break;
607	case ContainerFieldStateEnd:	634	case ContainerFieldStateEnd:
608	ast_error(pc, first_token, "declarations are not allowed between container fields");	635	ast_error(pc, first_token, "declarations are not allowed between container fields");
609	}	636	}
610		637
611	assert(container_field->type == NodeTypeStructField);	638	assert(container_field->type == NodeTypeStructField);
...	@@ -1754,12 +1781,21 @@ static AstNode ast_parse_primary_type_expr(ParseContext pc) {	...	@@ -1754,12 +1781,21 @@ static AstNode ast_parse_primary_type_expr(ParseContext pc) {
1754	if (unreachable != nullptr)	1781	if (unreachable != nullptr)
1755	return ast_create_node(pc, NodeTypeUnreachable, unreachable);	1782	return ast_create_node(pc, NodeTypeUnreachable, unreachable);
1756		1783
		1784
		1785	Buf *string_buf;
1757	Token *string_lit = eat_token_if(pc, TokenIdStringLiteral);	1786	Token *string_lit = eat_token_if(pc, TokenIdStringLiteral);
1758	if (string_lit == nullptr)	1787	if (string_lit != nullptr) {
1759	string_lit = eat_token_if(pc, TokenIdMultilineStringLiteral);	1788	string_buf = token_buf(string_lit);
		1789	} else {
		1790	Buf multiline_string_buf = BUF_INIT;
		1791	string_lit = ast_parse_multiline_string_literal(pc, &multiline_string_buf);
		1792	if (string_lit != nullptr) {
		1793	string_buf = buf_create_from_buf(&multiline_string_buf);
		1794	}
		1795	}
1760	if (string_lit != nullptr) {	1796	if (string_lit != nullptr) {
1761	AstNode *res = ast_create_node(pc, NodeTypeStringLiteral, string_lit);	1797	AstNode *res = ast_create_node(pc, NodeTypeStringLiteral, string_lit);
1762	res->data.string_literal.buf = token_buf(string_lit);	1798	res->data.string_literal.buf = string_buf;
1763	return res;	1799	return res;
1764	}	1800	}
1765		1801

test/stage1/behavior/misc.zig+42

...	@@ -166,6 +166,48 @@ test "multiline string" {	...	@@ -166,6 +166,48 @@ test "multiline string" {
166	expect(mem.eql(u8, s1, s2));	166	expect(mem.eql(u8, s1, s2));
167	}	167	}
168		168
		169	test "multiline string comments at start" {
		170	const s1 =
		171	//\\one
		172	\\two)
		173	\\three
		174	;
		175	const s2 = "two)\nthree";
		176	expect(mem.eql(u8, s1, s2));
		177	}
		178
		179	test "multiline string comments at end" {
		180	const s1 =
		181	\\one
		182	\\two)
		183	//\\three
		184	;
		185	const s2 = "one\ntwo)";
		186	expect(mem.eql(u8, s1, s2));
		187	}
		188
		189	test "multiline string comments in middle" {
		190	const s1 =
		191	\\one
		192	//\\two)
		193	\\three
		194	;
		195	const s2 = "one\nthree";
		196	expect(mem.eql(u8, s1, s2));
		197	}
		198
		199	test "multiline string comments at multiple places" {
		200	const s1 =
		201	\\one
		202	//\\two
		203	\\three
		204	//\\four
		205	\\five
		206	;
		207	const s2 = "one\nthree\nfive";
		208	expect(mem.eql(u8, s1, s2));
		209	}
		210
169	test "multiline C string" {	211	test "multiline C string" {
170	const s1 =	212	const s1 =
171	\\one	213	\\one