Commit 90075345519f165a0164c52b62b78453068c6ce6 | zig

author	Ian Johnson <ian@ianjohnson.dev> 2024-09-09 22:23:45-04:00
committer	Veikka Tuominen <git@vexu.eu> 2024-09-10 13:34:33+03:00
log	90075345519f165a0164c52b62b78453068c6ce6
tree	ca83ba36ab298b92db9be64cb46a42a1a608166b
parent	d6d09f4ea7cc590035bf8af6617f93767423c691

std.zig.tokenizer: simplify line-based tokens

Closes #21358
Closes #21360

This commit modifies the `multiline_string_literal_line`, `doc_comment`,
and `container_doc_comment` tokens to no longer include the line ending
as part of the token. This makes it easier to handle line endings (which
may be LF, CRLF, or in edge cases possibly nonexistent) consistently.

In the two issues linked above, Autodoc was already assuming this for
doc comments, and yielding incorrect results when handling files with
CRLF line endings (both in Markdown parsing and source rendering).

Applying the same simplification for multiline string literals also
brings `zig fmt` into conformance with
https://github.com/ziglang/zig-spec/issues/38 regarding formatting of
multiline strings with CRLF line endings: the spec says that `zig fmt`
should remove the CR from such line endings, but this was not previously
the case.

4 files changed, 40 insertions(+), 11 deletions(-)

lib
- std
  - zig
    AstGen.zig
    parser_test.zig
    render.zig
    tokenizer.zig

lib/std/zig/AstGen.zig+2-4

...	...	@@ -11721,16 +11721,14 @@ fn strLitNodeAsString(astgen: *AstGen, node: Ast.Node.Index) !IndexSlice {
11721	11721	var tok_i = start;
11722	11722	{
11723	11723	const slice = tree.tokenSlice(tok_i);
11724		const carriage_return_ending: usize = if (slice[slice.len - 2] == '\r') 2 else 1;
11725		const line_bytes = slice[2 .. slice.len - carriage_return_ending];
	11724	const line_bytes = slice[2..];
11726	11725	try string_bytes.appendSlice(gpa, line_bytes);
11727	11726	tok_i += 1;
11728	11727	}
11729	11728	// Following lines: each line prepends a newline.
11730	11729	while (tok_i <= end) : (tok_i += 1) {
11731	11730	const slice = tree.tokenSlice(tok_i);
11732		const carriage_return_ending: usize = if (slice[slice.len - 2] == '\r') 2 else 1;
11733		const line_bytes = slice[2 .. slice.len - carriage_return_ending];
	11731	const line_bytes = slice[2..];
11734	11732	try string_bytes.ensureUnusedCapacity(gpa, line_bytes.len + 1);
11735	11733	string_bytes.appendAssumeCapacity('\n');
11736	11734	string_bytes.appendSliceAssumeCapacity(line_bytes);

lib/std/zig/parser_test.zig+38

...	...	@@ -3087,6 +3087,22 @@ test "zig fmt: multiline string" {
3087	3087	);
3088	3088	}
3089	3089
	3090	test "zig fmt: multiline string with CRLF line endings" {
	3091	try testTransform("" ++
	3092	"const s =\r\n" ++
	3093	" \\\\one\r\n" ++
	3094	" \\\\two)\r\n" ++
	3095	" \\\\three\r\n" ++
	3096	";\r\n",
	3097	\\const s =
	3098	\\ \\one
	3099	\\ \\two)
	3100	\\ \\three
	3101	\\;
	3102	\\
	3103	);
	3104	}
	3105
3090	3106	test "zig fmt: values" {
3091	3107	try testCanonical(
3092	3108	\\test "values" {
...	...	@@ -4404,6 +4420,28 @@ test "zig fmt: invalid doc comments on comptime and test blocks" {
4404	4420	});
4405	4421	}
4406	4422
	4423	test "zig fmt: comments with CRLF line endings" {
	4424	try testTransform("" ++
	4425	"//! Top-level doc comment\r\n" ++
	4426	"//! Continuing to another line\r\n" ++
	4427	"\r\n" ++
	4428	"/// Regular doc comment\r\n" ++
	4429	"const S = struct {\r\n" ++
	4430	" // Regular comment\r\n" ++
	4431	" // More content\r\n" ++
	4432	"};\r\n",
	4433	\\//! Top-level doc comment
	4434	\\//! Continuing to another line
	4435	\\
	4436	\\/// Regular doc comment
	4437	\\const S = struct {
	4438	\\ // Regular comment
	4439	\\ // More content
	4440	\\};
	4441	\\
	4442	);
	4443	}
	4444
4407	4445	test "zig fmt: else comptime expr" {
4408	4446	try testCanonical(
4409	4447	\\comptime {

lib/std/zig/render.zig-3

...	...	@@ -3170,9 +3170,6 @@ fn discardAllParams(r: *Render, fn_proto_node: Ast.Node.Index) Error!void {
3170	3170	fn tokenSliceForRender(tree: Ast, token_index: Ast.TokenIndex) []const u8 {
3171	3171	var ret = tree.tokenSlice(token_index);
3172	3172	switch (tree.tokens.items(.tag)[token_index]) {
3173		.multiline_string_literal_line => {
3174		if (ret[ret.len - 1] == '\n') ret.len -= 1;
3175		},
3176	3173	.container_doc_comment, .doc_comment => {
3177	3174	ret = mem.trimRight(u8, ret, &std.ascii.whitespace);
3178	3175	},

lib/std/zig/tokenizer.zig-4

...	...	@@ -847,12 +847,10 @@ pub const Tokenizer = struct {
847	847	break;
848	848	},
849	849	'\n' => {
850		self.index += 1;
851	850	break;
852	851	},
853	852	'\r' => {
854	853	if (self.buffer[self.index + 1] == '\n') {
855		self.index += 2;
856	854	break;
857	855	} else {
858	856	state = .invalid;
...	...	@@ -1117,7 +1115,6 @@ pub const Tokenizer = struct {
1117	1115	},
1118	1116	'\r' => {
1119	1117	if (self.buffer[self.index + 1] == '\n') {
1120		self.index += 1;
1121	1118	result.tag = .doc_comment;
1122	1119	break;
1123	1120	} else {
...	...	@@ -1167,7 +1164,6 @@ pub const Tokenizer = struct {
1167	1164	},
1168	1165	'\r' => {
1169	1166	if (self.buffer[self.index + 1] == '\n') {
1170		self.index += 1;
1171	1167	break;
1172	1168	} else {
1173	1169	state = .invalid;