Commit 21328e0036db94c75083ab8e43ff9195000343f0 | zig

author	Andrew Kelley <andrew@ziglang.org> 2018-09-18 18:36:39-04:00
committer	Andrew Kelley <andrew@ziglang.org> 2018-09-18 18:36:39-04:00
log	21328e0036db94c75083ab8e43ff9195000343f0
tree	cb6ddae8aaf1d46c7fb33941c9a2ee64cd1e5ba0
parent	345f8db1c49efe3d9ca9859a296f24e728a3b43a
signature	Commit is signed but in an unrecognized format.

closes #1546

5 files changed, 46 insertions(+), 6 deletions(-)

std/zig/ast.zig+2

...	...	@@ -469,6 +469,7 @@ pub const Node = struct {
469	469	doc_comments: ?*DocComment,
470	470	decls: DeclList,
471	471	eof_token: TokenIndex,
	472	shebang: ?TokenIndex,
472	473
473	474	pub const DeclList = SegmentedList(*Node, 4);
474	475
...	...	@@ -480,6 +481,7 @@ pub const Node = struct {
480	481	}
481	482
482	483	pub fn firstToken(self: *const Root) TokenIndex {
	484	if (self.shebang) \|shebang\| return shebang;
483	485	return if (self.decls.len == 0) self.eof_token else (self.decls.at(0).*).firstToken();
484	486	}
485	487

std/zig/parse.zig+10

...	...	@@ -21,6 +21,7 @@ pub fn parse(allocator: *mem.Allocator, source: []const u8) !ast.Tree {
21	21	.base = ast.Node{ .id = ast.Node.Id.Root },
22	22	.decls = ast.Node.Root.DeclList.init(arena),
23	23	.doc_comments = null,
	24	.shebang = null,
24	25	// initialized when we get the eof token
25	26	.eof_token = undefined,
26	27	});
...	...	@@ -41,6 +42,15 @@ pub fn parse(allocator: *mem.Allocator, source: []const u8) !ast.Tree {
41	42	}
42	43	var tok_it = tree.tokens.iterator(0);
43	44
	45	// skip over shebang line
	46	shebang: {
	47	const shebang_tok_index = tok_it.index;
	48	const shebang_tok_ptr = tok_it.peek() orelse break :shebang;
	49	if (shebang_tok_ptr.id != Token.Id.ShebangLine) break :shebang;
	50	root_node.shebang = shebang_tok_index;
	51	_ = tok_it.next();
	52	}
	53
44	54	// skip over line comments at the top of the file
45	55	while (true) {
46	56	const next_tok = tok_it.peek() orelse break;

std/zig/parser_test.zig+8

...	...	@@ -1,3 +1,11 @@
	1	test "zig fmt: shebang line" {
	2	try testCanonical(
	3	\\#!/usr/bin/env zig
	4	\\pub fn main() void {}
	5	\\
	6	);
	7	}
	8
1	9	test "zig fmt: correctly move doc comments on struct fields" {
2	10	try testTransform(
3	11	\\pub const section_64 = extern struct {

std/zig/render.zig+7-1

...	...	@@ -67,8 +67,14 @@ fn renderRoot(
67	67	stream: var,
68	68	tree: *ast.Tree,
69	69	) (@typeOf(stream).Child.Error \|\| Error)!void {
70		// render all the line comments at the beginning of the file
71	70	var tok_it = tree.tokens.iterator(0);
	71
	72	// render the shebang line
	73	if (tree.root_node.shebang) \|shebang\| {
	74	try stream.write(tree.tokenSlice(shebang));
	75	}
	76
	77	// render all the line comments at the beginning of the file
72	78	while (tok_it.next()) \|token\| {
73	79	if (token.id != Token.Id.LineComment) break;
74	80	try stream.print("{}\n", mem.trimRight(u8, tree.tokenSlicePtr(token), " "));

std/zig/tokenizer.zig+19-5

...	...	@@ -145,6 +145,7 @@ pub const Token = struct {
145	145	LineComment,
146	146	DocComment,
147	147	BracketStarBracket,
	148	ShebangLine,
148	149	Keyword_align,
149	150	Keyword_and,
150	151	Keyword_asm,
...	...	@@ -208,11 +209,24 @@ pub const Tokenizer = struct {
208	209	}
209	210
210	211	pub fn init(buffer: []const u8) Tokenizer {
211		return Tokenizer{
212		.buffer = buffer,
213		.index = 0,
214		.pending_invalid_token = null,
215		};
	212	if (mem.startsWith(u8, buffer, "#!")) {
	213	const src_start = if (mem.indexOfScalar(u8, buffer, '\n')) \|i\| i + 1 else buffer.len;
	214	return Tokenizer{
	215	.buffer = buffer,
	216	.index = src_start,
	217	.pending_invalid_token = Token{
	218	.id = Token.Id.ShebangLine,
	219	.start = 0,
	220	.end = src_start,
	221	},
	222	};
	223	} else {
	224	return Tokenizer{
	225	.buffer = buffer,
	226	.index = 0,
	227	.pending_invalid_token = null,
	228	};
	229	}
216	230	}
217	231
218	232	const State = enum {