authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-22 00:28:07-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-22 00:28:59-04:00
log295bca9b5f397ff98e5a0162fcb2a9f5e0a3e35c
tree96ca941e76281f9868057db1b496856adc13e256
parent1dac9e71b525cb14eb358eaf0bc45b1554f5bf55

stage2 parser: don't append doc comments to the list

The DocComment AST node now only points to the first doc comment token. API users are expected to iterate over the following tokens directly. After this commit there are no more linked lists in use in the self-hosted AST API. Performance impact is negligible. Memory usage slightly reduced.

3 files changed, 60 insertions(+), 54 deletions(-)

lib/std/zig/ast.zig+8-9
......@@ -1,7 +1,6 @@
11const std = @import("../std.zig");
22const assert = std.debug.assert;
33const testing = std.testing;
4const LinkedList = std.SinglyLinkedList;
54const mem = std.mem;
65const Token = std.zig.Token;
76
......@@ -3013,9 +3012,10 @@ pub const Node = struct {
30133012
30143013 pub const DocComment = struct {
30153014 base: Node = Node{ .id = .DocComment },
3016 lines: LineList,
3017
3018 pub const LineList = LinkedList(TokenIndex);
3015 /// Points to the first doc comment token. API users are expected to iterate over the
3016 /// tokens array, looking for more doc comments, ignoring line comments, and stopping
3017 /// at the first other token.
3018 first_line: TokenIndex,
30193019
30203020 pub fn iterate(self: *const DocComment) Node.Iterator {
30213021 return .{ .parent_node = &self.base, .index = 0 };
......@@ -3026,14 +3026,13 @@ pub const Node = struct {
30263026 }
30273027
30283028 pub fn firstToken(self: *const DocComment) TokenIndex {
3029 return self.lines.first.?.data;
3029 return self.first_line;
30303030 }
30313031
3032 /// Returns the first doc comment line. Be careful, this may not be the desired behavior,
3033 /// which would require the tokens array.
30323034 pub fn lastToken(self: *const DocComment) TokenIndex {
3033 var node = self.lines.first.?;
3034 while (true) {
3035 node = node.next orelse return node.data;
3036 }
3035 return self.first_line;
30373036 }
30383037 };
30393038
lib/std/zig/parse.zig+15-30
......@@ -58,6 +58,8 @@ const Parser = struct {
5858 arena: std.heap.ArenaAllocator,
5959 gpa: *Allocator,
6060 source: []const u8,
61 /// TODO: Optimization idea: have this be several arrays of the token fields rather
62 /// than an array of structs.
6163 tokens: []const Token,
6264 tok_i: TokenIndex,
6365 errors: std.ArrayListUnmanaged(AstError),
......@@ -367,20 +369,13 @@ const Parser = struct {
367369
368370 /// Eat a multiline container doc comment
369371 fn parseContainerDocComments(p: *Parser) !?*Node {
370 var lines = Node.DocComment.LineList{};
371 var lines_it: *?*Node.DocComment.LineList.Node = &lines.first;
372
373 while (p.eatToken(.ContainerDocComment)) |line| {
374 lines_it = try p.llpush(TokenIndex, lines_it, line);
372 if (p.eatToken(.ContainerDocComment)) |first_line| {
373 while (p.eatToken(.ContainerDocComment)) |_| {}
374 const node = try p.arena.allocator.create(Node.DocComment);
375 node.* = .{ .first_line = first_line };
376 return &node.base;
375377 }
376
377 if (lines.first == null) return null;
378
379 const node = try p.arena.allocator.create(Node.DocComment);
380 node.* = .{
381 .lines = lines,
382 };
383 return &node.base;
378 return null;
384379 }
385380
386381 /// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block
......@@ -3210,20 +3205,13 @@ const Parser = struct {
32103205
32113206 /// Eat a multiline doc comment
32123207 fn parseDocComment(p: *Parser) !?*Node.DocComment {
3213 var lines = Node.DocComment.LineList{};
3214 var lines_it = &lines.first;
3215
3216 while (p.eatToken(.DocComment)) |line| {
3217 lines_it = try p.llpush(TokenIndex, lines_it, line);
3208 if (p.eatToken(.DocComment)) |first_line| {
3209 while (p.eatToken(.DocComment)) |_| {}
3210 const node = try p.arena.allocator.create(Node.DocComment);
3211 node.* = .{ .first_line = first_line };
3212 return node;
32183213 }
3219
3220 if (lines.first == null) return null;
3221
3222 const node = try p.arena.allocator.create(Node.DocComment);
3223 node.* = .{
3224 .lines = lines,
3225 };
3226 return node;
3214 return null;
32273215 }
32283216
32293217 fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool {
......@@ -3234,11 +3222,8 @@ const Parser = struct {
32343222 fn parseAppendedDocComment(p: *Parser, after_token: TokenIndex) !?*Node.DocComment {
32353223 const comment_token = p.eatToken(.DocComment) orelse return null;
32363224 if (p.tokensOnSameLine(after_token, comment_token)) {
3237 var lines = Node.DocComment.LineList{};
3238 _ = try p.llpush(TokenIndex, &lines.first, comment_token);
3239
32403225 const node = try p.arena.allocator.create(Node.DocComment);
3241 node.* = .{ .lines = lines };
3226 node.* = .{ .first_line = comment_token };
32423227 return node;
32433228 }
32443229 p.putBackToken(comment_token);
lib/std/zig/render.zig+37-15
......@@ -327,11 +327,18 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree,
327327
328328 .DocComment => {
329329 const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl);
330 var it = comment.lines.first;
331 while (it) |node| : (it = node.next) {
332 try renderToken(tree, stream, node.data, indent, start_col, .Newline);
333 if (node.next != null) {
330 const kind = tree.tokens[comment.first_line].id;
331 try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline);
332 var tok_i = comment.first_line + 1;
333 while (true) : (tok_i += 1) {
334 const tok_id = tree.tokens[tok_i].id;
335 if (tok_id == kind) {
334336 try stream.writeByteNTimes(' ', indent);
337 try renderToken(tree, stream, tok_i, indent, start_col, .Newline);
338 } else if (tok_id == .LineComment) {
339 continue;
340 } else {
341 break;
335342 }
336343 }
337344 },
......@@ -2428,17 +2435,32 @@ fn renderDocComments(
24282435 start_col: *usize,
24292436) (@TypeOf(stream).Error || Error)!void {
24302437 const comment = node.doc_comments orelse return;
2431 var it = comment.lines.first;
2432 const first_token = node.firstToken();
2433 while (it) |line_token_index_node| : (it = line_token_index_node.next) {
2434 const line_token_index = line_token_index_node.data;
2435 if (line_token_index < first_token) {
2436 try renderToken(tree, stream, line_token_index, indent, start_col, Space.Newline);
2437 try stream.writeByteNTimes(' ', indent);
2438 } else {
2439 try renderToken(tree, stream, line_token_index, indent, start_col, Space.NoComment);
2440 try stream.writeAll("\n");
2441 try stream.writeByteNTimes(' ', indent);
2438 return renderDocCommentsToken(tree, stream, comment, node.firstToken(), indent, start_col);
2439}
2440
2441fn renderDocCommentsToken(
2442 tree: *ast.Tree,
2443 stream: var,
2444 comment: *ast.Node.DocComment,
2445 first_token: ast.TokenIndex,
2446 indent: usize,
2447 start_col: *usize,
2448) (@TypeOf(stream).Error || Error)!void {
2449 var tok_i = comment.first_line;
2450 while (true) : (tok_i += 1) {
2451 switch (tree.tokens[tok_i].id) {
2452 .DocComment, .ContainerDocComment => {
2453 if (comment.first_line < first_token) {
2454 try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline);
2455 try stream.writeByteNTimes(' ', indent);
2456 } else {
2457 try renderToken(tree, stream, tok_i, indent, start_col, Space.NoComment);
2458 try stream.writeAll("\n");
2459 try stream.writeByteNTimes(' ', indent);
2460 }
2461 },
2462 .LineComment => continue,
2463 else => break,
24422464 }
24432465 }
24442466}