From 295bca9b5f397ff98e5a0162fcb2a9f5e0a3e35c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 22 May 2020 00:28:07 -0400 Subject: [PATCH] stage2 parser: don't append doc comments to the list The DocComment AST node now only points to the first doc comment token. API users are expected to iterate over the following tokens directly. After this commit there are no more linked lists in use in the self-hosted AST API. Performance impact is negligible. Memory usage slightly reduced. --- lib/std/zig/ast.zig | 17 +++++++------- lib/std/zig/parse.zig | 45 ++++++++++++------------------------ lib/std/zig/render.zig | 52 ++++++++++++++++++++++++++++++------------ 3 files changed, 60 insertions(+), 54 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index eff461fdd81a69fe87f45c9ae4968df25777d392..eb22ac2330fec16f03b416b8ea982ace951c9f37 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -1,7 +1,6 @@ const std = @import("../std.zig"); const assert = std.debug.assert; const testing = std.testing; -const LinkedList = std.SinglyLinkedList; const mem = std.mem; const Token = std.zig.Token; @@ -3013,9 +3012,10 @@ pub const Node = struct { pub const DocComment = struct { base: Node = Node{ .id = .DocComment }, - lines: LineList, - - pub const LineList = LinkedList(TokenIndex); + /// Points to the first doc comment token. API users are expected to iterate over the + /// tokens array, looking for more doc comments, ignoring line comments, and stopping + /// at the first other token. + first_line: TokenIndex, pub fn iterate(self: *const DocComment) Node.Iterator { return .{ .parent_node = &self.base, .index = 0 }; @@ -3026,14 +3026,13 @@ pub const Node = struct { } pub fn firstToken(self: *const DocComment) TokenIndex { - return self.lines.first.?.data; + return self.first_line; } + /// Returns the first doc comment line. Be careful, this may not be the desired behavior, + /// which would require the tokens array. pub fn lastToken(self: *const DocComment) TokenIndex { - var node = self.lines.first.?; - while (true) { - node = node.next orelse return node.data; - } + return self.first_line; } }; diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 818446efd61a4b5ba48014739a76b532686a176c..6d568cf5ec6d90ff138011f1c0f28eb312869ccd 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -58,6 +58,8 @@ const Parser = struct { arena: std.heap.ArenaAllocator, gpa: *Allocator, source: []const u8, + /// TODO: Optimization idea: have this be several arrays of the token fields rather + /// than an array of structs. tokens: []const Token, tok_i: TokenIndex, errors: std.ArrayListUnmanaged(AstError), @@ -367,20 +369,13 @@ const Parser = struct { /// Eat a multiline container doc comment fn parseContainerDocComments(p: *Parser) !?*Node { - var lines = Node.DocComment.LineList{}; - var lines_it: *?*Node.DocComment.LineList.Node = &lines.first; - - while (p.eatToken(.ContainerDocComment)) |line| { - lines_it = try p.llpush(TokenIndex, lines_it, line); + if (p.eatToken(.ContainerDocComment)) |first_line| { + while (p.eatToken(.ContainerDocComment)) |_| {} + const node = try p.arena.allocator.create(Node.DocComment); + node.* = .{ .first_line = first_line }; + return &node.base; } - - if (lines.first == null) return null; - - const node = try p.arena.allocator.create(Node.DocComment); - node.* = .{ - .lines = lines, - }; - return &node.base; + return null; } /// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block @@ -3210,20 +3205,13 @@ const Parser = struct { /// Eat a multiline doc comment fn parseDocComment(p: *Parser) !?*Node.DocComment { - var lines = Node.DocComment.LineList{}; - var lines_it = &lines.first; - - while (p.eatToken(.DocComment)) |line| { - lines_it = try p.llpush(TokenIndex, lines_it, line); + if (p.eatToken(.DocComment)) |first_line| { + while (p.eatToken(.DocComment)) |_| {} + const node = try p.arena.allocator.create(Node.DocComment); + node.* = .{ .first_line = first_line }; + return node; } - - if (lines.first == null) return null; - - const node = try p.arena.allocator.create(Node.DocComment); - node.* = .{ - .lines = lines, - }; - return node; + return null; } fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool { @@ -3234,11 +3222,8 @@ const Parser = struct { fn parseAppendedDocComment(p: *Parser, after_token: TokenIndex) !?*Node.DocComment { const comment_token = p.eatToken(.DocComment) orelse return null; if (p.tokensOnSameLine(after_token, comment_token)) { - var lines = Node.DocComment.LineList{}; - _ = try p.llpush(TokenIndex, &lines.first, comment_token); - const node = try p.arena.allocator.create(Node.DocComment); - node.* = .{ .lines = lines }; + node.* = .{ .first_line = comment_token }; return node; } p.putBackToken(comment_token); diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 236e645d43c29808c3756ec8c44b0c9e04580d82..bc38cf40674f3dc5f1bac60194b741e05aa51fee 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -327,11 +327,18 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, .DocComment => { const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl); - var it = comment.lines.first; - while (it) |node| : (it = node.next) { - try renderToken(tree, stream, node.data, indent, start_col, .Newline); - if (node.next != null) { + const kind = tree.tokens[comment.first_line].id; + try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline); + var tok_i = comment.first_line + 1; + while (true) : (tok_i += 1) { + const tok_id = tree.tokens[tok_i].id; + if (tok_id == kind) { try stream.writeByteNTimes(' ', indent); + try renderToken(tree, stream, tok_i, indent, start_col, .Newline); + } else if (tok_id == .LineComment) { + continue; + } else { + break; } } }, @@ -2428,17 +2435,32 @@ fn renderDocComments( start_col: *usize, ) (@TypeOf(stream).Error || Error)!void { const comment = node.doc_comments orelse return; - var it = comment.lines.first; - const first_token = node.firstToken(); - while (it) |line_token_index_node| : (it = line_token_index_node.next) { - const line_token_index = line_token_index_node.data; - if (line_token_index < first_token) { - try renderToken(tree, stream, line_token_index, indent, start_col, Space.Newline); - try stream.writeByteNTimes(' ', indent); - } else { - try renderToken(tree, stream, line_token_index, indent, start_col, Space.NoComment); - try stream.writeAll("\n"); - try stream.writeByteNTimes(' ', indent); + return renderDocCommentsToken(tree, stream, comment, node.firstToken(), indent, start_col); +} + +fn renderDocCommentsToken( + tree: *ast.Tree, + stream: var, + comment: *ast.Node.DocComment, + first_token: ast.TokenIndex, + indent: usize, + start_col: *usize, +) (@TypeOf(stream).Error || Error)!void { + var tok_i = comment.first_line; + while (true) : (tok_i += 1) { + switch (tree.tokens[tok_i].id) { + .DocComment, .ContainerDocComment => { + if (comment.first_line < first_token) { + try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline); + try stream.writeByteNTimes(' ', indent); + } else { + try renderToken(tree, stream, tok_i, indent, start_col, Space.NoComment); + try stream.writeAll("\n"); + try stream.writeByteNTimes(' ', indent); + } + }, + .LineComment => continue, + else => break, } } } -- 2.54.0