authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-27 17:39:55-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2021-05-27 17:39:55-04:00
log4bf8ec99521f2b8b056475ec703a06a8b149cca5
tree000dbc2db20fa4b2cd248339f30326b2b41e656d
parent7674a8b43d42434a2ed424ca0778bb348e1e3d70
parent80c86ec9608478ade64964e7206487b6178853af
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #8910 from mattbork/parser-small-allocs

Use shared scratch buffer to avoid allocating and freeing many small lists in parse.zig

1 files changed, 85 insertions(+), 92 deletions(-)

lib/std/zig/parse.zig+85-92
......@@ -43,11 +43,13 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!Tree {
4343 .errors = .{},
4444 .nodes = .{},
4545 .extra_data = .{},
46 .scratch = .{},
4647 .tok_i = 0,
4748 };
4849 defer parser.errors.deinit(gpa);
4950 defer parser.nodes.deinit(gpa);
5051 defer parser.extra_data.deinit(gpa);
52 defer parser.scratch.deinit(gpa);
5153
5254 // Empirically, Zig source code has a 2:1 ratio of tokens to AST nodes.
5355 // Make sure at least 1 so we can use appendAssumeCapacity on the root node below.
......@@ -93,17 +95,11 @@ const Parser = struct {
9395 errors: std.ArrayListUnmanaged(AstError),
9496 nodes: ast.NodeList,
9597 extra_data: std.ArrayListUnmanaged(Node.Index),
98 scratch: std.ArrayListUnmanaged(Node.Index),
9699
97100 const SmallSpan = union(enum) {
98101 zero_or_one: Node.Index,
99 multi: []Node.Index,
100
101 fn deinit(self: SmallSpan, gpa: *Allocator) void {
102 switch (self) {
103 .zero_or_one => {},
104 .multi => |list| gpa.free(list),
105 }
106 }
102 multi: Node.SubRange,
107103 };
108104
109105 const Members = struct {
......@@ -204,8 +200,8 @@ const Parser = struct {
204200 /// /
205201 /// TopLevelComptime <- KEYWORD_comptime BlockExpr
206202 fn parseContainerMembers(p: *Parser) !Members {
207 var list = std.ArrayList(Node.Index).init(p.gpa);
208 defer list.deinit();
203 const scratch_top = p.scratch.items.len;
204 defer p.scratch.shrinkRetainingCapacity(scratch_top);
209205
210206 var field_state: union(enum) {
211207 /// No fields have been seen.
......@@ -233,7 +229,7 @@ const Parser = struct {
233229 if (field_state == .seen) {
234230 field_state = .{ .end = test_decl_node };
235231 }
236 try list.append(test_decl_node);
232 try p.scratch.append(p.gpa, test_decl_node);
237233 }
238234 trailing = false;
239235 },
......@@ -254,7 +250,7 @@ const Parser = struct {
254250 field_state = .err;
255251 },
256252 }
257 try list.append(container_field);
253 try p.scratch.append(p.gpa, container_field);
258254 switch (p.token_tags[p.tok_i]) {
259255 .comma => {
260256 p.tok_i += 1;
......@@ -294,7 +290,7 @@ const Parser = struct {
294290 if (field_state == .seen) {
295291 field_state = .{ .end = comptime_node };
296292 }
297 try list.append(comptime_node);
293 try p.scratch.append(p.gpa, comptime_node);
298294 }
299295 trailing = false;
300296 },
......@@ -310,7 +306,7 @@ const Parser = struct {
310306 if (field_state == .seen) {
311307 field_state = .{ .end = top_level_decl };
312308 }
313 try list.append(top_level_decl);
309 try p.scratch.append(p.gpa, top_level_decl);
314310 }
315311 trailing = p.token_tags[p.tok_i - 1] == .semicolon;
316312 },
......@@ -320,7 +316,7 @@ const Parser = struct {
320316 if (field_state == .seen) {
321317 field_state = .{ .end = node };
322318 }
323 try list.append(node);
319 try p.scratch.append(p.gpa, node);
324320 }
325321 trailing = p.token_tags[p.tok_i - 1] == .semicolon;
326322 },
......@@ -338,7 +334,7 @@ const Parser = struct {
338334 if (field_state == .seen) {
339335 field_state = .{ .end = top_level_decl };
340336 }
341 try list.append(top_level_decl);
337 try p.scratch.append(p.gpa, top_level_decl);
342338 }
343339 trailing = p.token_tags[p.tok_i - 1] == .semicolon;
344340 },
......@@ -357,7 +353,7 @@ const Parser = struct {
357353 field_state = .err;
358354 },
359355 }
360 try list.append(container_field);
356 try p.scratch.append(p.gpa, container_field);
361357 switch (p.token_tags[p.tok_i]) {
362358 .comma => {
363359 p.tok_i += 1;
......@@ -393,7 +389,8 @@ const Parser = struct {
393389 }
394390 }
395391
396 switch (list.items.len) {
392 const items = p.scratch.items[scratch_top..];
393 switch (items.len) {
397394 0 => return Members{
398395 .len = 0,
399396 .lhs = 0,
......@@ -402,20 +399,20 @@ const Parser = struct {
402399 },
403400 1 => return Members{
404401 .len = 1,
405 .lhs = list.items[0],
402 .lhs = items[0],
406403 .rhs = 0,
407404 .trailing = trailing,
408405 },
409406 2 => return Members{
410407 .len = 2,
411 .lhs = list.items[0],
412 .rhs = list.items[1],
408 .lhs = items[0],
409 .rhs = items[1],
413410 .trailing = trailing,
414411 },
415412 else => {
416 const span = try p.listToSpan(list.items);
413 const span = try p.listToSpan(items);
417414 return Members{
418 .len = list.items.len,
415 .len = items.len,
419416 .lhs = span.start,
420417 .rhs = span.end,
421418 .trailing = trailing,
......@@ -648,7 +645,6 @@ const Parser = struct {
648645
649646 _ = p.eatToken(.identifier);
650647 const params = try p.parseParamDeclList();
651 defer params.deinit(p.gpa);
652648 const align_expr = try p.parseByteAlign();
653649 const section_expr = try p.parseLinkSection();
654650 const callconv_expr = try p.parseCallconv();
......@@ -671,8 +667,7 @@ const Parser = struct {
671667 .rhs = return_type_expr,
672668 },
673669 }),
674 .multi => |list| {
675 const span = try p.listToSpan(list);
670 .multi => |span| {
676671 return p.setNode(fn_proto_index, .{
677672 .tag = .fn_proto_multi,
678673 .main_token = fn_token,
......@@ -701,8 +696,7 @@ const Parser = struct {
701696 .rhs = return_type_expr,
702697 },
703698 }),
704 .multi => |list| {
705 const span = try p.listToSpan(list);
699 .multi => |span| {
706700 return p.setNode(fn_proto_index, .{
707701 .tag = .fn_proto,
708702 .main_token = fn_token,
......@@ -1894,20 +1888,20 @@ const Parser = struct {
18941888 });
18951889 }
18961890
1897 var statements = std.ArrayList(Node.Index).init(p.gpa);
1898 defer statements.deinit();
1891 const scratch_top = p.scratch.items.len;
1892 defer p.scratch.shrinkRetainingCapacity(scratch_top);
18991893
1900 try statements.appendSlice(&.{ stmt_one, stmt_two });
1894 try p.scratch.appendSlice(p.gpa, &.{ stmt_one, stmt_two });
19011895
19021896 while (true) {
19031897 const statement = try p.expectStatementRecoverable();
19041898 if (statement == 0) break;
1905 try statements.append(statement);
1899 try p.scratch.append(p.gpa, statement);
19061900 if (p.token_tags[p.tok_i] == .r_brace) break;
19071901 }
19081902 _ = try p.expectToken(.r_brace);
19091903 const semicolon = p.token_tags[p.tok_i - 2] == .semicolon;
1910 const statements_span = try p.listToSpan(statements.items);
1904 const statements_span = try p.listToSpan(p.scratch.items[scratch_top..]);
19111905 return p.addNode(.{
19121906 .tag = if (semicolon) .block_semicolon else .block,
19131907 .main_token = lbrace,
......@@ -2041,14 +2035,14 @@ const Parser = struct {
20412035 });
20422036 }
20432037
2044 var init_list = std.ArrayList(Node.Index).init(p.gpa);
2045 defer init_list.deinit();
2038 const scratch_top = p.scratch.items.len;
2039 defer p.scratch.shrinkRetainingCapacity(scratch_top);
20462040
2047 try init_list.append(field_init);
2041 try p.scratch.append(p.gpa, field_init);
20482042
20492043 while (true) {
20502044 const next = try p.expectFieldInit();
2051 try init_list.append(next);
2045 try p.scratch.append(p.gpa, next);
20522046
20532047 switch (p.token_tags[p.nextToken()]) {
20542048 .comma => {
......@@ -2068,7 +2062,7 @@ const Parser = struct {
20682062 },
20692063 }
20702064 }
2071 const span = try p.listToSpan(init_list.items);
2065 const span = try p.listToSpan(p.scratch.items[scratch_top..]);
20722066 return p.addNode(.{
20732067 .tag = if (p.token_tags[p.tok_i - 2] == .comma) .struct_init_comma else .struct_init,
20742068 .main_token = lbrace,
......@@ -2098,22 +2092,22 @@ const Parser = struct {
20982092 try p.warnExpected(.comma);
20992093 }
21002094
2101 var init_list = std.ArrayList(Node.Index).init(p.gpa);
2102 defer init_list.deinit();
2095 const scratch_top = p.scratch.items.len;
2096 defer p.scratch.shrinkRetainingCapacity(scratch_top);
21032097
2104 try init_list.append(elem_init);
2098 try p.scratch.append(p.gpa, elem_init);
21052099
21062100 var trailing_comma = true;
21072101 var next = try p.parseExpr();
21082102 while (next != 0) : (next = try p.parseExpr()) {
2109 try init_list.append(next);
2103 try p.scratch.append(p.gpa, next);
21102104 if (p.eatToken(.comma) == null) {
21112105 trailing_comma = false;
21122106 break;
21132107 }
21142108 }
21152109 _ = try p.expectToken(.r_brace);
2116 const span = try p.listToSpan(init_list.items);
2110 const span = try p.listToSpan(p.scratch.items[scratch_top..]);
21172111 return p.addNode(.{
21182112 .tag = if (trailing_comma) .array_init_comma else .array_init,
21192113 .main_token = lbrace,
......@@ -2188,18 +2182,18 @@ const Parser = struct {
21882182 try p.warnExpected(.comma);
21892183 }
21902184
2191 var param_list = std.ArrayList(Node.Index).init(p.gpa);
2192 defer param_list.deinit();
2185 const scratch_top = p.scratch.items.len;
2186 defer p.scratch.shrinkRetainingCapacity(scratch_top);
21932187
2194 try param_list.append(param_one);
2188 try p.scratch.append(p.gpa, param_one);
21952189
21962190 while (true) {
21972191 const next = try p.expectExpr();
2198 try param_list.append(next);
2192 try p.scratch.append(p.gpa, next);
21992193 switch (p.token_tags[p.nextToken()]) {
22002194 .comma => {
22012195 if (p.eatToken(.r_paren)) |_| {
2202 const span = try p.listToSpan(param_list.items);
2196 const span = try p.listToSpan(p.scratch.items[scratch_top..]);
22032197 return p.addNode(.{
22042198 .tag = .async_call_comma,
22052199 .main_token = lparen,
......@@ -2216,7 +2210,7 @@ const Parser = struct {
22162210 }
22172211 },
22182212 .r_paren => {
2219 const span = try p.listToSpan(param_list.items);
2213 const span = try p.listToSpan(p.scratch.items[scratch_top..]);
22202214 return p.addNode(.{
22212215 .tag = .async_call,
22222216 .main_token = lparen,
......@@ -2277,18 +2271,18 @@ const Parser = struct {
22772271 try p.warnExpected(.comma);
22782272 }
22792273
2280 var param_list = std.ArrayList(Node.Index).init(p.gpa);
2281 defer param_list.deinit();
2274 const scratch_top = p.scratch.items.len;
2275 defer p.scratch.shrinkRetainingCapacity(scratch_top);
22822276
2283 try param_list.append(param_one);
2277 try p.scratch.append(p.gpa, param_one);
22842278
22852279 while (true) {
22862280 const next = try p.expectExpr();
2287 try param_list.append(next);
2281 try p.scratch.append(p.gpa, next);
22882282 switch (p.token_tags[p.nextToken()]) {
22892283 .comma => {
22902284 if (p.eatToken(.r_paren)) |_| {
2291 const span = try p.listToSpan(param_list.items);
2285 const span = try p.listToSpan(p.scratch.items[scratch_top..]);
22922286 break :res try p.addNode(.{
22932287 .tag = .call_comma,
22942288 .main_token = lparen,
......@@ -2305,7 +2299,7 @@ const Parser = struct {
23052299 }
23062300 },
23072301 .r_paren => {
2308 const span = try p.listToSpan(param_list.items);
2302 const span = try p.listToSpan(p.scratch.items[scratch_top..]);
23092303 break :res try p.addNode(.{
23102304 .tag = .call,
23112305 .main_token = lparen,
......@@ -2602,15 +2596,15 @@ const Parser = struct {
26022596 if (comma_two == null) {
26032597 try p.warnExpected(.comma);
26042598 }
2605 var init_list = std.ArrayList(Node.Index).init(p.gpa);
2606 defer init_list.deinit();
2599 const scratch_top = p.scratch.items.len;
2600 defer p.scratch.shrinkRetainingCapacity(scratch_top);
26072601
2608 try init_list.appendSlice(&.{ field_init_one, field_init_two });
2602 try p.scratch.appendSlice(p.gpa, &.{ field_init_one, field_init_two });
26092603
26102604 while (true) {
26112605 const next = try p.expectFieldInit();
26122606 assert(next != 0);
2613 try init_list.append(next);
2607 try p.scratch.append(p.gpa, next);
26142608 switch (p.token_tags[p.nextToken()]) {
26152609 .comma => {
26162610 if (p.eatToken(.r_brace)) |_| break;
......@@ -2627,7 +2621,7 @@ const Parser = struct {
26272621 },
26282622 }
26292623 }
2630 const span = try p.listToSpan(init_list.items);
2624 const span = try p.listToSpan(p.scratch.items[scratch_top..]);
26312625 const trailing_comma = p.token_tags[p.tok_i - 2] == .comma;
26322626 return p.addNode(.{
26332627 .tag = if (trailing_comma) .struct_init_dot_comma else .struct_init_dot,
......@@ -2669,15 +2663,15 @@ const Parser = struct {
26692663 if (comma_two == null) {
26702664 try p.warnExpected(.comma);
26712665 }
2672 var init_list = std.ArrayList(Node.Index).init(p.gpa);
2673 defer init_list.deinit();
2666 const scratch_top = p.scratch.items.len;
2667 defer p.scratch.shrinkRetainingCapacity(scratch_top);
26742668
2675 try init_list.appendSlice(&.{ elem_init_one, elem_init_two });
2669 try p.scratch.appendSlice(p.gpa, &.{ elem_init_one, elem_init_two });
26762670
26772671 while (true) {
26782672 const next = try p.expectExpr();
26792673 if (next == 0) break;
2680 try init_list.append(next);
2674 try p.scratch.append(p.gpa, next);
26812675 switch (p.token_tags[p.nextToken()]) {
26822676 .comma => {
26832677 if (p.eatToken(.r_brace)) |_| break;
......@@ -2694,7 +2688,7 @@ const Parser = struct {
26942688 },
26952689 }
26962690 }
2697 const span = try p.listToSpan(init_list.items);
2691 const span = try p.listToSpan(p.scratch.items[scratch_top..]);
26982692 return p.addNode(.{
26992693 .tag = if (p.token_tags[p.tok_i - 2] == .comma) .array_init_dot_comma else .array_init_dot,
27002694 .main_token = lbrace,
......@@ -2924,13 +2918,13 @@ const Parser = struct {
29242918
29252919 _ = try p.expectToken(.colon);
29262920
2927 var list = std.ArrayList(Node.Index).init(p.gpa);
2928 defer list.deinit();
2921 const scratch_top = p.scratch.items.len;
2922 defer p.scratch.shrinkRetainingCapacity(scratch_top);
29292923
29302924 while (true) {
29312925 const output_item = try p.parseAsmOutputItem();
29322926 if (output_item == 0) break;
2933 try list.append(output_item);
2927 try p.scratch.append(p.gpa, output_item);
29342928 switch (p.token_tags[p.tok_i]) {
29352929 .comma => p.tok_i += 1,
29362930 .colon, .r_paren, .r_brace, .r_bracket => break, // All possible delimiters.
......@@ -2945,7 +2939,7 @@ const Parser = struct {
29452939 while (true) {
29462940 const input_item = try p.parseAsmInputItem();
29472941 if (input_item == 0) break;
2948 try list.append(input_item);
2942 try p.scratch.append(p.gpa, input_item);
29492943 switch (p.token_tags[p.tok_i]) {
29502944 .comma => p.tok_i += 1,
29512945 .colon, .r_paren, .r_brace, .r_bracket => break, // All possible delimiters.
......@@ -2971,7 +2965,7 @@ const Parser = struct {
29712965 }
29722966 }
29732967 const rparen = try p.expectToken(.r_paren);
2974 const span = try p.listToSpan(list.items);
2968 const span = try p.listToSpan(p.scratch.items[scratch_top..]);
29752969 return p.addNode(.{
29762970 .tag = .@"asm",
29772971 .main_token = asm_token,
......@@ -3192,16 +3186,16 @@ const Parser = struct {
31923186 });
31933187 }
31943188
3195 var list = std.ArrayList(Node.Index).init(p.gpa);
3196 defer list.deinit();
3189 const scratch_top = p.scratch.items.len;
3190 defer p.scratch.shrinkRetainingCapacity(scratch_top);
31973191
3198 try list.append(first_item);
3192 try p.scratch.append(p.gpa, first_item);
31993193 while (p.eatToken(.comma)) |_| {
32003194 const next_item = try p.parseSwitchItem();
32013195 if (next_item == 0) break;
3202 try list.append(next_item);
3196 try p.scratch.append(p.gpa, next_item);
32033197 }
3204 const span = try p.listToSpan(list.items);
3198 const span = try p.listToSpan(p.scratch.items[scratch_top..]);
32053199 const arrow_token = try p.expectToken(.equal_angle_bracket_right);
32063200 _ = try p.parsePtrPayload();
32073201 return p.addNode(.{
......@@ -3602,15 +3596,14 @@ const Parser = struct {
36023596 if (param != 0) break param;
36033597 } else unreachable;
36043598
3605 var list = std.ArrayList(Node.Index).init(p.gpa);
3606 defer list.deinit();
3607
3608 try list.appendSlice(&.{ param_one, param_two });
3599 const scratch_top = p.scratch.items.len;
3600 defer p.scratch.shrinkRetainingCapacity(scratch_top);
3601 try p.scratch.appendSlice(p.gpa, &.{ param_one, param_two });
36093602
36103603 while (true) {
36113604 switch (p.token_tags[p.nextToken()]) {
36123605 .comma => {},
3613 .r_paren => return SmallSpan{ .multi = list.toOwnedSlice() },
3606 .r_paren => return SmallSpan{ .multi = try p.listToSpan(p.scratch.items[scratch_top..]) },
36143607 .colon, .r_brace, .r_bracket => {
36153608 p.tok_i -= 1;
36163609 return p.failExpected(.r_paren);
......@@ -3623,10 +3616,10 @@ const Parser = struct {
36233616 },
36243617 }
36253618 if (p.eatToken(.r_paren)) |_| {
3626 return SmallSpan{ .multi = list.toOwnedSlice() };
3619 return SmallSpan{ .multi = try p.listToSpan(p.scratch.items[scratch_top..]) };
36273620 }
36283621 const param = try p.expectParamDecl();
3629 if (param != 0) try list.append(param);
3622 if (param != 0) try p.scratch.append(p.gpa, param);
36303623 }
36313624 }
36323625
......@@ -3635,14 +3628,14 @@ const Parser = struct {
36353628 fn ListParseFn(comptime nodeParseFn: anytype) (fn (p: *Parser) Error!Node.SubRange) {
36363629 return struct {
36373630 pub fn parse(p: *Parser) Error!Node.SubRange {
3638 var list = std.ArrayList(Node.Index).init(p.gpa);
3639 defer list.deinit();
3631 const scratch_top = p.scratch.items.len;
3632 defer p.scratch.shrinkRetainingCapacity(scratch_top);
36403633
36413634 while (true) {
36423635 const item = try nodeParseFn(p);
36433636 if (item == 0) break;
36443637
3645 try list.append(item);
3638 try p.scratch.append(p.gpa, item);
36463639
36473640 switch (p.token_tags[p.tok_i]) {
36483641 .comma => p.tok_i += 1,
......@@ -3655,7 +3648,7 @@ const Parser = struct {
36553648 },
36563649 }
36573650 }
3658 return p.listToSpan(list.items);
3651 return p.listToSpan(p.scratch.items[scratch_top..]);
36593652 }
36603653 }.parse;
36613654 }
......@@ -3746,18 +3739,18 @@ const Parser = struct {
37463739 },
37473740 }
37483741
3749 var list = std.ArrayList(Node.Index).init(p.gpa);
3750 defer list.deinit();
3742 const scratch_top = p.scratch.items.len;
3743 defer p.scratch.shrinkRetainingCapacity(scratch_top);
37513744
3752 try list.appendSlice(&.{ param_one, param_two });
3745 try p.scratch.appendSlice(p.gpa, &.{ param_one, param_two });
37533746
37543747 while (true) {
37553748 const param = try p.expectExpr();
3756 try list.append(param);
3749 try p.scratch.append(p.gpa, param);
37573750 switch (p.token_tags[p.nextToken()]) {
37583751 .comma => {
37593752 if (p.eatToken(.r_paren)) |_| {
3760 const params = try p.listToSpan(list.items);
3753 const params = try p.listToSpan(p.scratch.items[scratch_top..]);
37613754 return p.addNode(.{
37623755 .tag = .builtin_call_comma,
37633756 .main_token = builtin_token,
......@@ -3770,7 +3763,7 @@ const Parser = struct {
37703763 continue;
37713764 },
37723765 .r_paren => {
3773 const params = try p.listToSpan(list.items);
3766 const params = try p.listToSpan(p.scratch.items[scratch_top..]);
37743767 return p.addNode(.{
37753768 .tag = .builtin_call,
37763769 .main_token = builtin_token,