authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-11-21 23:25:16+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-11-23 12:13:39+02:00
log6fb689e97a4ec05fd97dfa05cc203e9bcb7cffa1
tree85f80dde57f698b4e84be75756a33eaf19c5d276
parent32ce2f91a92c23d46c6836a6dd68ae0f08bb04c5

parser: allow unnamed fields in structs

4 files changed, 209 insertions(+), 151 deletions(-)

lib/std/zig/Ast.zig+21-9
......@@ -559,6 +559,7 @@ pub fn firstToken(tree: Ast, node: Node.Index) TokenIndex {
559559 .container_field,
560560 => {
561561 const name_token = main_tokens[n];
562 if (token_tags[name_token + 1] != .colon) return name_token - end_offset;
562563 if (name_token > 0 and token_tags[name_token - 1] == .keyword_comptime) {
563564 end_offset += 1;
564565 }
......@@ -1320,33 +1321,39 @@ pub fn containerField(tree: Ast, node: Node.Index) full.ContainerField {
13201321 assert(tree.nodes.items(.tag)[node] == .container_field);
13211322 const data = tree.nodes.items(.data)[node];
13221323 const extra = tree.extraData(data.rhs, Node.ContainerField);
1324 const main_token = tree.nodes.items(.main_token)[node];
13231325 return tree.fullContainerField(.{
1324 .name_token = tree.nodes.items(.main_token)[node],
1326 .main_token = main_token,
13251327 .type_expr = data.lhs,
13261328 .value_expr = extra.value_expr,
13271329 .align_expr = extra.align_expr,
1330 .tuple_like = tree.tokens.items(.tag)[main_token + 1] != .colon,
13281331 });
13291332}
13301333
13311334pub fn containerFieldInit(tree: Ast, node: Node.Index) full.ContainerField {
13321335 assert(tree.nodes.items(.tag)[node] == .container_field_init);
13331336 const data = tree.nodes.items(.data)[node];
1337 const main_token = tree.nodes.items(.main_token)[node];
13341338 return tree.fullContainerField(.{
1335 .name_token = tree.nodes.items(.main_token)[node],
1339 .main_token = main_token,
13361340 .type_expr = data.lhs,
13371341 .value_expr = data.rhs,
13381342 .align_expr = 0,
1343 .tuple_like = tree.tokens.items(.tag)[main_token + 1] != .colon,
13391344 });
13401345}
13411346
13421347pub fn containerFieldAlign(tree: Ast, node: Node.Index) full.ContainerField {
13431348 assert(tree.nodes.items(.tag)[node] == .container_field_align);
13441349 const data = tree.nodes.items(.data)[node];
1350 const main_token = tree.nodes.items(.main_token)[node];
13451351 return tree.fullContainerField(.{
1346 .name_token = tree.nodes.items(.main_token)[node],
1352 .main_token = main_token,
13471353 .type_expr = data.lhs,
13481354 .value_expr = 0,
13491355 .align_expr = data.rhs,
1356 .tuple_like = tree.tokens.items(.tag)[main_token + 1] != .colon,
13501357 });
13511358}
13521359
......@@ -1944,10 +1951,14 @@ fn fullContainerField(tree: Ast, info: full.ContainerField.Components) full.Cont
19441951 .ast = info,
19451952 .comptime_token = null,
19461953 };
1947 // comptime name: type = init,
1948 // ^
1949 if (info.name_token > 0 and token_tags[info.name_token - 1] == .keyword_comptime) {
1950 result.comptime_token = info.name_token - 1;
1954 if (token_tags[info.main_token] == .keyword_comptime) {
1955 // comptime type = init,
1956 // ^
1957 result.comptime_token = info.main_token;
1958 } else if (info.main_token > 0 and token_tags[info.main_token - 1] == .keyword_comptime) {
1959 // comptime name: type = init,
1960 // ^
1961 result.comptime_token = info.main_token - 1;
19511962 }
19521963 return result;
19531964}
......@@ -2256,14 +2267,15 @@ pub const full = struct {
22562267 ast: Components,
22572268
22582269 pub const Components = struct {
2259 name_token: TokenIndex,
2270 main_token: TokenIndex,
22602271 type_expr: Node.Index,
22612272 value_expr: Node.Index,
22622273 align_expr: Node.Index,
2274 tuple_like: bool,
22632275 };
22642276
22652277 pub fn firstToken(cf: ContainerField) TokenIndex {
2266 return cf.comptime_token orelse cf.ast.name_token;
2278 return cf.comptime_token orelse cf.ast.main_token;
22672279 }
22682280 };
22692281
lib/std/zig/parse.zig+107-113
......@@ -272,53 +272,6 @@ const Parser = struct {
272272 trailing = false;
273273 },
274274 .keyword_comptime => switch (p.token_tags[p.tok_i + 1]) {
275 .identifier => {
276 p.tok_i += 1;
277 const identifier = p.tok_i;
278 defer last_field = identifier;
279 const container_field = try p.expectContainerFieldRecoverable();
280 if (container_field != 0) {
281 switch (field_state) {
282 .none => field_state = .seen,
283 .err, .seen => {},
284 .end => |node| {
285 try p.warnMsg(.{
286 .tag = .decl_between_fields,
287 .token = p.nodes.items(.main_token)[node],
288 });
289 try p.warnMsg(.{
290 .tag = .previous_field,
291 .is_note = true,
292 .token = last_field,
293 });
294 try p.warnMsg(.{
295 .tag = .next_field,
296 .is_note = true,
297 .token = identifier,
298 });
299 // Continue parsing; error will be reported later.
300 field_state = .err;
301 },
302 }
303 try p.scratch.append(p.gpa, container_field);
304 switch (p.token_tags[p.tok_i]) {
305 .comma => {
306 p.tok_i += 1;
307 trailing = true;
308 continue;
309 },
310 .r_brace, .eof => {
311 trailing = false;
312 break;
313 },
314 else => {},
315 }
316 // There is not allowed to be a decl after a field with no comma.
317 // Report error but recover parser.
318 try p.warn(.expected_comma_after_field);
319 p.findNextContainerMember();
320 }
321 },
322275 .l_brace => {
323276 if (doc_comment) |some| {
324277 try p.warnMsg(.{ .tag = .test_doc_comment, .token = some });
......@@ -349,53 +302,15 @@ const Parser = struct {
349302 },
350303 else => {
351304 p.tok_i += 1;
352 try p.warn(.expected_block_or_field);
353 },
354 },
355 .keyword_pub => {
356 p.tok_i += 1;
357 const top_level_decl = try p.expectTopLevelDeclRecoverable();
358 if (top_level_decl != 0) {
359 if (field_state == .seen) {
360 field_state = .{ .end = top_level_decl };
361 }
362 try p.scratch.append(p.gpa, top_level_decl);
363 }
364 trailing = p.token_tags[p.tok_i - 1] == .semicolon;
365 },
366 .keyword_usingnamespace => {
367 const node = try p.expectUsingNamespaceRecoverable();
368 if (node != 0) {
369 if (field_state == .seen) {
370 field_state = .{ .end = node };
371 }
372 try p.scratch.append(p.gpa, node);
373 }
374 trailing = p.token_tags[p.tok_i - 1] == .semicolon;
375 },
376 .keyword_const,
377 .keyword_var,
378 .keyword_threadlocal,
379 .keyword_export,
380 .keyword_extern,
381 .keyword_inline,
382 .keyword_noinline,
383 .keyword_fn,
384 => {
385 const top_level_decl = try p.expectTopLevelDeclRecoverable();
386 if (top_level_decl != 0) {
387 if (field_state == .seen) {
388 field_state = .{ .end = top_level_decl };
389 }
390 try p.scratch.append(p.gpa, top_level_decl);
391 }
392 trailing = p.token_tags[p.tok_i - 1] == .semicolon;
393 },
394 .identifier => {
395 const identifier = p.tok_i;
396 defer last_field = identifier;
397 const container_field = try p.expectContainerFieldRecoverable();
398 if (container_field != 0) {
305 const identifier = p.tok_i;
306 defer last_field = identifier;
307 const container_field = p.expectContainerField() catch |err| switch (err) {
308 error.OutOfMemory => return error.OutOfMemory,
309 error.ParseError => {
310 p.findNextContainerMember();
311 continue;
312 },
313 };
399314 switch (field_state) {
400315 .none => field_state = .seen,
401316 .err, .seen => {},
......@@ -435,7 +350,46 @@ const Parser = struct {
435350 // Report error but recover parser.
436351 try p.warn(.expected_comma_after_field);
437352 p.findNextContainerMember();
353 },
354 },
355 .keyword_pub => {
356 p.tok_i += 1;
357 const top_level_decl = try p.expectTopLevelDeclRecoverable();
358 if (top_level_decl != 0) {
359 if (field_state == .seen) {
360 field_state = .{ .end = top_level_decl };
361 }
362 try p.scratch.append(p.gpa, top_level_decl);
363 }
364 trailing = p.token_tags[p.tok_i - 1] == .semicolon;
365 },
366 .keyword_usingnamespace => {
367 const node = try p.expectUsingNamespaceRecoverable();
368 if (node != 0) {
369 if (field_state == .seen) {
370 field_state = .{ .end = node };
371 }
372 try p.scratch.append(p.gpa, node);
373 }
374 trailing = p.token_tags[p.tok_i - 1] == .semicolon;
375 },
376 .keyword_const,
377 .keyword_var,
378 .keyword_threadlocal,
379 .keyword_export,
380 .keyword_extern,
381 .keyword_inline,
382 .keyword_noinline,
383 .keyword_fn,
384 => {
385 const top_level_decl = try p.expectTopLevelDeclRecoverable();
386 if (top_level_decl != 0) {
387 if (field_state == .seen) {
388 field_state = .{ .end = top_level_decl };
389 }
390 try p.scratch.append(p.gpa, top_level_decl);
438391 }
392 trailing = p.token_tags[p.tok_i - 1] == .semicolon;
439393 },
440394 .eof, .r_brace => {
441395 if (doc_comment) |tok| {
......@@ -451,11 +405,57 @@ const Parser = struct {
451405 error.OutOfMemory => return error.OutOfMemory,
452406 error.ParseError => false,
453407 };
454 if (!c_container) {
455 try p.warn(.expected_container_members);
456 // This was likely not supposed to end yet; try to find the next declaration.
457 p.findNextContainerMember();
408 if (c_container) continue;
409
410 const identifier = p.tok_i;
411 defer last_field = identifier;
412 const container_field = p.expectContainerField() catch |err| switch (err) {
413 error.OutOfMemory => return error.OutOfMemory,
414 error.ParseError => {
415 p.findNextContainerMember();
416 continue;
417 },
418 };
419 switch (field_state) {
420 .none => field_state = .seen,
421 .err, .seen => {},
422 .end => |node| {
423 try p.warnMsg(.{
424 .tag = .decl_between_fields,
425 .token = p.nodes.items(.main_token)[node],
426 });
427 try p.warnMsg(.{
428 .tag = .previous_field,
429 .is_note = true,
430 .token = last_field,
431 });
432 try p.warnMsg(.{
433 .tag = .next_field,
434 .is_note = true,
435 .token = identifier,
436 });
437 // Continue parsing; error will be reported later.
438 field_state = .err;
439 },
458440 }
441 try p.scratch.append(p.gpa, container_field);
442 switch (p.token_tags[p.tok_i]) {
443 .comma => {
444 p.tok_i += 1;
445 trailing = true;
446 continue;
447 },
448 .r_brace, .eof => {
449 trailing = false;
450 break;
451 },
452 else => {},
453 }
454 // There is not allowed to be a decl after a field with no comma.
455 // Report error but recover parser.
456 try p.warn(.expected_comma_after_field);
457 p.findNextContainerMember();
458 continue;
459459 },
460460 }
461461 }
......@@ -875,12 +875,16 @@ const Parser = struct {
875875
876876 /// ContainerField <- KEYWORD_comptime? IDENTIFIER (COLON TypeExpr ByteAlign?)? (EQUAL Expr)?
877877 fn expectContainerField(p: *Parser) !Node.Index {
878 var main_token = p.tok_i;
878879 _ = p.eatToken(.keyword_comptime);
879 const name_token = p.assertToken(.identifier);
880 const tuple_like = p.token_tags[p.tok_i] != .identifier or p.token_tags[p.tok_i + 1] != .colon;
881 if (!tuple_like) {
882 main_token = p.assertToken(.identifier);
883 }
880884
881885 var align_expr: Node.Index = 0;
882886 var type_expr: Node.Index = 0;
883 if (p.eatToken(.colon)) |_| {
887 if (p.eatToken(.colon) != null or tuple_like) |_| {
884888 type_expr = try p.expectTypeExpr();
885889 align_expr = try p.parseByteAlign();
886890 }
......@@ -890,7 +894,7 @@ const Parser = struct {
890894 if (align_expr == 0) {
891895 return p.addNode(.{
892896 .tag = .container_field_init,
893 .main_token = name_token,
897 .main_token = main_token,
894898 .data = .{
895899 .lhs = type_expr,
896900 .rhs = value_expr,
......@@ -899,7 +903,7 @@ const Parser = struct {
899903 } else if (value_expr == 0) {
900904 return p.addNode(.{
901905 .tag = .container_field_align,
902 .main_token = name_token,
906 .main_token = main_token,
903907 .data = .{
904908 .lhs = type_expr,
905909 .rhs = align_expr,
......@@ -908,7 +912,7 @@ const Parser = struct {
908912 } else {
909913 return p.addNode(.{
910914 .tag = .container_field,
911 .main_token = name_token,
915 .main_token = main_token,
912916 .data = .{
913917 .lhs = type_expr,
914918 .rhs = try p.addExtra(Node.ContainerField{
......@@ -920,16 +924,6 @@ const Parser = struct {
920924 }
921925 }
922926
923 fn expectContainerFieldRecoverable(p: *Parser) error{OutOfMemory}!Node.Index {
924 return p.expectContainerField() catch |err| switch (err) {
925 error.OutOfMemory => return error.OutOfMemory,
926 error.ParseError => {
927 p.findNextContainerMember();
928 return null_node;
929 },
930 };
931 }
932
933927 /// Statement
934928 /// <- KEYWORD_comptime? VarDecl
935929 /// / KEYWORD_comptime BlockExprStatement
lib/std/zig/parser_test.zig+15-11
......@@ -1,3 +1,15 @@
1test "zig fmt: tuple struct" {
2 try testCanonical(
3 \\const T = struct {
4 \\ comptime u32,
5 \\ *u32 = 1,
6 \\ // needs to be wrapped in parentheses to not be parsed as a function decl
7 \\ (fn () void) align(1),
8 \\};
9 \\
10 );
11}
12
113test "zig fmt: preserves clobbers in inline asm with stray comma" {
214 try testCanonical(
315 \\fn foo() void {
......@@ -265,14 +277,6 @@ test "zig fmt: decl between fields" {
265277 });
266278}
267279
268test "zig fmt: eof after missing comma" {
269 try testError(
270 \\foo()
271 , &[_]Error{
272 .expected_comma_after_field,
273 });
274}
275
276280test "zig fmt: errdefer with payload" {
277281 try testCanonical(
278282 \\pub fn main() anyerror!void {
......@@ -5732,8 +5736,8 @@ test "recovery: missing semicolon" {
57325736test "recovery: invalid container members" {
57335737 try testError(
57345738 \\usingnamespace;
5735 \\foo+
5736 \\bar@,
5739 \\@foo()+
5740 \\@bar()@,
57375741 \\while (a == 2) { test "" {}}
57385742 \\test "" {
57395743 \\ a & b
......@@ -5741,7 +5745,7 @@ test "recovery: invalid container members" {
57415745 , &[_]Error{
57425746 .expected_expr,
57435747 .expected_comma_after_field,
5744 .expected_container_members,
5748 .expected_type_expr,
57455749 .expected_semi_after_stmt,
57465750 });
57475751}
lib/std/zig/render.zig+66-18
......@@ -40,14 +40,34 @@ pub fn renderTree(buffer: *std.ArrayList(u8), tree: Ast) Error!void {
4040/// Render all members in the given slice, keeping empty lines where appropriate
4141fn renderMembers(gpa: Allocator, ais: *Ais, tree: Ast, members: []const Ast.Node.Index) Error!void {
4242 if (members.len == 0) return;
43 try renderMember(gpa, ais, tree, members[0], .newline);
43 var any_non_tuple_like_fields = false;
44 for (members) |member| {
45 const tuple_like = switch (tree.nodes.items(.tag)[member]) {
46 .container_field_init => tree.containerFieldInit(member).ast.tuple_like,
47 .container_field_align => tree.containerFieldAlign(member).ast.tuple_like,
48 .container_field => tree.containerField(member).ast.tuple_like,
49 else => continue,
50 };
51 if (!tuple_like) {
52 any_non_tuple_like_fields = true;
53 break;
54 }
55 }
56 try renderMember(gpa, ais, tree, members[0], any_non_tuple_like_fields, .newline);
4457 for (members[1..]) |member| {
4558 try renderExtraNewline(ais, tree, member);
46 try renderMember(gpa, ais, tree, member, .newline);
59 try renderMember(gpa, ais, tree, member, any_non_tuple_like_fields, .newline);
4760 }
4861}
4962
50fn renderMember(gpa: Allocator, ais: *Ais, tree: Ast, decl: Ast.Node.Index, space: Space) Error!void {
63fn renderMember(
64 gpa: Allocator,
65 ais: *Ais,
66 tree: Ast,
67 decl: Ast.Node.Index,
68 any_non_tuple_like_fields: bool,
69 space: Space,
70) Error!void {
5171 const token_tags = tree.tokens.items(.tag);
5272 const main_tokens = tree.nodes.items(.main_token);
5373 const datas = tree.nodes.items(.data);
......@@ -161,9 +181,9 @@ fn renderMember(gpa: Allocator, ais: *Ais, tree: Ast, decl: Ast.Node.Index, spac
161181 try renderExpression(gpa, ais, tree, datas[decl].rhs, space);
162182 },
163183
164 .container_field_init => return renderContainerField(gpa, ais, tree, tree.containerFieldInit(decl), space),
165 .container_field_align => return renderContainerField(gpa, ais, tree, tree.containerFieldAlign(decl), space),
166 .container_field => return renderContainerField(gpa, ais, tree, tree.containerField(decl), space),
184 .container_field_init => return renderContainerField(gpa, ais, tree, tree.containerFieldInit(decl), any_non_tuple_like_fields, space),
185 .container_field_align => return renderContainerField(gpa, ais, tree, tree.containerFieldAlign(decl), any_non_tuple_like_fields, space),
186 .container_field => return renderContainerField(gpa, ais, tree, tree.containerField(decl), any_non_tuple_like_fields, space),
167187 .@"comptime" => return renderExpression(gpa, ais, tree, decl, space),
168188
169189 .root => unreachable,
......@@ -1158,18 +1178,31 @@ fn renderContainerField(
11581178 gpa: Allocator,
11591179 ais: *Ais,
11601180 tree: Ast,
1161 field: Ast.full.ContainerField,
1181 field_param: Ast.full.ContainerField,
1182 any_non_tuple_like_fields: bool,
11621183 space: Space,
11631184) Error!void {
1185 var field = field_param;
1186 if (field.ast.tuple_like and any_non_tuple_like_fields and field.ast.type_expr != 0
1187 and tree.nodes.items(.tag)[field.ast.type_expr] == .identifier
1188 ) {
1189 const ident = tree.nodes.items(.main_token)[field.ast.type_expr];
1190 field.ast.tuple_like = false;
1191 field.ast.main_token = ident;
1192 field.ast.type_expr = 0;
1193 }
1194
11641195 if (field.comptime_token) |t| {
11651196 try renderToken(ais, tree, t, .space); // comptime
11661197 }
11671198 if (field.ast.type_expr == 0 and field.ast.value_expr == 0) {
1168 return renderIdentifierComma(ais, tree, field.ast.name_token, space, .eagerly_unquote); // name
1199 return renderIdentifierComma(ais, tree, field.ast.main_token, space, .eagerly_unquote); // name
11691200 }
11701201 if (field.ast.type_expr != 0 and field.ast.value_expr == 0) {
1171 try renderIdentifier(ais, tree, field.ast.name_token, .none, .eagerly_unquote); // name
1172 try renderToken(ais, tree, field.ast.name_token + 1, .space); // :
1202 if (!field.ast.tuple_like) {
1203 try renderIdentifier(ais, tree, field.ast.main_token, .none, .eagerly_unquote); // name
1204 try renderToken(ais, tree, field.ast.main_token + 1, .space); // :
1205 }
11731206
11741207 if (field.ast.align_expr != 0) {
11751208 try renderExpression(gpa, ais, tree, field.ast.type_expr, .space); // type
......@@ -1184,13 +1217,14 @@ fn renderContainerField(
11841217 }
11851218 }
11861219 if (field.ast.type_expr == 0 and field.ast.value_expr != 0) {
1187 try renderIdentifier(ais, tree, field.ast.name_token, .space, .eagerly_unquote); // name
1188 try renderToken(ais, tree, field.ast.name_token + 1, .space); // =
1220 try renderIdentifier(ais, tree, field.ast.main_token, .space, .eagerly_unquote); // name
1221 try renderToken(ais, tree, field.ast.main_token + 1, .space); // =
11891222 return renderExpressionComma(gpa, ais, tree, field.ast.value_expr, space); // value
11901223 }
1191
1192 try renderIdentifier(ais, tree, field.ast.name_token, .none, .eagerly_unquote); // name
1193 try renderToken(ais, tree, field.ast.name_token + 1, .space); // :
1224 if (!field.ast.tuple_like) {
1225 try renderIdentifier(ais, tree, field.ast.main_token, .none, .eagerly_unquote); // name
1226 try renderToken(ais, tree, field.ast.main_token + 1, .space); // :
1227 }
11941228 try renderExpression(gpa, ais, tree, field.ast.type_expr, .space); // type
11951229
11961230 if (field.ast.align_expr != 0) {
......@@ -1901,6 +1935,20 @@ fn renderContainerDecl(
19011935 try renderToken(ais, tree, layout_token, .space);
19021936 }
19031937
1938 var any_non_tuple_like_fields = token_tags[container_decl.ast.main_token] != .keyword_struct;
1939 if (!any_non_tuple_like_fields) for (container_decl.ast.members) |member| {
1940 const tuple_like = switch (tree.nodes.items(.tag)[member]) {
1941 .container_field_init => tree.containerFieldInit(member).ast.tuple_like,
1942 .container_field_align => tree.containerFieldAlign(member).ast.tuple_like,
1943 .container_field => tree.containerField(member).ast.tuple_like,
1944 else => continue,
1945 };
1946 if (!tuple_like) {
1947 any_non_tuple_like_fields = true;
1948 break;
1949 }
1950 };
1951
19041952 var lbrace: Ast.TokenIndex = undefined;
19051953 if (container_decl.ast.enum_token) |enum_token| {
19061954 try renderToken(ais, tree, container_decl.ast.main_token, .none); // union
......@@ -1967,7 +2015,7 @@ fn renderContainerDecl(
19672015 // Print all the declarations on the same line.
19682016 try renderToken(ais, tree, lbrace, .space); // lbrace
19692017 for (container_decl.ast.members) |member| {
1970 try renderMember(gpa, ais, tree, member, .space);
2018 try renderMember(gpa, ais, tree, member, any_non_tuple_like_fields, .space);
19712019 }
19722020 return renderToken(ais, tree, rbrace, space); // rbrace
19732021 }
......@@ -1985,9 +2033,9 @@ fn renderContainerDecl(
19852033 .container_field_init,
19862034 .container_field_align,
19872035 .container_field,
1988 => try renderMember(gpa, ais, tree, member, .comma),
2036 => try renderMember(gpa, ais, tree, member, any_non_tuple_like_fields, .comma),
19892037
1990 else => try renderMember(gpa, ais, tree, member, .newline),
2038 else => try renderMember(gpa, ais, tree, member, any_non_tuple_like_fields, .newline),
19912039 }
19922040 }
19932041 ais.popIndent();