| ... | @@ -1,6 +1,7 @@ | ... | @@ -1,6 +1,7 @@ |
| 1 | const std = @import("std"); | 1 | const std = @import("std"); |
| 2 | const Io = std.Io; | 2 | const Io = std.Io; |
| 3 | const Allocator = std.mem.Allocator; | 3 | const Allocator = std.mem.Allocator; |
| | 4 | const Token = std.zig.Token; |
| 4 | | 5 | |
| 5 | test "zig fmt: remove extra whitespace at start and end of file with comment between" { | 6 | test "zig fmt: remove extra whitespace at start and end of file with comment between" { |
| 6 | try testTransform( | 7 | try testTransform( |
| ... | @@ -6426,3 +6427,263 @@ fn fuzzTestOneParse(_: void, smith: *std.testing.Smith) !void { | ... | @@ -6426,3 +6427,263 @@ fn fuzzTestOneParse(_: void, smith: *std.testing.Smith) !void { |
| 6426 | var fba: std.heap.FixedBufferAllocator = .init(&fixed_buffer_mem); | 6427 | var fba: std.heap.FixedBufferAllocator = .init(&fixed_buffer_mem); |
| 6427 | _ = std.zig.Ast.parseTokens(fba.allocator(), tokens.source(), tokens.list(), mode) catch return; | 6428 | _ = std.zig.Ast.parseTokens(fba.allocator(), tokens.source(), tokens.list(), mode) catch return; |
| 6428 | } | 6429 | } |
| | 6430 | |
| | 6431 | test "zig fmt: fuzz" { |
| | 6432 | try std.testing.fuzz({}, fuzzRender, .{}); |
| | 6433 | } |
| | 6434 | |
| | 6435 | fn parseTokens( |
| | 6436 | fba: Allocator, |
| | 6437 | source: [:0]const u8, |
| | 6438 | ) error{ SkipZigTest, OutOfMemory }!struct { |
| | 6439 | toks: std.zig.Ast.TokenList, |
| | 6440 | maybe_rewritable: bool, |
| | 6441 | skip_idempotency: bool, |
| | 6442 | } { |
| | 6443 | @disableInstrumentation(); |
| | 6444 | // Byte-order marker is stripped |
| | 6445 | var maybe_rewritable = std.mem.startsWith(u8, source, "\xEF\xBB\xBF"); |
| | 6446 | var skip_idempotency = false; // This should be able to be removed once all the bugs are fixed |
| | 6447 | |
| | 6448 | var tokens: std.zig.Ast.TokenList = .{}; |
| | 6449 | try tokens.ensureTotalCapacity(fba, source.len / 2); |
| | 6450 | var tokenizer: std.zig.Tokenizer = .init(source); |
| | 6451 | while (true) { |
| | 6452 | const tok = tokenizer.next(); |
| | 6453 | switch (tok.tag) { |
| | 6454 | .invalid, |
| | 6455 | .invalid_periodasterisks, |
| | 6456 | => return error.SkipZigTest, |
| | 6457 | // Extra colons can be removed |
| | 6458 | .keyword_asm, |
| | 6459 | // Qualifiers can be reordered |
| | 6460 | // keyword_const is intentionally excluded since it is used in other contexts and |
| | 6461 | // having only one qualifier will never lead to reordering. |
| | 6462 | .keyword_addrspace, |
| | 6463 | .keyword_align, |
| | 6464 | .keyword_allowzero, |
| | 6465 | .keyword_callconv, |
| | 6466 | .keyword_linksection, |
| | 6467 | .keyword_volatile, |
| | 6468 | => maybe_rewritable = true, |
| | 6469 | .builtin, |
| | 6470 | // Pointer casts can be reordered |
| | 6471 | => for ([_][]const u8{ |
| | 6472 | "ptrCast", |
| | 6473 | "alignCast", |
| | 6474 | "addrSpaceCast", |
| | 6475 | "constCast", |
| | 6476 | "volatileCast", |
| | 6477 | }) |id| { |
| | 6478 | if (std.mem.eql(u8, source[tok.loc.start + 1 .. tok.loc.end], id)) { |
| | 6479 | maybe_rewritable = false; |
| | 6480 | } |
| | 6481 | }, |
| | 6482 | // Quoted identifiers can be unquoted |
| | 6483 | .identifier => maybe_rewritable = maybe_rewritable or source[tok.loc.start] == '@', |
| | 6484 | else => {}, |
| | 6485 | // #23754 |
| | 6486 | .container_doc_comment, |
| | 6487 | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag), &.{.l_brace})) { |
| | 6488 | return error.SkipZigTest; |
| | 6489 | }, |
| | 6490 | // #24507 |
| | 6491 | .keyword_inline, |
| | 6492 | .keyword_for, |
| | 6493 | .keyword_while, |
| | 6494 | .l_brace, |
| | 6495 | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag), &.{ .identifier, .colon })) { |
| | 6496 | maybe_rewritable = true; |
| | 6497 | skip_idempotency = true; |
| | 6498 | }, |
| | 6499 | } |
| | 6500 | try tokens.append(fba, .{ |
| | 6501 | .tag = tok.tag, |
| | 6502 | .start = @intCast(tok.loc.start), |
| | 6503 | }); |
| | 6504 | if (tok.tag == .eof) |
| | 6505 | break; |
| | 6506 | } |
| | 6507 | return .{ |
| | 6508 | .toks = tokens, |
| | 6509 | .maybe_rewritable = maybe_rewritable, |
| | 6510 | .skip_idempotency = skip_idempotency, |
| | 6511 | }; |
| | 6512 | } |
| | 6513 | |
| | 6514 | /// Checks equivelence of non-whitespace characters. |
| | 6515 | /// If there are commas in `source`, then it is checked they are also present |
| | 6516 | /// in `rendered`. Extra commas in `rendered` are ignored. |
| | 6517 | fn isRewritten(source: [:0]const u8, rendered: [:0]const u8) bool { |
| | 6518 | @disableInstrumentation(); |
| | 6519 | var i: usize = 0; |
| | 6520 | for (source[0 .. source.len + 1]) |c| switch (c) { |
| | 6521 | ' ', '\r', '\t', '\n' => {}, |
| | 6522 | else => while (true) { |
| | 6523 | defer i += 1; |
| | 6524 | switch (rendered[i]) { |
| | 6525 | ' ', '\n' => {}, |
| | 6526 | ',' => if (c == ',') break, |
| | 6527 | else => |r| if (c != r) return false else break, |
| | 6528 | } |
| | 6529 | }, |
| | 6530 | }; |
| | 6531 | std.debug.assert(i >= rendered.len); |
| | 6532 | return false; |
| | 6533 | } |
| | 6534 | |
| | 6535 | /// Checks that no line ends in whitespace |
| | 6536 | fn checkBetweenTokens(src: []const u8, fmt_on: *bool) error{ |
| | 6537 | TrailingLineWhitespace, |
| | 6538 | DoubleEmptyLine, |
| | 6539 | }!void { |
| | 6540 | @disableInstrumentation(); |
| | 6541 | var pos: usize = 0; |
| | 6542 | while (true) { |
| | 6543 | const nl_pos = std.mem.indexOfScalarPos(u8, src, pos, '\n'); |
| | 6544 | var check_trailing = fmt_on.*; |
| | 6545 | |
| | 6546 | const line = src[pos .. nl_pos orelse src.len]; |
| | 6547 | if (std.mem.indexOfScalar(u8, line, '/')) |comment_start| { |
| | 6548 | const comment_content = line[comment_start..][2..]; |
| | 6549 | const trimmed_comment = std.mem.trim(u8, comment_content, &std.ascii.whitespace); |
| | 6550 | if (std.mem.eql(u8, trimmed_comment, "zig fmt: off")) { |
| | 6551 | fmt_on.* = false; |
| | 6552 | } else if (std.mem.eql(u8, trimmed_comment, "zig fmt: on")) { |
| | 6553 | fmt_on.* = true; |
| | 6554 | check_trailing = true; |
| | 6555 | } |
| | 6556 | } |
| | 6557 | |
| | 6558 | pos = nl_pos orelse break; |
| | 6559 | if (check_trailing and pos != 0) switch (src[pos - 1]) { |
| | 6560 | ' ', '\t', '\r' => return error.TrailingLineWhitespace, |
| | 6561 | '\n' => if (pos != 1 and src[pos - 2] == '\n') return error.DoubleEmptyLine, |
| | 6562 | else => {}, |
| | 6563 | }; |
| | 6564 | pos += 1; |
| | 6565 | } |
| | 6566 | } |
| | 6567 | |
| | 6568 | /// Ignores extre `.comma` tokens in `rendered` |
| | 6569 | fn reparseTokens( |
| | 6570 | fba: Allocator, |
| | 6571 | rendered: [:0]const u8, |
| | 6572 | expected_tags: [:.eof]const Token.Tag, |
| | 6573 | ) error{ |
| | 6574 | OutOfMemory, |
| | 6575 | SameLineMultilineStringLiteral, |
| | 6576 | TrailingLineWhitespace, |
| | 6577 | DoubleEmptyLine, |
| | 6578 | }!struct { |
| | 6579 | toks: std.zig.Ast.TokenList, |
| | 6580 | rewritten: bool, |
| | 6581 | } { |
| | 6582 | @disableInstrumentation(); |
| | 6583 | var rewritten = false; |
| | 6584 | var tokens: std.zig.Ast.TokenList = .{}; |
| | 6585 | var last_token_end: usize = 0; |
| | 6586 | var fmt_on = true; |
| | 6587 | |
| | 6588 | try tokens.ensureTotalCapacity(fba, expected_tags.len + 2); // 1 for EOF and 1 for maybe a comma |
| | 6589 | var tokenizer: std.zig.Tokenizer = .init(rendered); |
| | 6590 | var i: usize = 0; |
| | 6591 | while (true) { |
| | 6592 | const tok = tokenizer.next(); |
| | 6593 | try tokens.append(fba, .{ |
| | 6594 | .tag = tok.tag, |
| | 6595 | .start = @intCast(tok.loc.start), |
| | 6596 | }); |
| | 6597 | |
| | 6598 | const between = rendered[last_token_end..tok.loc.start]; |
| | 6599 | last_token_end = tok.loc.end; |
| | 6600 | try checkBetweenTokens(between, &fmt_on); |
| | 6601 | if (tok.tag == .multiline_string_literal_line and fmt_on) blk: { |
| | 6602 | if (tokens.len == 1) |
| | 6603 | break :blk; // first token |
| | 6604 | if (std.mem.indexOfScalar(u8, between, '\n') == null) |
| | 6605 | return error.SameLineMultilineStringLiteral; |
| | 6606 | } |
| | 6607 | if (tok.tag == expected_tags[i]) { |
| | 6608 | if (tok.tag == .eof) |
| | 6609 | break; |
| | 6610 | i += 1; |
| | 6611 | } else if (tok.tag != .comma or !fmt_on) { |
| | 6612 | rewritten = true; |
| | 6613 | } |
| | 6614 | } |
| | 6615 | std.debug.assert(i == expected_tags.len); |
| | 6616 | try checkBetweenTokens(rendered[last_token_end..], &fmt_on); |
| | 6617 | |
| | 6618 | return .{ .toks = tokens, .rewritten = rewritten }; |
| | 6619 | } |
| | 6620 | |
| | 6621 | fn fuzzRender(_: void, smith: *std.testing.Smith) !void { |
| | 6622 | @disableInstrumentation(); |
| | 6623 | |
| | 6624 | var src_buf: [512]u8 = undefined; |
| | 6625 | const src_len = smith.sliceWeighted(&src_buf, &.{ |
| | 6626 | .rangeLessThan(u32, 0, 32, 256), |
| | 6627 | .rangeLessThan(u32, 32, 64, 64), |
| | 6628 | .rangeLessThan(u32, 64, src_buf.len, 1), |
| | 6629 | }, &.{ |
| | 6630 | .rangeAtMost(u8, 0x20, 0x7e, 8), |
| | 6631 | .value(u8, '\n', 32), |
| | 6632 | .value(u8, '\t', 8), |
| | 6633 | .value(u8, '\r', 4), |
| | 6634 | .rangeAtMost(u8, 0x7f, 0xff, 1), |
| | 6635 | }); |
| | 6636 | src_buf[src_len] = 0; |
| | 6637 | |
| | 6638 | var fba_ctx = std.heap.FixedBufferAllocator.init(&fixed_buffer_mem); |
| | 6639 | fuzzRenderInner(src_buf[0..src_len :0], fba_ctx.allocator()) catch |e| return switch (e) { |
| | 6640 | error.OutOfMemory => {}, |
| | 6641 | else => e, |
| | 6642 | }; |
| | 6643 | } |
| | 6644 | |
| | 6645 | fn fuzzRenderInner(source: [:0]const u8, fba: Allocator) !void { |
| | 6646 | @disableInstrumentation(); |
| | 6647 | |
| | 6648 | const src_toks = try parseTokens(fba, source); |
| | 6649 | const src_tree = try std.zig.Ast.parseTokens(fba, source, src_toks.toks.slice(), .zig); |
| | 6650 | if (src_tree.errors.len != 0) |
| | 6651 | return; |
| | 6652 | for (src_tree.nodes.items(.tag)) |tag| switch (tag) { |
| | 6653 | // #24507 (`switch(x) { inline for (a) |a| a => {} }` to |
| | 6654 | // `switch(x) { { inline for (a) |a| a => {} }` since |
| | 6655 | // AST determines inline case token as one before the case expression's first) |
| | 6656 | .switch_case_inline, .switch_case_inline_one => return error.SkipZigTest, |
| | 6657 | else => {}, |
| | 6658 | }; |
| | 6659 | |
| | 6660 | var rendered_w: std.Io.Writer.Allocating = .init(fba); |
| | 6661 | try rendered_w.ensureUnusedCapacity(source.len + source.len / 2); |
| | 6662 | try src_tree.render(fba, &rendered_w.writer, .{}); |
| | 6663 | // `toOwnedSliceSentinel` is not used since it reallocates the entire |
| | 6664 | // list to save space which is useless for fixed buffer allocators. |
| | 6665 | try rendered_w.writer.writeByte(0); |
| | 6666 | const rendered = rendered_w.written()[0 .. rendered_w.written().len - 1 :0]; |
| | 6667 | |
| | 6668 | // First check that the non-whitespace characters match. This ensures that |
| | 6669 | // identifier names, numbers, comments, et cetera are preserved. |
| | 6670 | if (!src_toks.maybe_rewritable and isRewritten(source, rendered)) |
| | 6671 | return error.Rewritten; |
| | 6672 | // Next check that the tokens are the same since whitespace removal can change the tokens |
| | 6673 | const src_tags = src_toks.toks.items(.tag); |
| | 6674 | const rendered_toks = try reparseTokens(fba, rendered, src_tags[0 .. src_tags.len - 1 :.eof]); |
| | 6675 | if (!src_toks.maybe_rewritable and rendered_toks.rewritten) |
| | 6676 | return error.Rewritten; |
| | 6677 | |
| | 6678 | // Rerender the tree to check idempotency and that new commas |
| | 6679 | // and whitespace changes did not create an AST error. |
| | 6680 | const rendered_tree = try std.zig.Ast.parseTokens(fba, rendered, rendered_toks.toks.slice(), .zig); |
| | 6681 | if (rendered_tree.errors.len != 0) |
| | 6682 | return error.Rewritten; |
| | 6683 | if (!src_toks.skip_idempotency) { |
| | 6684 | var rerendered_w: std.Io.Writer.Allocating = .init(fba); |
| | 6685 | try rerendered_w.ensureUnusedCapacity(source.len); |
| | 6686 | try rendered_tree.render(fba, &rerendered_w.writer, .{}); |
| | 6687 | try std.testing.expectEqualStrings(rendered, rerendered_w.written()); |
| | 6688 | } |
| | 6689 | } |
Kendall Condon <goon.pri.low@gmail.com> 2025-07-28 13:24:31-04:00
lib