| ... | ... | @@ -1,6 +1,7 @@ |
| 1 | 1 | const std = @import("std"); |
| 2 | 2 | const Io = std.Io; |
| 3 | 3 | const Allocator = std.mem.Allocator; |
| 4 | const Token = std.zig.Token; |
| 4 | 5 | |
| 5 | 6 | test "zig fmt: remove extra whitespace at start and end of file with comment between" { |
| 6 | 7 | try testTransform( |
| ... | ... | @@ -6426,3 +6427,263 @@ fn fuzzTestOneParse(_: void, smith: *std.testing.Smith) !void { |
| 6426 | 6427 | var fba: std.heap.FixedBufferAllocator = .init(&fixed_buffer_mem); |
| 6427 | 6428 | _ = std.zig.Ast.parseTokens(fba.allocator(), tokens.source(), tokens.list(), mode) catch return; |
| 6428 | 6429 | } |
| 6430 | |
| 6431 | test "zig fmt: fuzz" { |
| 6432 | try std.testing.fuzz({}, fuzzRender, .{}); |
| 6433 | } |
| 6434 | |
| 6435 | fn parseTokens( |
| 6436 | fba: Allocator, |
| 6437 | source: [:0]const u8, |
| 6438 | ) error{ SkipZigTest, OutOfMemory }!struct { |
| 6439 | toks: std.zig.Ast.TokenList, |
| 6440 | maybe_rewritable: bool, |
| 6441 | skip_idempotency: bool, |
| 6442 | } { |
| 6443 | @disableInstrumentation(); |
| 6444 | // Byte-order marker is stripped |
| 6445 | var maybe_rewritable = std.mem.startsWith(u8, source, "\xEF\xBB\xBF"); |
| 6446 | var skip_idempotency = false; // This should be able to be removed once all the bugs are fixed |
| 6447 | |
| 6448 | var tokens: std.zig.Ast.TokenList = .{}; |
| 6449 | try tokens.ensureTotalCapacity(fba, source.len / 2); |
| 6450 | var tokenizer: std.zig.Tokenizer = .init(source); |
| 6451 | while (true) { |
| 6452 | const tok = tokenizer.next(); |
| 6453 | switch (tok.tag) { |
| 6454 | .invalid, |
| 6455 | .invalid_periodasterisks, |
| 6456 | => return error.SkipZigTest, |
| 6457 | // Extra colons can be removed |
| 6458 | .keyword_asm, |
| 6459 | // Qualifiers can be reordered |
| 6460 | // keyword_const is intentionally excluded since it is used in other contexts and |
| 6461 | // having only one qualifier will never lead to reordering. |
| 6462 | .keyword_addrspace, |
| 6463 | .keyword_align, |
| 6464 | .keyword_allowzero, |
| 6465 | .keyword_callconv, |
| 6466 | .keyword_linksection, |
| 6467 | .keyword_volatile, |
| 6468 | => maybe_rewritable = true, |
| 6469 | .builtin, |
| 6470 | // Pointer casts can be reordered |
| 6471 | => for ([_][]const u8{ |
| 6472 | "ptrCast", |
| 6473 | "alignCast", |
| 6474 | "addrSpaceCast", |
| 6475 | "constCast", |
| 6476 | "volatileCast", |
| 6477 | }) |id| { |
| 6478 | if (std.mem.eql(u8, source[tok.loc.start + 1 .. tok.loc.end], id)) { |
| 6479 | maybe_rewritable = false; |
| 6480 | } |
| 6481 | }, |
| 6482 | // Quoted identifiers can be unquoted |
| 6483 | .identifier => maybe_rewritable = maybe_rewritable or source[tok.loc.start] == '@', |
| 6484 | else => {}, |
| 6485 | // #23754 |
| 6486 | .container_doc_comment, |
| 6487 | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag), &.{.l_brace})) { |
| 6488 | return error.SkipZigTest; |
| 6489 | }, |
| 6490 | // #24507 |
| 6491 | .keyword_inline, |
| 6492 | .keyword_for, |
| 6493 | .keyword_while, |
| 6494 | .l_brace, |
| 6495 | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag), &.{ .identifier, .colon })) { |
| 6496 | maybe_rewritable = true; |
| 6497 | skip_idempotency = true; |
| 6498 | }, |
| 6499 | } |
| 6500 | try tokens.append(fba, .{ |
| 6501 | .tag = tok.tag, |
| 6502 | .start = @intCast(tok.loc.start), |
| 6503 | }); |
| 6504 | if (tok.tag == .eof) |
| 6505 | break; |
| 6506 | } |
| 6507 | return .{ |
| 6508 | .toks = tokens, |
| 6509 | .maybe_rewritable = maybe_rewritable, |
| 6510 | .skip_idempotency = skip_idempotency, |
| 6511 | }; |
| 6512 | } |
| 6513 | |
| 6514 | /// Checks equivelence of non-whitespace characters. |
| 6515 | /// If there are commas in `source`, then it is checked they are also present |
| 6516 | /// in `rendered`. Extra commas in `rendered` are ignored. |
| 6517 | fn isRewritten(source: [:0]const u8, rendered: [:0]const u8) bool { |
| 6518 | @disableInstrumentation(); |
| 6519 | var i: usize = 0; |
| 6520 | for (source[0 .. source.len + 1]) |c| switch (c) { |
| 6521 | ' ', '\r', '\t', '\n' => {}, |
| 6522 | else => while (true) { |
| 6523 | defer i += 1; |
| 6524 | switch (rendered[i]) { |
| 6525 | ' ', '\n' => {}, |
| 6526 | ',' => if (c == ',') break, |
| 6527 | else => |r| if (c != r) return false else break, |
| 6528 | } |
| 6529 | }, |
| 6530 | }; |
| 6531 | std.debug.assert(i >= rendered.len); |
| 6532 | return false; |
| 6533 | } |
| 6534 | |
| 6535 | /// Checks that no line ends in whitespace |
| 6536 | fn checkBetweenTokens(src: []const u8, fmt_on: *bool) error{ |
| 6537 | TrailingLineWhitespace, |
| 6538 | DoubleEmptyLine, |
| 6539 | }!void { |
| 6540 | @disableInstrumentation(); |
| 6541 | var pos: usize = 0; |
| 6542 | while (true) { |
| 6543 | const nl_pos = std.mem.indexOfScalarPos(u8, src, pos, '\n'); |
| 6544 | var check_trailing = fmt_on.*; |
| 6545 | |
| 6546 | const line = src[pos .. nl_pos orelse src.len]; |
| 6547 | if (std.mem.indexOfScalar(u8, line, '/')) |comment_start| { |
| 6548 | const comment_content = line[comment_start..][2..]; |
| 6549 | const trimmed_comment = std.mem.trim(u8, comment_content, &std.ascii.whitespace); |
| 6550 | if (std.mem.eql(u8, trimmed_comment, "zig fmt: off")) { |
| 6551 | fmt_on.* = false; |
| 6552 | } else if (std.mem.eql(u8, trimmed_comment, "zig fmt: on")) { |
| 6553 | fmt_on.* = true; |
| 6554 | check_trailing = true; |
| 6555 | } |
| 6556 | } |
| 6557 | |
| 6558 | pos = nl_pos orelse break; |
| 6559 | if (check_trailing and pos != 0) switch (src[pos - 1]) { |
| 6560 | ' ', '\t', '\r' => return error.TrailingLineWhitespace, |
| 6561 | '\n' => if (pos != 1 and src[pos - 2] == '\n') return error.DoubleEmptyLine, |
| 6562 | else => {}, |
| 6563 | }; |
| 6564 | pos += 1; |
| 6565 | } |
| 6566 | } |
| 6567 | |
| 6568 | /// Ignores extre `.comma` tokens in `rendered` |
| 6569 | fn reparseTokens( |
| 6570 | fba: Allocator, |
| 6571 | rendered: [:0]const u8, |
| 6572 | expected_tags: [:.eof]const Token.Tag, |
| 6573 | ) error{ |
| 6574 | OutOfMemory, |
| 6575 | SameLineMultilineStringLiteral, |
| 6576 | TrailingLineWhitespace, |
| 6577 | DoubleEmptyLine, |
| 6578 | }!struct { |
| 6579 | toks: std.zig.Ast.TokenList, |
| 6580 | rewritten: bool, |
| 6581 | } { |
| 6582 | @disableInstrumentation(); |
| 6583 | var rewritten = false; |
| 6584 | var tokens: std.zig.Ast.TokenList = .{}; |
| 6585 | var last_token_end: usize = 0; |
| 6586 | var fmt_on = true; |
| 6587 | |
| 6588 | try tokens.ensureTotalCapacity(fba, expected_tags.len + 2); // 1 for EOF and 1 for maybe a comma |
| 6589 | var tokenizer: std.zig.Tokenizer = .init(rendered); |
| 6590 | var i: usize = 0; |
| 6591 | while (true) { |
| 6592 | const tok = tokenizer.next(); |
| 6593 | try tokens.append(fba, .{ |
| 6594 | .tag = tok.tag, |
| 6595 | .start = @intCast(tok.loc.start), |
| 6596 | }); |
| 6597 | |
| 6598 | const between = rendered[last_token_end..tok.loc.start]; |
| 6599 | last_token_end = tok.loc.end; |
| 6600 | try checkBetweenTokens(between, &fmt_on); |
| 6601 | if (tok.tag == .multiline_string_literal_line and fmt_on) blk: { |
| 6602 | if (tokens.len == 1) |
| 6603 | break :blk; // first token |
| 6604 | if (std.mem.indexOfScalar(u8, between, '\n') == null) |
| 6605 | return error.SameLineMultilineStringLiteral; |
| 6606 | } |
| 6607 | if (tok.tag == expected_tags[i]) { |
| 6608 | if (tok.tag == .eof) |
| 6609 | break; |
| 6610 | i += 1; |
| 6611 | } else if (tok.tag != .comma or !fmt_on) { |
| 6612 | rewritten = true; |
| 6613 | } |
| 6614 | } |
| 6615 | std.debug.assert(i == expected_tags.len); |
| 6616 | try checkBetweenTokens(rendered[last_token_end..], &fmt_on); |
| 6617 | |
| 6618 | return .{ .toks = tokens, .rewritten = rewritten }; |
| 6619 | } |
| 6620 | |
| 6621 | fn fuzzRender(_: void, smith: *std.testing.Smith) !void { |
| 6622 | @disableInstrumentation(); |
| 6623 | |
| 6624 | var src_buf: [512]u8 = undefined; |
| 6625 | const src_len = smith.sliceWeighted(&src_buf, &.{ |
| 6626 | .rangeLessThan(u32, 0, 32, 256), |
| 6627 | .rangeLessThan(u32, 32, 64, 64), |
| 6628 | .rangeLessThan(u32, 64, src_buf.len, 1), |
| 6629 | }, &.{ |
| 6630 | .rangeAtMost(u8, 0x20, 0x7e, 8), |
| 6631 | .value(u8, '\n', 32), |
| 6632 | .value(u8, '\t', 8), |
| 6633 | .value(u8, '\r', 4), |
| 6634 | .rangeAtMost(u8, 0x7f, 0xff, 1), |
| 6635 | }); |
| 6636 | src_buf[src_len] = 0; |
| 6637 | |
| 6638 | var fba_ctx = std.heap.FixedBufferAllocator.init(&fixed_buffer_mem); |
| 6639 | fuzzRenderInner(src_buf[0..src_len :0], fba_ctx.allocator()) catch |e| return switch (e) { |
| 6640 | error.OutOfMemory => {}, |
| 6641 | else => e, |
| 6642 | }; |
| 6643 | } |
| 6644 | |
| 6645 | fn fuzzRenderInner(source: [:0]const u8, fba: Allocator) !void { |
| 6646 | @disableInstrumentation(); |
| 6647 | |
| 6648 | const src_toks = try parseTokens(fba, source); |
| 6649 | const src_tree = try std.zig.Ast.parseTokens(fba, source, src_toks.toks.slice(), .zig); |
| 6650 | if (src_tree.errors.len != 0) |
| 6651 | return; |
| 6652 | for (src_tree.nodes.items(.tag)) |tag| switch (tag) { |
| 6653 | // #24507 (`switch(x) { inline for (a) |a| a => {} }` to |
| 6654 | // `switch(x) { { inline for (a) |a| a => {} }` since |
| 6655 | // AST determines inline case token as one before the case expression's first) |
| 6656 | .switch_case_inline, .switch_case_inline_one => return error.SkipZigTest, |
| 6657 | else => {}, |
| 6658 | }; |
| 6659 | |
| 6660 | var rendered_w: std.Io.Writer.Allocating = .init(fba); |
| 6661 | try rendered_w.ensureUnusedCapacity(source.len + source.len / 2); |
| 6662 | try src_tree.render(fba, &rendered_w.writer, .{}); |
| 6663 | // `toOwnedSliceSentinel` is not used since it reallocates the entire |
| 6664 | // list to save space which is useless for fixed buffer allocators. |
| 6665 | try rendered_w.writer.writeByte(0); |
| 6666 | const rendered = rendered_w.written()[0 .. rendered_w.written().len - 1 :0]; |
| 6667 | |
| 6668 | // First check that the non-whitespace characters match. This ensures that |
| 6669 | // identifier names, numbers, comments, et cetera are preserved. |
| 6670 | if (!src_toks.maybe_rewritable and isRewritten(source, rendered)) |
| 6671 | return error.Rewritten; |
| 6672 | // Next check that the tokens are the same since whitespace removal can change the tokens |
| 6673 | const src_tags = src_toks.toks.items(.tag); |
| 6674 | const rendered_toks = try reparseTokens(fba, rendered, src_tags[0 .. src_tags.len - 1 :.eof]); |
| 6675 | if (!src_toks.maybe_rewritable and rendered_toks.rewritten) |
| 6676 | return error.Rewritten; |
| 6677 | |
| 6678 | // Rerender the tree to check idempotency and that new commas |
| 6679 | // and whitespace changes did not create an AST error. |
| 6680 | const rendered_tree = try std.zig.Ast.parseTokens(fba, rendered, rendered_toks.toks.slice(), .zig); |
| 6681 | if (rendered_tree.errors.len != 0) |
| 6682 | return error.Rewritten; |
| 6683 | if (!src_toks.skip_idempotency) { |
| 6684 | var rerendered_w: std.Io.Writer.Allocating = .init(fba); |
| 6685 | try rerendered_w.ensureUnusedCapacity(source.len); |
| 6686 | try rendered_tree.render(fba, &rerendered_w.writer, .{}); |
| 6687 | try std.testing.expectEqualStrings(rendered, rerendered_w.written()); |
| 6688 | } |
| 6689 | } |
Kendall Condon <goon.pri.low@gmail.com> 2025-07-28 13:24:31-04:00
lib