| ... | ... | @@ -7279,83 +7279,61 @@ test "zig fmt: fuzz" { |
| 7279 | 7279 | try std.testing.fuzz({}, fuzzRender, .{}); |
| 7280 | 7280 | } |
| 7281 | 7281 | |
| 7282 | | fn parseTokens( |
| 7283 | | fba: Allocator, |
| 7284 | | source: [:0]const u8, |
| 7285 | | ) error{ SkipZigTest, OutOfMemory }!struct { |
| 7286 | | toks: std.zig.Ast.TokenList, |
| 7287 | | maybe_rewritable: bool, |
| 7288 | | skip_idempotency: bool, |
| 7289 | | } { |
| 7282 | fn isRewritable(source: []const u8, tokens: std.zig.Ast.TokenList.Slice) !bool { |
| 7290 | 7283 | @disableInstrumentation(); |
| 7284 | |
| 7291 | 7285 | // Byte-order marker is stripped |
| 7292 | 7286 | var maybe_rewritable = std.mem.startsWith(u8, source, "\xEF\xBB\xBF"); |
| 7293 | | var skip_idempotency = false; // This should be able to be removed once all the bugs are fixed |
| 7294 | | |
| 7295 | | var tokens: std.zig.Ast.TokenList = .{}; |
| 7296 | | try tokens.ensureTotalCapacity(fba, source.len / 2); |
| 7297 | | var tokenizer: std.zig.Tokenizer = .init(source); |
| 7298 | | while (true) { |
| 7299 | | const tok = tokenizer.next(); |
| 7300 | | switch (tok.tag) { |
| 7301 | | .invalid, |
| 7302 | | .invalid_periodasterisks, |
| 7303 | | => return error.SkipZigTest, |
| 7304 | | // Extra colons can be removed |
| 7305 | | .keyword_asm, |
| 7306 | | // Qualifiers can be reordered |
| 7307 | | // keyword_const is intentionally excluded since it is used in other contexts and |
| 7308 | | // having only one qualifier will never lead to reordering. |
| 7309 | | .keyword_addrspace, |
| 7310 | | .keyword_align, |
| 7311 | | .keyword_allowzero, |
| 7312 | | .keyword_callconv, |
| 7313 | | .keyword_linksection, |
| 7314 | | .keyword_volatile, |
| 7315 | | => maybe_rewritable = true, |
| 7316 | | .builtin, |
| 7317 | | // Pointer casts can be reordered |
| 7318 | | => for ([_][]const u8{ |
| 7319 | | "ptrCast", |
| 7320 | | "alignCast", |
| 7321 | | "addrSpaceCast", |
| 7322 | | "constCast", |
| 7323 | | "volatileCast", |
| 7324 | | }) |id| { |
| 7325 | | if (std.mem.eql(u8, source[tok.loc.start + 1 .. tok.loc.end], id)) { |
| 7326 | | maybe_rewritable = false; |
| 7327 | | } |
| 7328 | | }, |
| 7329 | | // Quoted identifiers can be unquoted |
| 7330 | | .identifier => maybe_rewritable = maybe_rewritable or source[tok.loc.start] == '@', |
| 7331 | | else => {}, |
| 7332 | | // #23754 |
| 7333 | | .container_doc_comment, |
| 7334 | | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag), &.{.l_brace})) { |
| 7335 | | return error.SkipZigTest; |
| 7336 | | }, |
| 7337 | | // #24507 |
| 7338 | | .keyword_inline, |
| 7339 | | .keyword_for, |
| 7340 | | .keyword_while, |
| 7341 | | .l_brace, |
| 7342 | | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag), &.{ .identifier, .colon })) { |
| 7287 | // The above variable can not yet be replaced by returns since error.SkipZigTest still needs to |
| 7288 | // be checked for. |
| 7289 | |
| 7290 | for (0.., tokens.items(.tag), tokens.items(.start)) |i, tag, start| switch (tag) { |
| 7291 | // Extra colons can be removed |
| 7292 | .keyword_asm, |
| 7293 | // Qualifiers can be reordered |
| 7294 | // keyword_const is intentionally excluded since it is used in other contexts and |
| 7295 | // having only one qualifier will never lead to reordering. |
| 7296 | .keyword_addrspace, |
| 7297 | .keyword_align, |
| 7298 | .keyword_allowzero, |
| 7299 | .keyword_callconv, |
| 7300 | .keyword_linksection, |
| 7301 | .keyword_volatile, |
| 7302 | => maybe_rewritable = true, |
| 7303 | .builtin, |
| 7304 | // Pointer casts can be reordered |
| 7305 | => for ([_][]const u8{ |
| 7306 | "ptrCast", |
| 7307 | "alignCast", |
| 7308 | "addrSpaceCast", |
| 7309 | "constCast", |
| 7310 | "volatileCast", |
| 7311 | }) |id| { |
| 7312 | if (std.mem.startsWith(u8, source[start + 1 ..], id)) { |
| 7343 | 7313 | maybe_rewritable = true; |
| 7344 | | skip_idempotency = true; |
| 7345 | | }, |
| 7346 | | } |
| 7347 | | try tokens.append(fba, .{ |
| 7348 | | .tag = tok.tag, |
| 7349 | | .start = @intCast(tok.loc.start), |
| 7350 | | }); |
| 7351 | | if (tok.tag == .eof) |
| 7352 | | break; |
| 7353 | | } |
| 7354 | | return .{ |
| 7355 | | .toks = tokens, |
| 7356 | | .maybe_rewritable = maybe_rewritable, |
| 7357 | | .skip_idempotency = skip_idempotency, |
| 7314 | } |
| 7315 | }, |
| 7316 | // Quoted identifiers can be unquoted |
| 7317 | .identifier => if (source[start] == '@') { |
| 7318 | maybe_rewritable = true; |
| 7319 | }, |
| 7320 | else => {}, |
| 7321 | // #23754 |
| 7322 | .container_doc_comment, |
| 7323 | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag)[0..i], &.{.l_brace})) { |
| 7324 | return error.SkipZigTest; // Can cause I.B. |
| 7325 | }, |
| 7326 | // #24507 |
| 7327 | .keyword_inline, |
| 7328 | .keyword_for, |
| 7329 | .keyword_while, |
| 7330 | .l_brace, |
| 7331 | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag)[0..i], &.{ .identifier, .colon })) { |
| 7332 | return error.SkipZigTest; // Can cause I.B. due to double rendering of zig fmt on/off |
| 7333 | }, |
| 7358 | 7334 | }; |
| 7335 | |
| 7336 | return maybe_rewritable; |
| 7359 | 7337 | } |
| 7360 | 7338 | |
| 7361 | 7339 | /// Checks equivelence of non-whitespace characters. |
| ... | ... | @@ -7468,34 +7446,29 @@ fn reparseTokens( |
| 7468 | 7446 | fn fuzzRender(_: void, smith: *std.testing.Smith) !void { |
| 7469 | 7447 | @disableInstrumentation(); |
| 7470 | 7448 | |
| 7471 | | var src_buf: [512]u8 = undefined; |
| 7472 | | const src_len = smith.sliceWeighted(&src_buf, &.{ |
| 7473 | | .rangeLessThan(u32, 0, 32, 256), |
| 7474 | | .rangeLessThan(u32, 32, 64, 64), |
| 7475 | | .rangeLessThan(u32, 64, src_buf.len, 1), |
| 7476 | | }, &.{ |
| 7477 | | .rangeAtMost(u8, 0x20, 0x7e, 8), |
| 7478 | | .value(u8, '\n', 32), |
| 7479 | | .value(u8, '\t', 8), |
| 7480 | | .value(u8, '\r', 4), |
| 7481 | | .rangeAtMost(u8, 0x7f, 0xff, 1), |
| 7482 | | }); |
| 7483 | | src_buf[src_len] = 0; |
| 7484 | | |
| 7449 | var ast_smith: std.zig.AstSmith = .init(smith); |
| 7450 | try ast_smith.generateSource(); |
| 7485 | 7451 | var fba_ctx = std.heap.FixedBufferAllocator.init(&fixed_buffer_mem); |
| 7486 | | fuzzRenderInner(src_buf[0..src_len :0], fba_ctx.allocator()) catch |e| return switch (e) { |
| 7487 | | error.OutOfMemory => {}, |
| 7488 | | else => e, |
| 7452 | var opt_rendered: ?[]const u8 = null; |
| 7453 | fuzzRenderInner(&ast_smith, fba_ctx.allocator(), &opt_rendered) catch |e| switch (e) { |
| 7454 | error.SkipZigTest, error.OutOfMemory, error.WriteFailed => return error.SkipZigTest, |
| 7455 | else => |failure| { |
| 7456 | ast_smith.logSource(); |
| 7457 | if (opt_rendered) |rendered| { |
| 7458 | logRenderedSource(rendered); |
| 7459 | } |
| 7460 | return failure; |
| 7461 | }, |
| 7489 | 7462 | }; |
| 7490 | 7463 | } |
| 7491 | 7464 | |
| 7492 | | fn fuzzRenderInner(source: [:0]const u8, fba: Allocator) !void { |
| 7465 | fn fuzzRenderInner(ast_smith: *std.zig.AstSmith, fba: Allocator, opt_rendered: *?[]const u8) !void { |
| 7493 | 7466 | @disableInstrumentation(); |
| 7494 | 7467 | |
| 7495 | | const src_toks = try parseTokens(fba, source); |
| 7496 | | const src_tree = try std.zig.Ast.parseTokens(fba, source, src_toks.toks.slice(), .zig); |
| 7497 | | if (src_tree.errors.len != 0) |
| 7498 | | return; |
| 7468 | const source = ast_smith.source(); |
| 7469 | const src_rewritable = try isRewritable(source, ast_smith.tokens()); |
| 7470 | const src_tree = try std.zig.Ast.parseTokens(fba, source, ast_smith.tokens(), .zig); |
| 7471 | std.debug.assert(src_tree.errors.len == 0); |
| 7499 | 7472 | for (src_tree.nodes.items(.tag)) |tag| switch (tag) { |
| 7500 | 7473 | // #24507 (`switch(x) { inline for (a) |a| a => {} }` to |
| 7501 | 7474 | // `switch(x) { { inline for (a) |a| a => {} }` since |
| ... | ... | @@ -7511,15 +7484,16 @@ fn fuzzRenderInner(source: [:0]const u8, fba: Allocator) !void { |
| 7511 | 7484 | // list to save space which is useless for fixed buffer allocators. |
| 7512 | 7485 | try rendered_w.writer.writeByte(0); |
| 7513 | 7486 | const rendered = rendered_w.written()[0 .. rendered_w.written().len - 1 :0]; |
| 7487 | opt_rendered.* = rendered; |
| 7514 | 7488 | |
| 7515 | 7489 | // First check that the non-whitespace characters match. This ensures that |
| 7516 | 7490 | // identifier names, numbers, comments, et cetera are preserved. |
| 7517 | | if (!src_toks.maybe_rewritable and isRewritten(source, rendered)) |
| 7491 | if (!src_rewritable and isRewritten(source, rendered)) |
| 7518 | 7492 | return error.Rewritten; |
| 7519 | 7493 | // Next check that the tokens are the same since whitespace removal can change the tokens |
| 7520 | | const src_tags = src_toks.toks.items(.tag); |
| 7494 | const src_tags = ast_smith.tokens().items(.tag); |
| 7521 | 7495 | const rendered_toks = try reparseTokens(fba, rendered, src_tags[0 .. src_tags.len - 1 :.eof]); |
| 7522 | | if (!src_toks.maybe_rewritable and rendered_toks.rewritten) |
| 7496 | if (!src_rewritable and rendered_toks.rewritten) |
| 7523 | 7497 | return error.Rewritten; |
| 7524 | 7498 | |
| 7525 | 7499 | // Rerender the tree to check idempotency and that new commas |
| ... | ... | @@ -7527,10 +7501,39 @@ fn fuzzRenderInner(source: [:0]const u8, fba: Allocator) !void { |
| 7527 | 7501 | const rendered_tree = try std.zig.Ast.parseTokens(fba, rendered, rendered_toks.toks.slice(), .zig); |
| 7528 | 7502 | if (rendered_tree.errors.len != 0) |
| 7529 | 7503 | return error.Rewritten; |
| 7530 | | if (!src_toks.skip_idempotency) { |
| 7531 | | var rerendered_w: std.Io.Writer.Allocating = .init(fba); |
| 7532 | | try rerendered_w.ensureUnusedCapacity(source.len); |
| 7533 | | try rendered_tree.render(fba, &rerendered_w.writer, .{}); |
| 7534 | | try std.testing.expectEqualStrings(rendered, rerendered_w.written()); |
| 7535 | | } |
| 7504 | var rerendered_w: std.Io.Writer.Allocating = .init(fba); |
| 7505 | try rerendered_w.ensureUnusedCapacity(source.len); |
| 7506 | try rendered_tree.render(fba, &rerendered_w.writer, .{}); |
| 7507 | try std.testing.expectEqualStrings(rendered, rerendered_w.written()); |
| 7508 | } |
| 7509 | |
| 7510 | fn logRenderedSource(source: []const u8) void { |
| 7511 | var buf: [256]u8 = undefined; |
| 7512 | const ls = std.debug.lockStderr(&buf); |
| 7513 | defer std.debug.unlockStderr(); |
| 7514 | logRenderedSourceInner(source, ls.terminal()) catch {}; |
| 7515 | } |
| 7516 | |
| 7517 | fn logRenderedSourceInner(source: []const u8, t: std.Io.Terminal) std.Io.Writer.Error!void { |
| 7518 | const w = t.writer; |
| 7519 | |
| 7520 | t.setColor(.dim) catch {}; |
| 7521 | try w.writeAll("=== Rendered Source ===\n"); |
| 7522 | t.setColor(.reset) catch {}; |
| 7523 | |
| 7524 | for (0.., source) |i, c| switch (c) { |
| 7525 | ' '...0x7e => try w.writeByte(c), |
| 7526 | '\n' => { |
| 7527 | if (i != 0 and source[i - 1] == ' ') { |
| 7528 | try w.writeAll("⏎"); |
| 7529 | } |
| 7530 | try w.writeByte('\n'); |
| 7531 | }, |
| 7532 | else => { |
| 7533 | t.setColor(.cyan) catch {}; |
| 7534 | try w.print("\\x{x:0>2}", .{c}); |
| 7535 | t.setColor(.reset) catch {}; |
| 7536 | }, |
| 7537 | }; |
| 7538 | try w.writeAll("␃\n"); |
| 7536 | 7539 | } |
Kendall Condon <goon.pri.low@gmail.com> 2026-03-22 17:19:41-04:00
lib