authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-08 10:10:01+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-08 15:49:17+02:00
log9953d7edca95ddd6ee0a517f33fa8f390cfe04c8
treea2f356dd72c254c70d2b3f1969d706f775b9de55
parent64d5ee389912ea3e89eae18c0993a5728bc902dc
signaturelock-open Commit is signed but in an unrecognized format.

std.zig.AstSmith: delete (for now)

The current AstSmith is hand-written based on the formal PEG. However, there is a significant impedance mismatch here since PEGs are a parser specification not a language specification like CFGs. In other words, it is not possible to directly translate an PEG into code that efficiently generates all possible inputs recognized by the PEG. Requirements for a new AstSmith: 1. Must be automatically generated based on the formal grammar specification. 2. Must have linear runtime. The current AstSmith is handwritten and unfortunately does not exactly match the formal grammar. Tests like this need to have a single source of truth and also need to be maintainable so I consider automatic generation non-negotiable. I did attempt to write a tool that takes a PEG as input and outputs an AstSmith implementation. However, I no longer believe it is possible to do a direct translation from an arbitrary PEG to an efficient AstSmith, the lookahead and ordered choice features have worst-case exponential runtime. Conceptually, PEGs are simple to translate into parsers while CFGs are simple to translate in to generators or smiths. I believe the path forward is to translate our PEG into a CFG and generate a smith based on that. I consider that task out of scope for this branch though.

3 files changed, 0 insertions(+), 2860 deletions(-)

lib/std/zig.zig-2
......@@ -29,7 +29,6 @@ pub const primitives = @import("zig/primitives.zig");
2929pub const isPrimitive = primitives.isPrimitive;
3030pub const Ast = @import("zig/Ast.zig");
3131pub const AstGen = @import("zig/AstGen.zig");
32pub const AstSmith = @import("zig/AstSmith.zig");
3332pub const Zir = @import("zig/Zir.zig");
3433pub const Zoir = @import("zig/Zoir.zig");
3534pub const ZonGen = @import("zig/ZonGen.zig");
......@@ -1865,7 +1864,6 @@ fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8
18651864test {
18661865 _ = Ast;
18671866 _ = AstRlAnnotate;
1868 _ = AstSmith;
18691867 _ = BuiltinFn;
18701868 _ = Client;
18711869 _ = ErrorBundle;
lib/std/zig/AstSmith.zig deleted-2595
......@@ -1,2595 +0,0 @@
1//! Generates a valid AST and corresponding source.
2//!
3//! This is based directly off grammer.peg
4
5const std = @import("../std.zig");
6const assert = std.debug.assert;
7const Token = std.zig.Token;
8const Smith = std.testing.Smith;
9const Weight = Smith.Weight;
10const AstSmith = @This();
11
12smith: *Smith,
13
14source_buf: [16384]u8,
15source_len: usize,
16
17token_tag_buf: [2048]Token.Tag,
18token_start_buf: [2048]std.zig.Ast.ByteOffset,
19tokens_len: usize,
20
21not_token: ?Token.Tag,
22not_token_comptime: bool,
23/// ExprSuffix
24/// <- KEYWORD_or
25/// / KEYWORD_and
26/// / CompareOp
27/// / BitwiseOp
28/// / BitShiftOp
29/// / AdditionOp
30/// / MultiplyOp
31/// / EXCLAMATIONMARK
32/// / SuffixOp
33/// / FnCallArguments
34not_expr_suffix: bool,
35/// LabelableExpr
36/// <- Block
37/// / SwitchExpr
38/// / LoopExpr
39not_labelable_expr: ?enum { colon, expr },
40not_label: bool,
41not_break_label: bool,
42not_block_expr: bool,
43not_expr_statement: bool,
44
45prev_ids_buf: [256]struct { start: u16, len: u16 },
46/// This may be larger than `prev_ids` in which case,
47/// x % prev_ids.len = next index
48/// @min(x, prev_ids) = length
49prev_ids_len: usize,
50
51/// `generate` must be called on the returned value before any other methods
52pub fn init(smith: *Smith) AstSmith {
53 return .{
54 .smith = smith,
55
56 .source_buf = undefined,
57 .source_len = 0,
58
59 .token_tag_buf = undefined,
60 .token_start_buf = undefined,
61 .tokens_len = 0,
62
63 .not_token = null,
64 .not_token_comptime = false,
65 .not_expr_suffix = false,
66 .not_labelable_expr = null,
67 .not_label = false,
68 .not_break_label = false,
69 .not_block_expr = false,
70 .not_expr_statement = false,
71
72 .prev_ids_buf = undefined,
73 .prev_ids_len = 0,
74 };
75}
76
77pub fn source(t: *AstSmith) [:0]u8 {
78 return t.source_buf[0..t.source_len :0];
79}
80
81/// The Slice is not backed by a MultiArrayList, so calling deinit or toMultiArrayList is illegal.
82pub fn tokens(t: *AstSmith) std.zig.Ast.TokenList.Slice {
83 var slice: std.zig.Ast.TokenList.Slice = .{
84 .ptrs = undefined,
85 .len = t.tokens_len,
86 .capacity = t.tokens_len,
87 };
88 comptime assert(slice.ptrs.len == 2);
89 slice.ptrs[@intFromEnum(std.zig.Ast.TokenList.Field.tag)] = @ptrCast(&t.token_tag_buf);
90 slice.ptrs[@intFromEnum(std.zig.Ast.TokenList.Field.start)] = @ptrCast(&t.token_start_buf);
91 return slice;
92}
93
94pub const Error = error{ OutOfMemory, SkipZigTest };
95const SourceError = error{SkipZigTest};
96
97pub fn generate(a: *AstSmith, gpa: std.mem.Allocator) Error!std.zig.Ast {
98 try a.generateSource();
99 const ast = try std.zig.Ast.parseTokens(gpa, a.source(), a.tokens(), .zig);
100 assert(ast.errors.len == 0);
101 return ast;
102}
103
104pub fn generateSource(a: *AstSmith) SourceError!void {
105 try a.pegRoot();
106 try a.ensureSourceCapacity(1);
107 a.source_buf[a.source_len] = 0;
108 try a.addTokenTag(.eof);
109}
110
111/// For choices which can introduce a variable number of expressions, this should be used to reduce
112/// unbounded recursion.
113//
114// `inline` to propogate caller's return address
115inline fn smithListItemBool(a: *AstSmith) bool {
116 return a.smith.boolWeighted(63, 1);
117}
118
119/// For choices which can introduce a variable number of expressions, this should be used to reduce
120/// unbounded recursion.
121//
122// `inline` to propogate caller's return address
123inline fn smithListItemEos(a: *AstSmith) bool {
124 return a.smith.eosWeightedSimple(1, 63);
125}
126
127fn sourceCapacity(a: *AstSmith) []u8 {
128 return a.source_buf[a.source_len..];
129}
130
131fn sourceCapacityLen(a: *AstSmith) usize {
132 return a.source_buf.len - a.source_len;
133}
134
135fn ensureSourceCapacity(a: *AstSmith, n: usize) SourceError!void {
136 if (a.sourceCapacityLen() < n) return error.SkipZigTest;
137}
138
139fn addSourceByte(a: *AstSmith, byte: u8) SourceError!void {
140 try a.ensureSourceCapacity(1);
141 a.addSourceByteAssumeCapacity(byte);
142}
143
144fn addSourceByteAssumeCapacity(a: *AstSmith, byte: u8) void {
145 a.sourceCapacity()[0] = byte;
146 a.source_len += 1;
147}
148
149fn addSource(a: *AstSmith, bytes: []const u8) SourceError!void {
150 try a.ensureSourceCapacity(bytes.len);
151 a.addSourceAssumeCapacity(bytes);
152}
153
154fn addSourceAssumeCapacity(a: *AstSmith, bytes: []const u8) void {
155 @memcpy(a.sourceCapacity()[0..bytes.len], bytes);
156 a.source_len += bytes.len;
157}
158
159fn addSourceAsSlice(a: *AstSmith, len: usize) SourceError![]u8 {
160 try a.ensureSourceCapacity(len);
161 return a.addSourceAsSliceAssumeCapacity(len);
162}
163
164fn addSourceAsSliceAssumeCapacity(a: *AstSmith, len: usize) []u8 {
165 const slice = a.sourceCapacity()[0..len];
166 a.source_len += len;
167 return slice;
168}
169
170fn tokenCapacityLen(a: *AstSmith) usize {
171 return a.token_tag_buf.len - a.tokens_len;
172}
173
174fn ensureTokenCapacity(a: *AstSmith, n: usize) SourceError!void {
175 if (a.tokenCapacityLen() < n) return error.SkipZigTest;
176}
177
178fn isAlphanumeric(c: u8) bool {
179 return switch (c) {
180 '_', 'a'...'z', 'A'...'Z', '0'...'9' => true,
181 else => false,
182 };
183}
184
185/// For tokens starting with alphanumerics, this ensures
186/// previous tokens followed by end_of_word aren't altered.
187///
188/// end_of_word <- ![a-zA-Z0-9_] skip
189fn preservePegEndOfWord(a: *AstSmith) SourceError!void {
190 if (a.source_len > 0 and isAlphanumeric(a.source_buf[a.source_len - 1])) {
191 try a.addSourceByte(' ');
192 }
193}
194
195/// Assumes the token has not been written yet
196fn addTokenTag(a: *AstSmith, tag: Token.Tag) SourceError!void {
197 assert(tag != a.not_token);
198 a.not_token = null;
199
200 if (a.not_token_comptime) assert(tag != .keyword_comptime);
201 a.not_token_comptime = false;
202
203 if (a.not_label and tag == .identifier) {
204 a.not_token = .colon;
205 }
206 a.not_label = false;
207
208 if (a.not_break_label and tag == .colon) {
209 a.not_token = .identifier;
210 }
211 a.not_break_label = false;
212
213 if (a.not_labelable_expr) |part| switch (part) {
214 .colon => a.not_labelable_expr = if (tag == .colon) .expr else null,
215 .expr => switch (tag) {
216 .l_brace => unreachable,
217 .keyword_inline => {},
218 .keyword_for => unreachable,
219 .keyword_while => unreachable,
220 .keyword_switch => unreachable,
221 else => a.not_labelable_expr = null,
222 },
223 };
224
225 a.not_expr_suffix = false;
226 a.not_block_expr = false;
227 a.not_expr_statement = false;
228
229 try a.ensureTokenCapacity(1);
230 a.token_tag_buf[a.tokens_len] = tag;
231 a.token_start_buf[a.tokens_len] = @intCast(a.source_len);
232 a.tokens_len += 1;
233}
234
235/// Asserts the token has a lexeme (those without have corresponding methods)
236fn pegToken(a: *AstSmith, tag: Token.Tag) SourceError!void {
237 const lexeme = tag.lexeme().?;
238
239 switch (lexeme[0]) {
240 '_', 'a'...'z', 'A'...'Z', '0'...'9' => try a.preservePegEndOfWord(),
241 '.' => if (a.tokens_len > 0 and switch (a.source_buf[a.source_len - 1]) {
242 '.' => true,
243 '0'...'9', 'a'...'z', 'A'...'Z' => a.token_tag_buf[a.tokens_len - 1] == .number_literal,
244 else => false,
245 }) {
246 try a.addSourceByte(' ');
247 },
248 '+', '-' => if (a.tokens_len > 0 and a.token_tag_buf[a.tokens_len - 1] == .number_literal and
249 switch (a.source_buf[a.source_len - 1]) {
250 'e', 'E', 'p', 'P' => true,
251 else => false,
252 })
253 {
254 // Would otherwise be tokenized as the sign of a float's exponent
255 //
256 // e.g. "0xFE" ++ "+" ++ "2" (number_literal, plus, number_literal)
257 try a.addSourceByte(' ');
258 },
259 else => {},
260 }
261
262 if (isAlphanumeric(lexeme[0])) try a.preservePegEndOfWord();
263
264 try a.addTokenTag(tag);
265 try a.addSource(lexeme);
266 try a.pegSkip();
267}
268
269/// Asserts `a.source_len != 0`
270fn pegTokenWhitespaceAround(a: *AstSmith, tag: Token.Tag) SourceError!void {
271 switch (a.source_buf[a.source_len - 1]) {
272 ' ', '\n' => {},
273 else => try a.addSourceByte(' '),
274 }
275 try a.addTokenTag(tag);
276 try a.addSource(tag.lexeme().?);
277 switch (a.smith.value(enum { space, line_break, cr_line_break })) {
278 // This is not the same as 'skip' since comments are not whitespace
279 .space => try a.addSourceByte(' '),
280 .line_break => try a.addSourceByte('\n'),
281 .cr_line_break => try a.addSource("\r\n"),
282 }
283 try a.pegSkip();
284}
285
286/// Root <- skip ContainerMembers eof
287fn pegRoot(a: *AstSmith) SourceError!void {
288 try a.pegSkip();
289 try a.pegContainerMembers();
290}
291
292/// ContainerMembers <- container_doc_comment? ContainerDeclaration* (ContainerField COMMA)*
293/// (ContainerField / ContainerDeclaration*)
294fn pegContainerMembers(a: *AstSmith) SourceError!void {
295 if (a.smith.boolWeighted(63, 1)) {
296 try a.pegContainerDocComment();
297 }
298 while (!a.smithListItemEos()) {
299 try a.pegContainerDeclaration();
300 }
301 while (!a.smithListItemEos()) {
302 try a.pegContainerField();
303 try a.pegToken(.comma);
304 }
305 if (a.smithListItemBool()) {
306 if (a.smith.value(bool)) {
307 try a.pegContainerField();
308 } else while (true) {
309 try a.pegContainerDeclaration();
310 if (a.smithListItemEos()) break;
311 }
312 }
313}
314
315/// ContainerDeclaration <- TestDecl / ComptimeDecl / doc_comment? KEYWORD_pub? Decl
316fn pegContainerDeclaration(a: *AstSmith) SourceError!void {
317 switch (a.smith.value(enum { TestDecl, ComptimeDecl, Decl })) {
318 .TestDecl => try a.pegTestDecl(),
319 .ComptimeDecl => try a.pegComptimeDecl(),
320 .Decl => {
321 try a.pegMaybeDocComment();
322 if (a.smith.value(bool)) {
323 try a.pegToken(.keyword_pub);
324 }
325 try a.pegDecl();
326 },
327 }
328}
329
330/// KEYWORD_test (STRINGLITERALSINGLE / IDENTIFIER)? Block
331fn pegTestDecl(a: *AstSmith) SourceError!void {
332 try a.pegToken(.keyword_test);
333 switch (a.smith.value(enum { none, string, id })) {
334 .none => {},
335 .string => try a.pegStringLiteralSingle(),
336 .id => try a.pegIdentifier(),
337 }
338 try a.pegBlock();
339}
340
341/// ComptimeDecl <- KEYWORD_comptime Block
342fn pegComptimeDecl(a: *AstSmith) SourceError!void {
343 try a.pegToken(.keyword_comptime);
344 try a.pegBlock();
345}
346
347/// Decl
348/// <- (KEYWORD_export / KEYWORD_inline / KEYWORD_noinline)? FnProto (SEMICOLON / Block)
349/// / KEYWORD_extern STRINGLITERALSINGLE? FnProto SEMICOLON
350/// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal?
351/// GlobalVarDecl
352fn pegDecl(a: *AstSmith) SourceError!void {
353 const Modifier = enum(u8) {
354 none,
355 @"export",
356 @"extern",
357 extern_library,
358 @"inline",
359 @"noinline",
360 };
361 const is_fn = a.smith.value(bool);
362 const fn_modifiers = Smith.baselineWeights(Modifier);
363 const var_modifiers: []const Weight = &.{.rangeAtMost(Modifier, .none, .extern_library, 1)};
364 const modifier = a.smith.valueWeighted(Modifier, if (is_fn) fn_modifiers else var_modifiers);
365
366 switch (modifier) {
367 .none => {},
368 .@"export" => try a.pegToken(.keyword_export),
369 .@"extern" => try a.pegToken(.keyword_extern),
370 .extern_library => {
371 try a.pegToken(.keyword_extern);
372 try a.pegStringLiteralSingle();
373 },
374 .@"inline" => try a.pegToken(.keyword_inline),
375 .@"noinline" => try a.pegToken(.keyword_noinline),
376 }
377
378 if (is_fn) {
379 try a.pegFnProto();
380 if (modifier == .@"extern" or modifier == .extern_library or a.smith.value(bool)) {
381 try a.pegToken(.semicolon);
382 } else {
383 try a.pegBlock();
384 }
385 } else {
386 if (a.smith.value(bool)) try a.pegToken(.keyword_threadlocal);
387 try a.pegGlobalVarDecl();
388 }
389}
390
391/// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? AddrSpace?
392/// LinkSection? CallConv? EXCLAMATIONMARK? TypeExpr !ExprSuffix
393fn pegFnProto(a: *AstSmith) SourceError!void {
394 try a.pegToken(.keyword_fn);
395 if (a.smith.value(bool)) {
396 try a.pegIdentifier();
397 }
398 try a.pegToken(.l_paren);
399 try a.pegParamDeclList();
400 try a.pegToken(.r_paren);
401 if (a.smith.value(bool)) {
402 try a.pegByteAlign();
403 }
404 if (a.smith.value(bool)) {
405 try a.pegAddrSpace();
406 }
407 if (a.smith.value(bool)) {
408 try a.pegLinkSection();
409 }
410 if (a.smith.value(bool)) {
411 try a.pegCallConv();
412 }
413 if (a.smith.value(bool)) {
414 try a.pegToken(.bang);
415 }
416 try a.pegTypeExpr();
417 a.not_expr_suffix = true;
418}
419
420/// VarDeclProto <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign?
421/// AddrSpace? LinkSection?
422fn pegVarDeclProto(a: *AstSmith) SourceError!void {
423 try a.pegToken(if (a.smith.value(bool)) .keyword_var else .keyword_const);
424 try a.pegIdentifier();
425
426 if (a.smith.value(bool)) {
427 try a.pegToken(.colon);
428 try a.pegTypeExpr();
429 }
430
431 if (a.smith.value(bool)) {
432 try a.pegByteAlign();
433 }
434
435 if (a.smith.value(bool)) {
436 try a.pegAddrSpace();
437 }
438
439 if (a.smith.value(bool)) {
440 try a.pegLinkSection();
441 }
442}
443
444/// GlobalVarDecl <- VarDeclProto (EQUAL Expr)? SEMICOLON
445fn pegGlobalVarDecl(a: *AstSmith) SourceError!void {
446 try a.pegVarDeclProto();
447 if (a.smithListItemBool()) {
448 try a.pegToken(.equal);
449 try a.pegExpr();
450 }
451 try a.pegToken(.semicolon);
452}
453
454/// ContainerField <- doc_comment? (KEYWORD_comptime / !KEYWORD_comptime) !KEYWORD_fn
455/// (IDENTIFIER COLON !(IDENTIFIER COLON)) TypeExpr ByteAlign? (EQUAL Expr)?
456fn pegContainerField(a: *AstSmith) SourceError!void {
457 try a.pegMaybeDocComment();
458 if (a.smith.value(bool)) {
459 try a.pegToken(.keyword_comptime);
460 }
461 if (a.smith.value(bool)) {
462 try a.pegIdentifier();
463 try a.pegToken(.colon);
464 } else {
465 a.not_token = .keyword_fn;
466 a.not_token_comptime = true;
467 a.not_label = true;
468 }
469 try a.pegTypeExpr();
470 if (a.smith.value(bool)) {
471 try a.pegByteAlign();
472 }
473 if (a.smith.value(bool)) {
474 try a.pegToken(.equal);
475 try a.pegExpr();
476 }
477}
478
479/// BlockStatement
480/// <- Statement
481/// / KEYWORD_defer BlockExprStatement
482/// / KEYWORD_errdefer BlockExprStatement
483/// / !ExprStatement (KEYWORD_comptime !BlockExpr)? VarAssignStatement
484fn pegBlockStatement(a: *AstSmith) SourceError!void {
485 const Kind = enum {
486 statement,
487 defer_statement,
488 errdefer_statement,
489 var_assign,
490 comptime_var_assign,
491 };
492 const weights = Smith.baselineWeights(Kind) ++ &[1]Weight{.value(Kind, .statement, 4)};
493 switch (a.smith.valueWeighted(Kind, weights)) {
494 .statement => try a.pegStatement(),
495 .defer_statement, .errdefer_statement => |kind| {
496 try a.pegToken(switch (kind) {
497 .defer_statement => .keyword_defer,
498 .errdefer_statement => .keyword_errdefer,
499 else => unreachable,
500 });
501 try a.pegBlockExprStatement();
502 },
503 .var_assign, .comptime_var_assign => |kind| {
504 a.not_expr_statement = true;
505 if (kind == .comptime_var_assign) {
506 try a.pegToken(.keyword_comptime);
507 a.not_block_expr = true;
508 }
509 try a.pegVarAssignStatement();
510 },
511 }
512}
513
514/// Statement
515/// <- ExprStatement
516/// / KEYWORD_suspend BlockExprStatement
517/// / !ExprStatement (KEYWORD_comptime !BlockExpr)? AssignExpr SEMICOLON
518///
519/// ExprStatement
520/// <- IfStatement
521/// / LabeledStatement
522/// / KEYWORD_nosuspend BlockExprStatement
523/// / KEYWORD_comptime BlockExpr
524fn pegStatement(a: *AstSmith) SourceError!void {
525 switch (a.smith.value(enum {
526 if_statement,
527 labeled_statement,
528 comptime_block_expr,
529
530 nosuspend_statement,
531 suspend_statement,
532 assign_expr,
533 comptime_assign_expr,
534 })) {
535 .if_statement => try a.pegIfStatement(),
536 .labeled_statement => try a.pegLabeledStatement(),
537 .comptime_block_expr => {
538 try a.pegToken(.keyword_comptime);
539 try a.pegBlockExpr();
540 },
541
542 .nosuspend_statement,
543 .suspend_statement,
544 => |kind| {
545 try a.pegToken(switch (kind) {
546 .nosuspend_statement => .keyword_nosuspend,
547 .suspend_statement => .keyword_suspend,
548 else => unreachable,
549 });
550 try a.pegBlockExprStatement();
551 },
552 .assign_expr, .comptime_assign_expr => |kind| {
553 a.not_expr_statement = true;
554 if (kind == .comptime_assign_expr) {
555 try a.pegToken(.keyword_comptime);
556 a.not_block_expr = true;
557 }
558 try a.pegAssignExpr();
559 try a.pegToken(.semicolon);
560 },
561 }
562}
563
564/// IfStatement
565/// <- IfPrefix BlockExpr ( KEYWORD_else Payload? Statement )?
566/// / IfPrefix !BlockExpr AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement )
567fn pegIfStatement(a: *AstSmith) SourceError!void {
568 try a.pegIfPrefix();
569 const is_assign = a.smith.value(bool);
570 if (!is_assign) {
571 try a.pegBlockExpr();
572 } else {
573 a.not_block_expr = true;
574 try a.pegAssignExpr();
575 }
576 if (a.not_token != .keyword_else and a.smithListItemBool()) {
577 try a.pegToken(.keyword_else);
578 if (a.smith.value(bool)) {
579 try a.pegPayload();
580 }
581 try a.pegStatement();
582 } else if (is_assign) {
583 try a.pegToken(.semicolon);
584 } else {
585 a.not_token = .keyword_else;
586 }
587}
588
589/// LabeledStatement <- BlockLabel? (Block / LoopStatement / SwitchExpr)
590fn pegLabeledStatement(a: *AstSmith) SourceError!void {
591 if (a.smith.value(bool)) {
592 try a.pegBlockLabel();
593 }
594 switch (a.smith.value(enum { block, loop_statement, switch_expr })) {
595 .block => try a.pegBlock(),
596 .loop_statement => try a.pegLoopStatement(),
597 .switch_expr => try a.pegSwitchExpr(),
598 }
599}
600
601/// LoopStatement <- KEYWORD_inline? (ForStatement / WhileStatement)
602fn pegLoopStatement(a: *AstSmith) SourceError!void {
603 if (a.smith.value(bool)) {
604 try a.pegToken(.keyword_inline);
605 }
606 if (a.smith.value(bool)) {
607 try a.pegForStatement();
608 } else {
609 try a.pegWhileStatement();
610 }
611}
612
613/// ForStatement
614/// <- ForPrefix BlockExpr ( KEYWORD_else Statement / !KEYWORD_else )
615/// / ForPrefix !BlockExpr AssignExpr ( SEMICOLON / KEYWORD_else Statement )
616fn pegForStatement(a: *AstSmith) SourceError!void {
617 try a.pegForPrefix();
618 const is_assign = a.smith.value(bool);
619 if (!is_assign) {
620 try a.pegBlockExpr();
621 } else {
622 a.not_block_expr = true;
623 try a.pegAssignExpr();
624 }
625 if (a.not_token != .keyword_else and a.smithListItemBool()) {
626 try a.pegToken(.keyword_else);
627 try a.pegStatement();
628 } else if (is_assign) {
629 try a.pegToken(.semicolon);
630 } else {
631 a.not_token = .keyword_else;
632 }
633}
634
635/// WhileStatement
636/// <- WhilePrefix BlockExpr ( KEYWORD_else Payload? Statement )?
637/// / WhilePrefix !BlockExpr AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement )
638fn pegWhileStatement(a: *AstSmith) SourceError!void {
639 try a.pegWhilePrefix();
640 const is_assign = a.smith.value(bool);
641 if (!is_assign) {
642 try a.pegBlockExpr();
643 } else {
644 a.not_block_expr = true;
645 try a.pegAssignExpr();
646 }
647 if (a.not_token != .keyword_else and a.smithListItemBool()) {
648 try a.pegToken(.keyword_else);
649 if (a.smith.value(bool)) {
650 try a.pegPayload();
651 }
652 try a.pegStatement();
653 } else if (is_assign) {
654 try a.pegToken(.semicolon);
655 } else {
656 a.not_token = .keyword_else;
657 }
658}
659
660/// BlockExprStatement
661/// <- BlockExpr
662/// / !BlockExpr AssignExpr SEMICOLON
663fn pegBlockExprStatement(a: *AstSmith) SourceError!void {
664 if (a.smith.value(bool)) {
665 try a.pegBlockExpr();
666 } else {
667 a.not_block_expr = true;
668 try a.pegAssignExpr();
669 try a.pegToken(.semicolon);
670 }
671}
672
673/// BlockExpr <- BlockLabel? Block
674fn pegBlockExpr(a: *AstSmith) SourceError!void {
675 if (a.smith.value(bool)) {
676 try a.pegBlockLabel();
677 }
678 try a.pegBlock();
679}
680
681/// VarAssignStatement <- (Expr / VarDeclProto) (COMMA (Expr / VarDeclProto))* EQUAL Expr SEMICOLON
682fn pegVarAssignStatement(a: *AstSmith) SourceError!void {
683 while (true) {
684 if (a.smith.value(bool)) {
685 try a.pegVarDeclProto();
686 } else {
687 try a.pegExpr();
688 }
689
690 if (a.smithListItemEos()) {
691 break;
692 } else {
693 try a.pegToken(.comma);
694 }
695 }
696
697 try a.pegToken(.equal);
698 try a.pegExpr();
699 try a.pegToken(.semicolon);
700}
701
702/// AssignExpr <- Expr (AssignOp Expr / (COMMA Expr)+ EQUAL Expr)?
703fn pegAssignExpr(a: *AstSmith) SourceError!void {
704 try a.pegExpr();
705 if (a.smith.value(bool)) {
706 if (!a.smithListItemBool()) {
707 try a.pegAssignOp();
708 } else {
709 while (true) {
710 try a.pegToken(.comma);
711 try a.pegExpr();
712 if (a.smithListItemEos()) break;
713 }
714 try a.pegToken(.equal);
715 }
716 try a.pegExpr();
717 }
718}
719
720/// SingleAssignExpr <- Expr (AssignOp Expr)?
721fn pegSingleAssignExpr(a: *AstSmith) SourceError!void {
722 try a.pegExpr();
723 if (a.smith.value(bool)) {
724 try a.pegAssignOp();
725 try a.pegExpr();
726 }
727}
728
729/// Expr <- BoolOrExpr
730const pegExpr = pegBoolOrExpr;
731
732/// BoolOrExpr <- BoolAndExpr (KEYWORD_or BoolAndExpr)*
733fn pegBoolOrExpr(a: *AstSmith) SourceError!void {
734 try a.pegBoolAndExpr();
735 while (!a.not_expr_suffix and !a.smithListItemEos()) {
736 try a.pegTokenWhitespaceAround(.keyword_or);
737 try a.pegBoolAndExpr();
738 }
739}
740
741/// BoolAndExpr <- CompareExpr (KEYWORD_and CompareExpr)*
742fn pegBoolAndExpr(a: *AstSmith) SourceError!void {
743 try a.pegCompareExpr();
744 while (!a.not_expr_suffix and !a.smithListItemEos()) {
745 try a.pegTokenWhitespaceAround(.keyword_and);
746 try a.pegCompareExpr();
747 }
748}
749
750/// CompareExpr <- BitwiseExpr (CompareOp BitwiseExpr)?
751fn pegCompareExpr(a: *AstSmith) SourceError!void {
752 try a.pegBitwiseExpr();
753 if (!a.not_expr_suffix and a.smithListItemBool()) {
754 try a.pegCompareOp();
755 try a.pegBitwiseExpr();
756 }
757}
758
759/// BitwiseExpr <- BitShiftExpr (BitwiseOp BitShiftExpr)*
760fn pegBitwiseExpr(a: *AstSmith) SourceError!void {
761 try a.pegBitShiftExpr();
762 while (!a.not_expr_suffix and !a.smithListItemEos()) {
763 try a.pegBitwiseOp();
764 try a.pegBitShiftExpr();
765 }
766}
767
768/// BitShiftExpr <- AdditionExpr (BitShiftOp AdditionExpr)*
769fn pegBitShiftExpr(a: *AstSmith) SourceError!void {
770 try a.pegAdditionExpr();
771 while (!a.not_expr_suffix and !a.smithListItemEos()) {
772 try a.pegBitShiftOp();
773 try a.pegAdditionExpr();
774 }
775}
776
777/// AdditionExpr <- MultiplyExpr (AdditionOp MultiplyExpr)*
778fn pegAdditionExpr(a: *AstSmith) SourceError!void {
779 try a.pegMultiplyExpr();
780 while (!a.not_expr_suffix and !a.smithListItemEos()) {
781 try a.pegAdditionOp();
782 try a.pegMultiplyExpr();
783 }
784}
785
786/// MultiplyExpr <- PrefixExpr (MultiplyOp PrefixExpr)*
787fn pegMultiplyExpr(a: *AstSmith) SourceError!void {
788 try a.pegPrefixExpr();
789 while (!a.not_expr_suffix and !a.smithListItemEos()) {
790 try a.pegMultiplyOp();
791 try a.pegPrefixExpr();
792 }
793}
794
795/// PrefixExpr <- PrefixOp* PrimaryExpr
796fn pegPrefixExpr(a: *AstSmith) SourceError!void {
797 while (!a.smithListItemEos()) {
798 try a.pegPrefixOp();
799 }
800 try a.pegPrimaryExpr();
801}
802
803/// PrimaryExpr
804/// <- AsmExpr
805/// / IfExpr
806/// / KEYWORD_break (BreakLabel / !BreakLabel) (Expr !ExprSuffix / !SinglePtrTypeStart)
807/// / KEYWORD_comptime Expr !ExprSuffix
808/// / KEYWORD_nosuspend Expr !ExprSuffix
809/// / KEYWORD_continue (BreakLabel / !BreakLabel) (Expr !ExprSuffix / !SinglePtrTypeStart)
810/// / KEYWORD_resume Expr !ExprSuffix
811/// / KEYWORD_return (Expr !ExprSuffix / !SinglePtrTypeStart)
812/// / BlockLabel? LoopExpr
813/// / Block
814/// / CurlySuffixExpr
815fn pegPrimaryExpr(a: *AstSmith) SourceError!void {
816 const Kind = enum(u8) {
817 curly_suffix_expr,
818 @"return",
819 @"continue",
820 @"break",
821 block,
822 asm_expr,
823 // Always contain more expressions
824 if_expr,
825 loop_expr,
826 @"resume",
827 @"comptime",
828 @"nosuspend",
829 };
830
831 switch (a.smith.valueWeighted(Kind, &.{
832 .value(Kind, .curly_suffix_expr, 75),
833 .rangeAtMost(Kind, .@"return", .asm_expr, 4),
834 .rangeAtMost(Kind, .if_expr, .@"nosuspend", 1),
835 })) {
836 .curly_suffix_expr => try a.pegCurlySuffixExpr(),
837
838 .block => if (a.not_labelable_expr != .expr and !a.not_block_expr and !a.not_expr_statement) {
839 try a.pegBlock();
840 } else {
841 // Group
842 try a.pegToken(.l_paren);
843 try a.pegBlock();
844 try a.pegToken(.r_paren);
845 },
846 .asm_expr => try a.pegAsmExpr(),
847 .if_expr => if (!a.not_expr_statement) {
848 try a.pegIfExpr();
849 } else {
850 // Group
851 try a.pegToken(.l_paren);
852 try a.pegIfExpr();
853 try a.pegToken(.r_paren);
854 },
855 .loop_expr => {
856 const group = a.not_labelable_expr == .expr or a.not_expr_statement;
857 if (group) try a.pegToken(.l_paren);
858 if (!a.not_label and a.not_token != .identifier and a.smith.value(bool)) {
859 try a.pegBlockLabel();
860 }
861 try a.pegLoopExpr();
862 if (group) try a.pegToken(.r_paren);
863 },
864
865 .@"return",
866 .@"comptime",
867 .@"nosuspend",
868 .@"resume",
869 .@"break",
870 .@"continue",
871 => |t| {
872 const group = a.not_expr_statement and (t == .@"nosuspend" or t == .@"comptime");
873 if (group) try a.pegToken(.l_paren);
874
875 const kw: Token.Tag, const label, const expr = switch (t) {
876 .@"return" => .{ .keyword_return, false, a.smithListItemBool() },
877 .@"comptime" => .{ .keyword_comptime, false, true },
878 .@"nosuspend" => .{ .keyword_nosuspend, false, true },
879 .@"resume" => .{ .keyword_resume, false, true },
880 .@"break" => .{ .keyword_break, a.smith.value(bool), a.smithListItemBool() },
881 .@"continue" => .{ .keyword_continue, a.smith.value(bool), a.smithListItemBool() },
882 else => unreachable,
883 };
884 try a.pegToken(kw);
885 if (label) {
886 try a.pegBreakLabel();
887 } else {
888 a.not_break_label = true;
889 }
890 if (expr) {
891 try a.pegExpr();
892 a.not_expr_suffix = true;
893 } else {
894 a.not_token = .asterisk;
895 }
896
897 if (group) try a.pegToken(.r_paren);
898 },
899 }
900}
901
902/// IfExpr <- IfPrefix Expr (KEYWORD_else Payload? Expr)? !ExprSuffix
903fn pegIfExpr(a: *AstSmith) SourceError!void {
904 try a.pegIfPrefix();
905 try a.pegExpr();
906 const Else = enum { none, @"else", else_payload };
907 switch (if (a.not_token != .keyword_else) a.smith.value(Else) else .none) {
908 .none => a.not_token = .keyword_else,
909 .@"else" => {
910 try a.pegToken(.keyword_else);
911 try a.pegExpr();
912 },
913 .else_payload => {
914 try a.pegToken(.keyword_else);
915 try a.pegPayload();
916 try a.pegExpr();
917 },
918 }
919 a.not_expr_suffix = true;
920}
921
922/// Block <- LBRACE Statement* RBRACE
923fn pegBlock(a: *AstSmith) SourceError!void {
924 try a.pegToken(.l_brace);
925 while (!a.smithListItemEos()) {
926 try a.pegBlockStatement();
927 }
928 try a.pegToken(.r_brace);
929}
930
931/// LoopExpr <- KEYWORD_inline? (ForExpr / WhileExpr)
932fn pegLoopExpr(a: *AstSmith) SourceError!void {
933 if (a.smith.value(bool)) {
934 try a.pegToken(.keyword_inline);
935 }
936
937 if (a.smith.value(bool)) {
938 try a.pegForExpr();
939 } else {
940 try a.pegWhileExpr();
941 }
942}
943
944/// ForExpr <- ForPrefix Expr (KEYWORD_else Expr / !KEYWORD_else) !ExprSuffix
945fn pegForExpr(a: *AstSmith) SourceError!void {
946 try a.pegForPrefix();
947 try a.pegExpr();
948 if (a.not_token != .keyword_else and a.smith.value(bool)) {
949 try a.pegToken(.keyword_else);
950 try a.pegExpr();
951 } else {
952 a.not_token = .keyword_else;
953 }
954 a.not_expr_suffix = true;
955}
956
957/// WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)? !ExprSuffix
958fn pegWhileExpr(a: *AstSmith) SourceError!void {
959 try a.pegWhilePrefix();
960 try a.pegExpr();
961 const Else = enum { none, @"else", else_payload };
962 switch (if (a.not_token != .keyword_else) a.smith.value(Else) else .none) {
963 .none => a.not_token = .keyword_else,
964 .@"else" => {
965 try a.pegToken(.keyword_else);
966 try a.pegExpr();
967 },
968 .else_payload => {
969 try a.pegToken(.keyword_else);
970 try a.pegPayload();
971 try a.pegExpr();
972 },
973 }
974 a.not_expr_suffix = true;
975}
976
977/// CurlySuffixExpr <- TypeExpr InitList?
978fn pegCurlySuffixExpr(a: *AstSmith) SourceError!void {
979 try a.pegTypeExpr();
980 if (!a.not_expr_suffix and a.smith.value(bool)) {
981 try a.pegInitList();
982 }
983}
984
985/// InitList
986/// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE
987/// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE
988/// / LBRACE RBRACE
989fn pegInitList(a: *AstSmith) SourceError!void {
990 try a.pegToken(.l_brace);
991 if (a.smithListItemBool()) {
992 if (a.smith.value(bool)) {
993 try a.pegFieldInit();
994 while (!a.smithListItemEos()) {
995 try a.pegToken(.comma);
996 try a.pegFieldInit();
997 }
998 } else {
999 try a.pegExpr();
1000 while (!a.smithListItemEos()) {
1001 try a.pegToken(.comma);
1002 try a.pegExpr();
1003 }
1004 }
1005 if (a.smith.value(bool)) {
1006 try a.pegToken(.comma);
1007 }
1008 }
1009 try a.pegToken(.r_brace);
1010}
1011
1012/// PrefixTypeOp* ErrorUnionExpr
1013fn pegTypeExpr(a: *AstSmith) SourceError!void {
1014 while (!a.smithListItemEos()) {
1015 try a.pegPrefixTypeOp();
1016 }
1017 try a.pegErrorUnionExpr();
1018}
1019
1020/// ErrorUnionExpr <- SuffixExpr (EXCLAMATIONMARK TypeExpr)?
1021fn pegErrorUnionExpr(a: *AstSmith) SourceError!void {
1022 try a.pegSuffixExpr();
1023 if (!a.not_expr_suffix and a.smithListItemBool()) {
1024 try a.pegToken(.bang);
1025 try a.pegTypeExpr();
1026 }
1027}
1028
1029/// SuffixExpr
1030/// <- PrimaryTypeExpr (SuffixOp / FnCallArguments)*
1031fn pegSuffixExpr(a: *AstSmith) SourceError!void {
1032 try a.pegPrimaryTypeExpr();
1033 while (!a.not_expr_suffix and !a.smithListItemEos()) {
1034 if (a.smith.value(bool)) {
1035 try a.pegSuffixOp();
1036 } else {
1037 try a.pegFnCallArguments();
1038 }
1039 }
1040}
1041
1042/// PrimaryTypeExpr
1043/// <- BUILTINIDENTIFIER FnCallArguments
1044/// / CHAR_LITERAL
1045/// / ContainerDecl
1046/// / DOT IDENTIFIER
1047/// / DOT InitList
1048/// / ErrorSetDecl
1049/// / FLOAT
1050/// / FnProto
1051/// / GroupedExpr
1052/// / LabeledTypeExpr
1053/// / IDENTIFIER !(COLON LabelableExpr)
1054/// / IfTypeExpr
1055/// / INTEGER
1056/// / KEYWORD_comptime TypeExpr !ExprSuffix
1057/// / KEYWORD_error DOT IDENTIFIER
1058/// / KEYWORD_anyframe
1059/// / KEYWORD_unreachable
1060/// / STRINGLITERAL
1061fn pegPrimaryTypeExpr(a: *AstSmith) SourceError!void {
1062 const Kind = enum(u8) {
1063 identifier,
1064 float,
1065 integer,
1066 char_literal,
1067 string_literal,
1068 enum_literal,
1069 error_literal,
1070 unreachable_type,
1071 anyframe_type,
1072
1073 // Containing zero or more expressions
1074 builtin_call,
1075 array_literal,
1076 container_decl,
1077 fn_proto,
1078 error_set,
1079
1080 // Containing one or more epressions
1081 grouped,
1082 labeled_type_expr,
1083 if_type_expr,
1084 comptime_expr,
1085 };
1086
1087 switch (a.smith.valueWeighted(Kind, &.{
1088 .rangeAtMost(Kind, .identifier, .anyframe_type, 5),
1089 .rangeAtMost(Kind, .builtin_call, .error_set, 2),
1090 .rangeAtMost(Kind, .grouped, .comptime_expr, 1),
1091 })) {
1092 .identifier => if (a.not_token != .identifier) {
1093 try a.pegIdentifier();
1094 a.not_labelable_expr = .colon;
1095 } else {
1096 // Group
1097 try a.pegToken(.l_paren);
1098 try a.pegIdentifier();
1099 try a.pegToken(.r_paren);
1100 },
1101 .float => try a.pegFloat(),
1102 .integer => try a.pegInteger(),
1103 .char_literal => try a.pegCharLiteral(),
1104 .string_literal => try a.pegStringLiteral(),
1105 .enum_literal => {
1106 try a.pegToken(.period);
1107 try a.pegIdentifier();
1108 },
1109 .error_literal => {
1110 try a.pegToken(.keyword_error);
1111 try a.pegToken(.period);
1112 try a.pegIdentifier();
1113 },
1114 .unreachable_type => try a.pegToken(.keyword_unreachable),
1115 .anyframe_type => try a.pegToken(.keyword_anyframe),
1116
1117 .builtin_call => {
1118 try a.pegBuiltinIdentifier();
1119 try a.pegFnCallArguments();
1120 },
1121 .array_literal => {
1122 try a.pegToken(.period);
1123 try a.pegInitList();
1124 },
1125 .container_decl => try a.pegContainerDecl(),
1126 .fn_proto => if (a.not_token != .keyword_fn) {
1127 try a.pegFnProto();
1128 } else {
1129 // Group
1130 try a.pegToken(.l_paren);
1131 try a.pegFnProto();
1132 try a.pegToken(.r_paren);
1133 },
1134 .error_set => try a.pegErrorSetDecl(),
1135
1136 .grouped => try a.pegGroupedExpr(),
1137 .labeled_type_expr => try a.pegLabeledTypeExpr(),
1138 .if_type_expr => if (!a.not_expr_statement) {
1139 try a.pegIfTypeExpr();
1140 } else {
1141 // Group
1142 try a.pegToken(.l_paren);
1143 try a.pegIfTypeExpr();
1144 try a.pegToken(.r_paren);
1145 },
1146 .comptime_expr => if (!a.not_token_comptime and !a.not_expr_statement) {
1147 try a.pegToken(.keyword_comptime);
1148 try a.pegTypeExpr();
1149 } else {
1150 // Group
1151 try a.pegToken(.l_paren);
1152 try a.pegToken(.keyword_comptime);
1153 try a.pegTypeExpr();
1154 try a.pegToken(.r_paren);
1155 },
1156 }
1157}
1158
1159/// ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto
1160fn pegContainerDecl(a: *AstSmith) SourceError!void {
1161 switch (a.smith.value(enum { auto, @"extern", @"packed" })) {
1162 .auto => {},
1163 .@"extern" => try a.pegToken(.keyword_extern),
1164 .@"packed" => try a.pegToken(.keyword_packed),
1165 }
1166 try a.pegContainerDeclAuto();
1167}
1168
1169/// ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE
1170fn pegErrorSetDecl(a: *AstSmith) SourceError!void {
1171 try a.pegToken(.keyword_error);
1172 try a.pegToken(.l_brace);
1173 try a.pegIdentifierList();
1174 try a.pegToken(.r_brace);
1175}
1176
1177/// GroupedExpr <- LPAREN Expr RPAREN
1178fn pegGroupedExpr(a: *AstSmith) SourceError!void {
1179 try a.pegToken(.l_paren);
1180 try a.pegExpr();
1181 try a.pegToken(.r_paren);
1182}
1183
1184/// IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? !ExprSuffix
1185fn pegIfTypeExpr(a: *AstSmith) SourceError!void {
1186 try a.pegIfPrefix();
1187 try a.pegTypeExpr();
1188 const Else = enum { none, @"else", else_payload };
1189 switch (if (a.not_token != .keyword_else) a.smith.value(Else) else .none) {
1190 .none => a.not_token = .keyword_else,
1191 .@"else" => {
1192 try a.pegToken(.keyword_else);
1193 try a.pegTypeExpr();
1194 },
1195 .else_payload => {
1196 try a.pegToken(.keyword_else);
1197 try a.pegPayload();
1198 try a.pegTypeExpr();
1199 },
1200 }
1201 a.not_expr_suffix = true;
1202}
1203
1204/// LabeledTypeExpr
1205/// <- BlockLabel Block
1206/// / BlockLabel? LoopTypeExpr
1207/// / BlockLabel? SwitchExpr
1208fn pegLabeledTypeExpr(a: *AstSmith) SourceError!void {
1209 const kind = a.smith.value(enum { block, loop, @"switch" });
1210 const not_any = a.not_labelable_expr == .expr or a.not_expr_statement;
1211 const no_label = a.not_label or a.not_token == .identifier;
1212 const no_block = no_label or a.not_block_expr;
1213 const group = not_any or (kind == .block and no_block);
1214 if (group) try a.pegToken(.l_paren);
1215
1216 switch (kind) {
1217 .block => {
1218 try a.pegBlockLabel();
1219 try a.pegBlock();
1220 },
1221 .loop => {
1222 if (!no_label and a.smith.value(bool)) {
1223 try a.pegBlockLabel();
1224 }
1225 try a.pegLoopTypeExpr();
1226 },
1227 .@"switch" => {
1228 if (!no_label and a.smith.value(bool)) {
1229 try a.pegBlockLabel();
1230 }
1231 try a.pegSwitchExpr();
1232 },
1233 }
1234
1235 if (group) try a.pegToken(.r_paren);
1236}
1237
1238/// LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr)
1239fn pegLoopTypeExpr(a: *AstSmith) SourceError!void {
1240 if (a.smith.value(bool)) {
1241 try a.pegToken(.keyword_inline);
1242 }
1243
1244 if (a.smith.value(bool)) {
1245 try a.pegForTypeExpr();
1246 } else {
1247 try a.pegWhileTypeExpr();
1248 }
1249}
1250
1251/// ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr / !KEYWORD_else) !ExprSuffix
1252fn pegForTypeExpr(a: *AstSmith) SourceError!void {
1253 try a.pegForPrefix();
1254 try a.pegTypeExpr();
1255 if (a.not_token != .keyword_else and a.smith.value(bool)) {
1256 try a.pegToken(.keyword_else);
1257 try a.pegTypeExpr();
1258 } else {
1259 a.not_token = .keyword_else;
1260 }
1261 a.not_expr_suffix = true;
1262}
1263
1264/// WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? !ExprSuffix
1265fn pegWhileTypeExpr(a: *AstSmith) SourceError!void {
1266 try a.pegWhilePrefix();
1267 try a.pegTypeExpr();
1268 const Else = enum { none, @"else", else_payload };
1269 switch (if (a.not_token != .keyword_else) a.smith.value(Else) else .none) {
1270 .none => a.not_token = .keyword_else,
1271 .@"else" => {
1272 try a.pegToken(.keyword_else);
1273 try a.pegTypeExpr();
1274 },
1275 .else_payload => {
1276 try a.pegToken(.keyword_else);
1277 try a.pegPayload();
1278 try a.pegTypeExpr();
1279 },
1280 }
1281 a.not_expr_suffix = true;
1282}
1283
1284/// SwitchExpr <- KEYWORD_switch LPAREN Expr RPAREN LBRACE SwitchProngList RBRACE
1285fn pegSwitchExpr(a: *AstSmith) SourceError!void {
1286 try a.pegToken(.keyword_switch);
1287 try a.pegToken(.l_paren);
1288 try a.pegExpr();
1289 try a.pegToken(.r_paren);
1290
1291 try a.pegToken(.l_brace);
1292 try a.pegSwitchProngList();
1293 try a.pegToken(.r_brace);
1294}
1295
1296/// AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN
1297fn pegAsmExpr(a: *AstSmith) SourceError!void {
1298 try a.pegToken(.keyword_asm);
1299 if (a.smith.value(bool)) {
1300 try a.pegToken(.keyword_volatile);
1301 }
1302 try a.pegToken(.l_paren);
1303 try a.pegExpr();
1304 if (a.smith.value(bool)) {
1305 try a.pegAsmOutput();
1306 }
1307 try a.pegToken(.r_paren);
1308}
1309
1310/// AsmOutput <- COLON AsmOutputList AsmInput?
1311fn pegAsmOutput(a: *AstSmith) SourceError!void {
1312 try a.pegToken(.colon);
1313 try a.pegAsmOutputList();
1314 if (a.smith.value(bool)) {
1315 try a.pegAsmInput();
1316 }
1317}
1318
1319/// AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERALSINGLE LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN
1320fn pegAsmOutputItem(a: *AstSmith) SourceError!void {
1321 try a.pegToken(.l_bracket);
1322 try a.pegIdentifier();
1323 try a.pegToken(.r_bracket);
1324 try a.pegStringLiteralSingle();
1325 try a.pegToken(.l_paren);
1326 if (a.smith.value(bool)) {
1327 try a.pegToken(.arrow);
1328 try a.pegTypeExpr();
1329 } else {
1330 try a.pegIdentifier();
1331 }
1332 try a.pegToken(.r_paren);
1333}
1334
1335/// AsmInput <- COLON AsmInputList AsmClobbers?
1336fn pegAsmInput(a: *AstSmith) SourceError!void {
1337 try a.pegToken(.colon);
1338 try a.pegAsmInputList();
1339 if (a.smith.value(bool)) {
1340 try a.pegAsmClobbers();
1341 }
1342}
1343
1344/// AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERALSINGLE LPAREN Expr RPAREN
1345fn pegAsmInputItem(a: *AstSmith) SourceError!void {
1346 try a.pegToken(.l_bracket);
1347 try a.pegIdentifier();
1348 try a.pegToken(.r_bracket);
1349 try a.pegStringLiteralSingle();
1350 try a.pegToken(.l_paren);
1351 try a.pegExpr();
1352 try a.pegToken(.r_paren);
1353}
1354
1355/// AsmClobbers <- COLON Expr
1356fn pegAsmClobbers(a: *AstSmith) SourceError!void {
1357 try a.pegToken(.colon);
1358 try a.pegExpr();
1359}
1360
1361/// BreakLabel <- COLON IDENTIFIER
1362fn pegBreakLabel(a: *AstSmith) SourceError!void {
1363 try a.pegToken(.colon);
1364 try a.pegIdentifier();
1365}
1366
1367/// BlockLabel <- IDENTIFIER COLON
1368fn pegBlockLabel(a: *AstSmith) SourceError!void {
1369 try a.pegIdentifier();
1370 try a.pegToken(.colon);
1371}
1372
1373/// FieldInit <- DOT IDENTIFIER EQUAL Expr
1374fn pegFieldInit(a: *AstSmith) SourceError!void {
1375 try a.pegToken(.period);
1376 try a.pegIdentifier();
1377 try a.pegToken(.equal);
1378 try a.pegExpr();
1379}
1380
1381/// WhileContinueExpr <- COLON LPAREN AssignExpr RPAREN
1382fn pegWhileContinueExpr(a: *AstSmith) SourceError!void {
1383 try a.pegToken(.colon);
1384 try a.pegToken(.l_paren);
1385 try a.pegAssignExpr();
1386 try a.pegToken(.r_paren);
1387}
1388
1389/// LinkSection <- KEYWORD_linksection LPAREN Expr RPAREN
1390fn pegLinkSection(a: *AstSmith) SourceError!void {
1391 try a.pegToken(.keyword_linksection);
1392 try a.pegToken(.l_paren);
1393 try a.pegExpr();
1394 try a.pegToken(.r_paren);
1395}
1396
1397/// AddrSpace <- KEYWORD_addrspace LPAREN Expr RPAREN
1398fn pegAddrSpace(a: *AstSmith) SourceError!void {
1399 try a.pegToken(.keyword_addrspace);
1400 try a.pegToken(.l_paren);
1401 try a.pegExpr();
1402 try a.pegToken(.r_paren);
1403}
1404
1405/// CallConv <- KEYWORD_callconv LPAREN Expr RPAREN
1406fn pegCallConv(a: *AstSmith) SourceError!void {
1407 try a.pegToken(.keyword_callconv);
1408 try a.pegToken(.l_paren);
1409 try a.pegExpr();
1410 try a.pegToken(.r_paren);
1411}
1412
1413/// ParamDecl <- doc_comment? (KEYWORD_noalias / KEYWORD_comptime)?
1414/// ((IDENTIFIER COLON) / !KEYWORD_comptime !(IDENTIFIER COLON))
1415/// ParamType
1416fn pegParamDecl(a: *AstSmith) SourceError!void {
1417 try a.pegMaybeDocComment();
1418 const modifier = a.smith.value(enum { none, @"noalias", @"comptime" });
1419 switch (modifier) {
1420 .none => a.not_token_comptime = true,
1421 .@"noalias" => try a.pegToken(.keyword_noalias),
1422 .@"comptime" => try a.pegToken(.keyword_comptime),
1423 }
1424 if (a.smith.value(bool)) {
1425 try a.pegIdentifier();
1426 try a.pegToken(.colon);
1427 } else {
1428 a.not_label = true;
1429 }
1430 try a.pegParamType();
1431}
1432
1433/// ParamType
1434/// <- KEYWORD_anytype
1435/// / TypeExpr
1436fn pegParamType(a: *AstSmith) SourceError!void {
1437 if (a.smith.value(bool)) {
1438 try a.pegToken(.keyword_anytype);
1439 } else {
1440 try a.pegTypeExpr();
1441 }
1442}
1443
1444/// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload?
1445fn pegIfPrefix(a: *AstSmith) SourceError!void {
1446 try a.pegToken(.keyword_if);
1447 try a.pegToken(.l_paren);
1448 try a.pegExpr();
1449 try a.pegToken(.r_paren);
1450 try a.pegPtrPayload();
1451}
1452
1453/// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
1454fn pegWhilePrefix(a: *AstSmith) SourceError!void {
1455 try a.pegToken(.keyword_while);
1456 try a.pegToken(.l_paren);
1457 try a.pegExpr();
1458 try a.pegToken(.r_paren);
1459
1460 if (a.smith.value(bool)) {
1461 try a.pegPtrPayload();
1462 }
1463
1464 if (a.smith.value(bool)) {
1465 try a.pegWhileContinueExpr();
1466 }
1467}
1468
1469/// ForPrefix <- KEYWORD_for LPAREN ForArgumentsList RPAREN PtrListPayload
1470///
1471/// An additional requirement checked in the Parser is that the number of
1472/// arguments and payload elements are the same.
1473fn pegForPrefix(a: *AstSmith) SourceError!void {
1474 try a.pegToken(.keyword_for);
1475 try a.pegToken(.l_paren);
1476 const n = try a.pegForArgumentsList();
1477 try a.pegToken(.r_paren);
1478 try a.pegPtrListPayload(n);
1479}
1480
1481/// Payload <- PIPE IDENTIFIER PIPE
1482fn pegPayload(a: *AstSmith) SourceError!void {
1483 try a.pegToken(.pipe);
1484 try a.pegIdentifier();
1485 try a.pegToken(.pipe);
1486}
1487
1488/// PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE
1489fn pegPtrPayload(a: *AstSmith) SourceError!void {
1490 try a.pegToken(.pipe);
1491 if (a.smith.value(bool)) {
1492 try a.pegToken(.asterisk);
1493 }
1494 try a.pegIdentifier();
1495 try a.pegToken(.pipe);
1496}
1497
1498/// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE
1499fn pegPtrIndexPayload(a: *AstSmith) SourceError!void {
1500 try a.pegToken(.pipe);
1501 if (a.smith.value(bool)) {
1502 try a.pegToken(.asterisk);
1503 }
1504 try a.pegIdentifier();
1505 if (a.smith.value(bool)) {
1506 try a.pegToken(.comma);
1507 try a.pegIdentifier();
1508 }
1509 try a.pegToken(.pipe);
1510}
1511
1512/// PtrListPayload <- PIPE ASTERISK? IDENTIFIER (COMMA ASTERISK? IDENTIFIER)* COMMA? PIPE
1513fn pegPtrListPayload(a: *AstSmith, n: usize) SourceError!void {
1514 try a.pegToken(.pipe);
1515 if (a.smith.value(bool)) {
1516 try a.pegToken(.asterisk);
1517 }
1518 try a.pegIdentifier();
1519
1520 for (1..n) |_| {
1521 try a.pegToken(.comma);
1522 if (a.smith.value(bool)) {
1523 try a.pegToken(.asterisk);
1524 }
1525 try a.pegIdentifier();
1526 }
1527
1528 if (a.smith.value(bool)) {
1529 try a.pegToken(.comma);
1530 }
1531 try a.pegToken(.pipe);
1532}
1533
1534/// SwitchProng <- KEYWORD_inline? SwitchCase EQUALRARROW PtrIndexPayload? SingleAssignExpr
1535fn pegSwitchProng(a: *AstSmith) SourceError!void {
1536 if (a.smith.value(bool)) {
1537 try a.pegToken(.keyword_inline);
1538 }
1539 try a.pegSwitchCase();
1540 try a.pegToken(.equal_angle_bracket_right);
1541 if (a.smith.value(bool)) {
1542 try a.pegPtrIndexPayload();
1543 }
1544 try a.pegSingleAssignExpr();
1545}
1546
1547/// SwitchCase
1548/// <- SwitchItem (COMMA SwitchItem)* COMMA?
1549/// / KEYWORD_else
1550fn pegSwitchCase(a: *AstSmith) SourceError!void {
1551 if (a.smith.value(bool)) {
1552 try a.pegSwitchItem();
1553 while (!a.smithListItemEos()) {
1554 try a.pegToken(.comma);
1555 try a.pegSwitchItem();
1556 }
1557 if (a.smith.value(bool)) {
1558 try a.pegToken(.comma);
1559 }
1560 } else {
1561 try a.pegToken(.keyword_else);
1562 }
1563}
1564
1565/// SwitchItem <- Expr (DOT3 Expr)?
1566fn pegSwitchItem(a: *AstSmith) SourceError!void {
1567 try a.pegExpr();
1568 if (a.smith.value(bool)) {
1569 try a.pegToken(.ellipsis3);
1570 try a.pegExpr();
1571 }
1572}
1573
1574/// ForArgumentsList <- ForItem (COMMA ForItem)* COMMA?
1575fn pegForArgumentsList(a: *AstSmith) SourceError!usize {
1576 try a.pegForItem();
1577 var n: usize = 1;
1578 while (!a.smithListItemEos()) {
1579 try a.pegToken(.comma);
1580 try a.pegForItem();
1581 n += 1;
1582 }
1583 if (a.smith.value(bool)) {
1584 try a.pegToken(.comma);
1585 }
1586 return n;
1587}
1588
1589/// ForItem <- Expr (DOT2 Expr?)?
1590fn pegForItem(a: *AstSmith) SourceError!void {
1591 try a.pegExpr();
1592 const components = a.smith.valueRangeAtMost(u2, 0, 2);
1593 if (components >= 1) try a.pegToken(.ellipsis2);
1594 if (components >= 2) try a.pegExpr();
1595}
1596
1597/// AssignOp
1598/// <- ASTERISKEQUAL
1599/// / ASTERISKPIPEEQUAL
1600/// / SLASHEQUAL
1601/// / PERCENTEQUAL
1602/// / PLUSEQUAL
1603/// / PLUSPIPEEQUAL
1604/// / MINUSEQUAL
1605/// / MINUSPIPEEQUAL
1606/// / LARROW2EQUAL
1607/// / LARROW2PIPEEQUAL
1608/// / RARROW2EQUAL
1609/// / AMPERSANDEQUAL
1610/// / CARETEQUAL
1611/// / PIPEEQUAL
1612/// / ASTERISKPERCENTEQUAL
1613/// / PLUSPERCENTEQUAL
1614/// / MINUSPERCENTEQUAL
1615/// / EQUAL
1616fn pegAssignOp(a: *AstSmith) SourceError!void {
1617 const tags = [_]Token.Tag{
1618 .asterisk_equal,
1619 .asterisk_pipe_equal,
1620 .slash_equal,
1621 .percent_equal,
1622 .plus_equal,
1623 .plus_pipe_equal,
1624 .minus_equal,
1625 .minus_pipe_equal,
1626 .angle_bracket_angle_bracket_left_equal,
1627 .angle_bracket_angle_bracket_left_pipe_equal,
1628 .angle_bracket_angle_bracket_right_equal,
1629 .ampersand_equal,
1630 .caret_equal,
1631 .pipe_equal,
1632 .asterisk_percent_equal,
1633 .plus_percent_equal,
1634 .minus_percent_equal,
1635 .equal,
1636 };
1637 try a.pegToken(tags[a.smith.index(tags.len)]);
1638}
1639
1640/// CompareOp
1641/// <- EQUALEQUAL
1642/// / EXCLAMATIONMARKEQUAL
1643/// / LARROW
1644/// / RARROW
1645/// / LARROWEQUAL
1646/// / RARROWEQUAL
1647fn pegCompareOp(a: *AstSmith) SourceError!void {
1648 const tags = [_]Token.Tag{
1649 .equal_equal,
1650 .bang_equal,
1651 .angle_bracket_left,
1652 .angle_bracket_right,
1653 .angle_bracket_left_equal,
1654 .angle_bracket_right_equal,
1655 };
1656 try a.pegTokenWhitespaceAround(tags[a.smith.index(tags.len)]);
1657}
1658
1659/// BitwiseOp
1660/// <- AMPERSAND
1661/// / CARET
1662/// / PIPE
1663/// / KEYWORD_orelse
1664/// / KEYWORD_catch Payload?
1665fn pegBitwiseOp(a: *AstSmith) SourceError!void {
1666 const tags = [_]Token.Tag{
1667 .ampersand,
1668 .caret,
1669 .pipe,
1670 .keyword_orelse,
1671 .keyword_catch,
1672 };
1673 const tag = tags[a.smith.index(tags.len)];
1674 try a.pegTokenWhitespaceAround(tag);
1675 if (tag == .keyword_catch and a.smith.value(bool)) {
1676 try a.pegPayload();
1677 }
1678}
1679
1680/// BitShiftOp
1681/// <- LARROW2
1682/// / RARROW2
1683/// / LARROW2PIPE
1684fn pegBitShiftOp(a: *AstSmith) SourceError!void {
1685 const tags = [_]Token.Tag{
1686 .angle_bracket_angle_bracket_left,
1687 .angle_bracket_angle_bracket_right,
1688 .angle_bracket_angle_bracket_left_pipe,
1689 };
1690 try a.pegTokenWhitespaceAround(tags[a.smith.index(tags.len)]);
1691}
1692
1693/// AdditionOp
1694/// <- PLUS
1695/// / MINUS
1696/// / PLUS2
1697/// / PLUSPERCENT
1698/// / MINUSPERCENT
1699/// / PLUSPIPE
1700/// / MINUSPIPE
1701fn pegAdditionOp(a: *AstSmith) SourceError!void {
1702 const tags = [_]Token.Tag{
1703 .plus,
1704 .minus,
1705 .plus_plus,
1706 .plus_percent,
1707 .minus_percent,
1708 .plus_pipe,
1709 .minus_pipe,
1710 };
1711 try a.pegTokenWhitespaceAround(tags[a.smith.index(tags.len)]);
1712}
1713
1714/// MultiplyOp
1715/// <- PIPE2
1716/// / ASTERISK
1717/// / SLASH
1718/// / PERCENT
1719/// / ASTERISKPERCENT
1720/// / ASTERISKPIPE
1721fn pegMultiplyOp(a: *AstSmith) SourceError!void {
1722 const tags = [_]Token.Tag{
1723 .asterisk,
1724 .pipe_pipe,
1725 .slash,
1726 .percent,
1727 .asterisk_percent,
1728 .asterisk_pipe,
1729 };
1730 const start = @as(u8, 2) * @intFromBool(a.not_token == .asterisk);
1731 try a.pegTokenWhitespaceAround(tags[a.smith.valueRangeLessThan(u8, start, tags.len)]);
1732}
1733
1734/// PrefixOp
1735/// <- EXCLAMATIONMARK
1736/// / MINUS
1737/// / TILDE
1738/// / MINUSPERCENT
1739/// / AMPERSAND
1740/// / KEYWORD_try
1741fn pegPrefixOp(a: *AstSmith) SourceError!void {
1742 const tags = [_]Token.Tag{
1743 .bang,
1744 .minus,
1745 .tilde,
1746 .minus_percent,
1747 .ampersand,
1748 .keyword_try,
1749 };
1750 try a.pegToken(tags[a.smith.index(tags.len)]);
1751}
1752
1753/// PrefixTypeOp
1754/// <- QUESTIONMARK
1755/// / KEYWORD_anyframe MINUSRARROW
1756/// / (ManyPtrTypeStart / SliceTypeStart) KEYWORD_allowzero? ByteAlign? AddrSpace?
1757/// KEYWORD_const? KEYWORD_volatile?
1758/// / SinglePtrTypeStart KEYWORD_allowzero? BitAlign? AddrSpace?
1759/// KEYWORD_const? KEYWORD_volatile?
1760/// / ArrayTypeStart
1761fn pegPrefixTypeOp(a: *AstSmith) SourceError!void {
1762 switch (a.smith.value(enum {
1763 optional,
1764 anyframe_arrow,
1765 array,
1766 single_pointer,
1767 many_pointer,
1768 slice,
1769 })) {
1770 .optional => try a.pegToken(.question_mark),
1771 .anyframe_arrow => {
1772 try a.pegToken(.keyword_anyframe);
1773 try a.pegToken(.arrow);
1774 },
1775 .array => try a.pegArrayTypeStart(),
1776 .single_pointer, .many_pointer, .slice => |kind| {
1777 const is_single = kind == .single_pointer and a.not_token != .asterisk;
1778 if (is_single) {
1779 try a.pegSinglePtrTypeStart();
1780 } else if (kind == .many_pointer) {
1781 try a.pegManyPtrTypeStart();
1782 } else {
1783 try a.pegSliceTypeStart();
1784 }
1785
1786 if (a.smith.value(bool)) {
1787 try a.pegToken(.keyword_allowzero);
1788 }
1789 if (a.smith.value(bool)) {
1790 if (is_single) {
1791 try a.pegBitAlign();
1792 } else {
1793 try a.pegByteAlign();
1794 }
1795 }
1796 if (a.smith.value(bool)) {
1797 try a.pegAddrSpace();
1798 }
1799 if (a.smith.value(bool)) {
1800 try a.pegToken(.keyword_const);
1801 }
1802 if (a.smith.value(bool)) {
1803 try a.pegToken(.keyword_volatile);
1804 }
1805 },
1806 }
1807}
1808
1809/// SuffixOp
1810/// <- LBRACKET Expr (DOT2 Expr? (COLON Expr)?)? RBRACKET
1811/// / DOT IDENTIFIER
1812/// / DOTASTERISK
1813/// / DOTQUESTIONMARK
1814fn pegSuffixOp(a: *AstSmith) SourceError!void {
1815 switch (a.smith.value(enum { slice, field, deref, unwrap })) {
1816 .slice => {
1817 try a.pegToken(.l_bracket);
1818 try a.pegExpr();
1819
1820 if (a.smith.value(bool)) {
1821 try a.pegToken(.ellipsis2);
1822 if (a.smith.value(bool))
1823 try a.pegExpr();
1824 if (a.smith.value(bool)) {
1825 try a.pegToken(.colon);
1826 try a.pegExpr();
1827 }
1828 }
1829
1830 try a.pegToken(.r_bracket);
1831 },
1832 .field => {
1833 try a.pegToken(.period);
1834 try a.pegIdentifier();
1835 },
1836 .deref => try a.pegToken(.period_asterisk),
1837 .unwrap => {
1838 try a.pegToken(.period);
1839 try a.pegToken(.question_mark);
1840 },
1841 }
1842}
1843
1844/// FnCallArguments <- LPAREN ExprList RPAREN
1845fn pegFnCallArguments(a: *AstSmith) SourceError!void {
1846 try a.pegToken(.l_paren);
1847 try a.pegExprList();
1848 try a.pegToken(.r_paren);
1849}
1850
1851/// SliceTypeStart <- LBRACKET (COLON Expr)? RBRACKET
1852fn pegSliceTypeStart(a: *AstSmith) SourceError!void {
1853 try a.pegToken(.l_bracket);
1854 if (a.smith.value(bool)) {
1855 try a.pegToken(.colon);
1856 try a.pegExpr();
1857 }
1858 try a.pegToken(.r_bracket);
1859}
1860
1861/// SinglePtrTypeStart <- ASTERISK
1862fn pegSinglePtrTypeStart(a: *AstSmith) SourceError!void {
1863 try a.pegToken(.asterisk);
1864}
1865
1866/// ManyPtrTypeStart <- LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET
1867fn pegManyPtrTypeStart(a: *AstSmith) SourceError!void {
1868 try a.pegToken(.l_bracket);
1869 try a.pegToken(.asterisk);
1870 switch (a.smith.value(enum { many, many_c, many_sentinel })) {
1871 .many => {},
1872 .many_c => {
1873 // No need for `preservePegEndOfWord` because the previous token is an asterisk
1874 try a.addTokenTag(.identifier);
1875 try a.addSourceByte('c');
1876 },
1877 .many_sentinel => {
1878 try a.pegToken(.colon);
1879 try a.pegExpr();
1880 },
1881 }
1882 try a.pegToken(.r_bracket);
1883}
1884
1885/// ArrayTypeStart <- LBRACKET !ASTERISK Expr (COLON Expr)? RBRACKET
1886fn pegArrayTypeStart(a: *AstSmith) SourceError!void {
1887 try a.pegToken(.l_bracket);
1888 a.not_token = .asterisk;
1889 try a.pegExpr();
1890 if (a.smith.value(bool)) {
1891 try a.pegToken(.colon);
1892 try a.pegExpr();
1893 }
1894 try a.pegToken(.r_bracket);
1895}
1896
1897/// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE
1898fn pegContainerDeclAuto(a: *AstSmith) SourceError!void {
1899 try a.pegContainerDeclType();
1900 try a.pegToken(.l_brace);
1901 try a.pegContainerMembers();
1902 try a.pegToken(.r_brace);
1903}
1904
1905/// ContainerDeclType
1906/// <- KEYWORD_struct (LPAREN Expr RPAREN)?
1907/// / KEYWORD_opaque
1908/// / KEYWORD_enum (LPAREN Expr RPAREN)?
1909/// / KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / !KEYWORD_enum Expr) RPAREN)?
1910fn pegContainerDeclType(a: *AstSmith) SourceError!void {
1911 switch (a.smith.value(enum { @"struct", @"opaque", @"enum", @"union" })) {
1912 .@"struct", .@"enum" => |c| {
1913 const is_struct = c == .@"struct" or a.not_token == .keyword_enum;
1914 try a.pegToken(if (is_struct) .keyword_struct else .keyword_enum);
1915 if (a.smith.value(bool)) {
1916 try a.pegToken(.l_paren);
1917 try a.pegExpr();
1918 try a.pegToken(.r_paren);
1919 }
1920 },
1921 .@"opaque" => try a.pegToken(.keyword_opaque),
1922 .@"union" => {
1923 try a.pegToken(.keyword_union);
1924 switch (a.smith.value(enum { no_tag, expr_tag, enum_tag, enum_expr_tag })) {
1925 .no_tag => {},
1926 .expr_tag => {
1927 try a.pegToken(.l_paren);
1928 a.not_token = .keyword_enum;
1929 try a.pegExpr();
1930 try a.pegToken(.r_paren);
1931 },
1932 .enum_tag => {
1933 try a.pegToken(.l_paren);
1934 try a.pegToken(.keyword_enum);
1935 try a.pegToken(.r_paren);
1936 },
1937 .enum_expr_tag => {
1938 try a.pegToken(.l_paren);
1939 try a.pegToken(.keyword_enum);
1940 try a.pegToken(.l_paren);
1941 try a.pegExpr();
1942 try a.pegToken(.r_paren);
1943 try a.pegToken(.r_paren);
1944 },
1945 }
1946 },
1947 }
1948}
1949
1950/// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN
1951fn pegByteAlign(a: *AstSmith) SourceError!void {
1952 try a.pegToken(.keyword_align);
1953 try a.pegToken(.l_paren);
1954 try a.pegExpr();
1955 try a.pegToken(.r_paren);
1956}
1957
1958/// BitAlign <- KEYWORD_align LPAREN Expr (COLON Expr COLON Expr)? RPAREN
1959fn pegBitAlign(a: *AstSmith) SourceError!void {
1960 try a.pegToken(.keyword_align);
1961 try a.pegToken(.l_paren);
1962 try a.pegExpr();
1963 if (a.smith.value(bool)) {
1964 try a.pegToken(.colon);
1965 try a.pegExpr();
1966 try a.pegToken(.colon);
1967 try a.pegExpr();
1968 }
1969 try a.pegToken(.r_paren);
1970}
1971
1972/// IdentifierList <- (doc_comment? IDENTIFIER COMMA)* (doc_comment? IDENTIFIER)?
1973fn pegIdentifierList(a: *AstSmith) SourceError!void {
1974 while (!a.smith.eos()) {
1975 try a.pegMaybeDocComment();
1976 try a.pegIdentifier();
1977 try a.pegToken(.comma);
1978 }
1979 if (a.smith.value(bool)) {
1980 try a.pegMaybeDocComment();
1981 try a.pegIdentifier();
1982 }
1983}
1984
1985/// SwitchProngList <- (SwitchProng COMMA)* SwitchProng?
1986fn pegSwitchProngList(a: *AstSmith) SourceError!void {
1987 while (!a.smithListItemEos()) {
1988 try a.pegSwitchProng();
1989 try a.pegToken(.comma);
1990 }
1991 if (a.smithListItemBool()) {
1992 try a.pegSwitchProng();
1993 }
1994}
1995
1996/// AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem?
1997fn pegAsmOutputList(a: *AstSmith) SourceError!void {
1998 while (!a.smithListItemEos()) {
1999 try a.pegAsmOutputItem();
2000 try a.pegToken(.comma);
2001 }
2002 if (a.smithListItemBool()) {
2003 try a.pegAsmOutputItem();
2004 }
2005}
2006
2007/// AsmInputList <- (AsmInputItem COMMA)* AsmInputItem?
2008fn pegAsmInputList(a: *AstSmith) SourceError!void {
2009 while (!a.smithListItemEos()) {
2010 try a.pegAsmInputItem();
2011 try a.pegToken(.comma);
2012 }
2013 if (a.smithListItemBool()) {
2014 try a.pegAsmInputItem();
2015 }
2016}
2017
2018/// ParamDeclList <- (ParamDecl COMMA)* (ParamDecl / DOT3 COMMA?)?
2019fn pegParamDeclList(a: *AstSmith) SourceError!void {
2020 while (!a.smithListItemEos()) {
2021 try a.pegParamDecl();
2022 try a.pegToken(.comma);
2023 }
2024 const Final = enum { none, dot3, dot3_comma, param };
2025 switch (a.smith.valueWeighted(Final, &.{
2026 .rangeLessThan(Final, .none, .param, 2),
2027 .value(Final, .param, 1),
2028 })) {
2029 .none => {},
2030 .dot3 => try a.pegToken(.ellipsis3),
2031 .dot3_comma => {
2032 try a.pegToken(.ellipsis3);
2033 try a.pegToken(.comma);
2034 },
2035 .param => try a.pegParamDecl(),
2036 }
2037}
2038
2039/// ExprList <- (Expr COMMA)* Expr?
2040fn pegExprList(a: *AstSmith) SourceError!void {
2041 while (!a.smithListItemEos()) {
2042 try a.pegExpr();
2043 try a.pegToken(.comma);
2044 }
2045 if (a.smithListItemBool()) {
2046 try a.pegExpr();
2047 }
2048}
2049
2050/// container_doc_comment <- ('//!' non_control_utf8* [ \n]* skip)+
2051fn pegContainerDocComment(a: *AstSmith) SourceError!void {
2052 while (true) {
2053 try a.addTokenTag(.container_doc_comment);
2054 try a.pegGenericLine("//!", .any);
2055 try a.pegSkip();
2056 if (a.smith.eos()) break;
2057 }
2058}
2059
2060/// doc_comment?
2061fn pegMaybeDocComment(a: *AstSmith) SourceError!void {
2062 // A specific hash is provided here since this function is likely to be inlined,
2063 // however having all doc comments with the same uid is beneficial.
2064 if (a.smith.boolWeightedWithHash(63, 1, 0x39b94392)) {
2065 try a.pegDocComment();
2066 }
2067}
2068
2069/// doc_comment <- ('///' non_control_utf8* [ \n]* skip)+
2070fn pegDocComment(a: *AstSmith) SourceError!void {
2071 if (a.source_len > 0 and a.source_buf[a.source_len - 1] != '\n') {
2072 try a.addSourceByte('\n');
2073 }
2074 while (true) {
2075 try a.addTokenTag(.doc_comment);
2076 try a.pegGenericLine("///", .doc_comment);
2077 try a.pegSkip();
2078 if (a.smith.eosWeightedSimple(1, 3)) break;
2079 }
2080}
2081
2082/// line_comment <- '//' ![!/] non_control_utf8* / '////' non_control_utf8*
2083fn pegLineComment(a: *AstSmith) SourceError!void {
2084 return a.pegGenericLine("//", .line_comment);
2085}
2086
2087/// line_string <- '\\\\' non_control_utf8* [ \n]*
2088fn pegLineString(a: *AstSmith) SourceError!void {
2089 try a.addTokenTag(.multiline_string_literal_line);
2090 return a.pegGenericLine("\\\\", .any);
2091}
2092
2093/// non_control_utf8 <- [\040-\377]
2094///
2095/// Used for line, doc, and container comments as well as
2096/// multiline string literal lines.
2097fn pegGenericLine(
2098 a: *AstSmith,
2099 prefix: []const u8,
2100 /// Adds constraints to what the line contains
2101 prefix_kind: enum { any, line_comment, doc_comment },
2102) SourceError!void {
2103 const cr = a.smith.value(bool);
2104 const newline_len = @intFromBool(cr) + @as(usize, 1);
2105
2106 try a.ensureSourceCapacity(prefix.len + newline_len);
2107 a.addSourceAssumeCapacity(prefix);
2108
2109 const line = a.variableChar(newline_len, 0, &.{
2110 .rangeAtMost(u8, ' ', 0x7f - 1, 1),
2111 .rangeAtMost(u8, 0x7f + 1, 0xff, 1),
2112 });
2113 if (line.len >= 1) switch (prefix_kind) {
2114 .any => {},
2115 .line_comment => {
2116 // Convert doc comments to quadruple slashes when possible;
2117 // Otherwise, and for container doc comments, erase the '/' or '!'
2118 if (line[0] == '/' and line.len >= 2) {
2119 line[1] = '/';
2120 } else if (line[0] == '/' or line[0] == '!') {
2121 line[0] = ' ';
2122 }
2123 },
2124 .doc_comment => {
2125 // Avoid quadruple slashes
2126 if (line[0] == '/') {
2127 line[0] = ' ';
2128 }
2129 },
2130 };
2131
2132 if (cr) a.addSourceByteAssumeCapacity('\r');
2133 a.addSourceByteAssumeCapacity('\n');
2134}
2135
2136/// skip <- ([ \n] / line_comment)*
2137fn pegSkip(a: *AstSmith) SourceError!void {
2138 if (a.smith.boolWeighted(63, 1)) {
2139 while (true) {
2140 const Kind = enum {
2141 space,
2142 line_break,
2143 cr_line_break,
2144 line_comment,
2145 line_comment_zig_fmt_off,
2146 line_comment_zig_fmt_on,
2147 };
2148
2149 const weights = Smith.baselineWeights(Kind) ++
2150 [_]Weight{.value(Kind, .space, 11)};
2151 switch (a.smith.valueWeighted(Kind, weights)) {
2152 .space => try a.addSourceByte(' '),
2153 .line_break => try a.addSourceByte('\n'),
2154 .cr_line_break => try a.addSource("\r\n"),
2155 .line_comment => try a.pegLineComment(),
2156 .line_comment_zig_fmt_off => try a.addSource("//zig fmt: off\n"),
2157 .line_comment_zig_fmt_on => try a.addSource("//zig fmt: on\n"),
2158 }
2159
2160 if (a.smith.eos()) break;
2161 }
2162 }
2163}
2164
2165const bin_weights: []const Weight = &.{.rangeAtMost(u8, '0', '1', 1)};
2166const oct_weights: []const Weight = &.{.rangeAtMost(u8, '0', '7', 1)};
2167const dec_weights: []const Weight = &.{.rangeAtMost(u8, '0', '9', 1)};
2168const hex_weights: []const Weight = &.{
2169 .rangeAtMost(u8, '0', '9', 1),
2170 .rangeAtMost(u8, 'a', 'f', 1),
2171 .rangeAtMost(u8, 'A', 'F', 1),
2172};
2173
2174/// Asserts enough capacity for at `min + reserved_capacity`
2175fn variableChar(
2176 a: *AstSmith,
2177 reserved_capacity: usize,
2178 min: usize,
2179 weights: []const Weight,
2180) []u8 {
2181 const capacity = a.sourceCapacity();
2182 const max_out = capacity.len - reserved_capacity;
2183
2184 const len_weights: [3]Weight = .{
2185 .rangeAtMost(u32, @intCast(min), @min(2, max_out), 32678),
2186 // For the below `.rangeAtMost` is not used because max may be less than min.
2187 // In this case, the weights are omitted.
2188 .{ .min = 3, .max = @min(16, max_out), .weight = 512 },
2189 // Still allow much longer sequences to test parsing overflows
2190 .{ .min = 17, .max = @min(256, max_out), .weight = 1 },
2191 };
2192 const n_weights = @as(usize, 1) + @intFromBool(max_out >= 3) + @intFromBool(max_out >= 17);
2193
2194 const len = a.smith.sliceWeighted(capacity, len_weights[0..n_weights], weights);
2195 a.source_len += len;
2196 return capacity[0..len];
2197}
2198
2199/// char_escape
2200/// <- "\\x" hex hex
2201/// / "\\u{" hex+ "}"
2202/// / "\\" [nr\\t'"]
2203/// char_char
2204/// <- multibyte_utf8
2205/// / char_escape
2206/// / ![\\'\n] non_control_ascii
2207///
2208/// string_char
2209/// <- multibyte_utf8
2210/// / char_escape
2211/// / ![\\"\n] non_control_ascii
2212fn pegChar(a: *AstSmith, quote: u8) SourceError!void {
2213 const Char = enum(u8) {
2214 ascii,
2215 unicode_2,
2216 unicode_3,
2217 unicode_4,
2218 hex_escape,
2219 unicode_escape,
2220 char_escape,
2221 };
2222 const weights = Smith.baselineWeights(Char) ++ &[_]Weight{.value(Char, .ascii, 32)};
2223 switch (a.smith.valueWeighted(Char, weights)) {
2224 .ascii => try a.addSourceByte(a.smith.valueWeighted(u8, &.{
2225 .rangeAtMost(u8, ' ', quote - 1, 1),
2226 .rangeAtMost(u8, quote + 1, '\\' - 1, 1),
2227 .rangeAtMost(u8, '\\' + 1, 0x7e, 1),
2228 })),
2229 .unicode_2 => assert(2 == std.unicode.wtf8Encode(
2230 a.smith.valueRangeLessThan(u21, 0x80, 0x800),
2231 try a.addSourceAsSlice(2),
2232 ) catch unreachable),
2233 .unicode_3 => assert(3 == std.unicode.wtf8Encode(
2234 a.smith.valueRangeLessThan(u21, 0x800, 0x10000),
2235 try a.addSourceAsSlice(3),
2236 ) catch unreachable),
2237 .unicode_4 => assert(4 == std.unicode.wtf8Encode(
2238 a.smith.valueRangeLessThan(u21, 0x10000, 0x110000),
2239 try a.addSourceAsSlice(4),
2240 ) catch unreachable),
2241 .hex_escape => {
2242 try a.ensureSourceCapacity(4);
2243 a.addSourceAssumeCapacity("\\x");
2244 a.smith.bytesWeighted(a.addSourceAsSliceAssumeCapacity(2), hex_weights);
2245 },
2246 .unicode_escape => {
2247 try a.ensureSourceCapacity(5);
2248 a.addSourceAssumeCapacity("\\u{");
2249 _ = a.variableChar(1, 1, hex_weights);
2250 a.addSourceByteAssumeCapacity('}');
2251 },
2252 .char_escape => {
2253 try a.ensureSourceCapacity(2);
2254 a.addSourceByteAssumeCapacity('\\');
2255 a.addSourceByteAssumeCapacity(a.smith.valueWeighted(u8, &.{
2256 .value(u8, 'n', 1),
2257 .value(u8, 'r', 1),
2258 .value(u8, 't', 1),
2259 .value(u8, '\\', 1),
2260 .value(u8, '\'', 1),
2261 .value(u8, '"', 1),
2262 }));
2263 },
2264 }
2265}
2266
2267/// CHAR_LITERAL <- ['] char_char ['] skip
2268fn pegCharLiteral(a: *AstSmith) SourceError!void {
2269 try a.addTokenTag(.char_literal);
2270 try a.addSourceByte('\'');
2271 try a.pegChar('\'');
2272 try a.addSourceByte('\'');
2273 try a.pegSkip();
2274}
2275
2276///FLOAT
2277/// <- '0x' hex_int '.' hex_int ([pP] [-+]? dec_int)? skip
2278/// / dec_int '.' dec_int ([eE] [-+]? dec_int)? skip
2279/// / '0x' hex_int [pP] [-+]? dec_int skip
2280/// / dec_int [eE] [-+]? dec_int skip
2281fn pegFloat(a: *AstSmith) SourceError!void {
2282 try a.preservePegEndOfWord();
2283 try a.addTokenTag(.number_literal);
2284
2285 const hex = a.smith.value(bool);
2286 const exp = a.smith.value(packed struct(u3) {
2287 kind: enum(u2) { none, no_sign, minus, plus },
2288 upper: bool,
2289 });
2290 const dot = exp.kind == .none or a.smith.value(bool);
2291
2292 var reserved: usize = @intFromBool(hex) * "0x".len + "0".len + @intFromBool(dot) * ".0".len +
2293 switch (exp.kind) {
2294 .none => 0,
2295 .no_sign => "e0".len,
2296 .minus => "e-0".len,
2297 .plus => "e+0".len,
2298 };
2299 try a.ensureSourceCapacity(reserved);
2300
2301 if (hex) {
2302 reserved -= 2;
2303 a.addSourceAssumeCapacity("0x");
2304 }
2305 const digits = if (hex) hex_weights else dec_weights;
2306
2307 reserved -= 1;
2308 _ = a.variableChar(reserved, 1, digits);
2309
2310 if (dot) {
2311 reserved -= 2;
2312 a.addSourceByteAssumeCapacity('.');
2313 _ = a.variableChar(reserved, 1, digits);
2314 }
2315
2316 if (exp.kind != .none) {
2317 reserved -= 1;
2318 const case_diff = @as(u8, 'a' - 'A') * @intFromBool(exp.upper);
2319 a.addSourceByteAssumeCapacity(@as(u8, if (hex) 'p' else 'e') - case_diff);
2320
2321 if (exp.kind != .no_sign) {
2322 reserved -= 1;
2323 a.addSourceByteAssumeCapacity(if (exp.kind == .plus) '+' else '-');
2324 }
2325
2326 reserved -= 1;
2327 assert(reserved == 0);
2328 _ = a.variableChar(reserved, 1, dec_weights);
2329 }
2330}
2331
2332///INTEGER
2333/// <- '0b' bin_int skip
2334/// / '0o' oct_int skip
2335/// / '0x' hex_int skip
2336/// / dec_int skip
2337fn pegInteger(a: *AstSmith) SourceError!void {
2338 try a.preservePegEndOfWord();
2339 try a.addTokenTag(.number_literal);
2340 const Base = enum { bin, dec, oct, hex };
2341 const base_weights: []const Weight = Smith.baselineWeights(Base) ++
2342 &[_]Weight{ .value(Base, .dec, 6), .value(Base, .hex, 2) };
2343 const digits, const prefix = switch (a.smith.valueWeighted(Base, base_weights)) {
2344 .bin => .{ bin_weights, "0b" },
2345 .oct => .{ oct_weights, "0o" },
2346 .dec => .{ dec_weights, "" },
2347 .hex => .{ hex_weights, "0x" },
2348 };
2349 try a.ensureSourceCapacity(prefix.len + 1);
2350 if (prefix.len != 0) a.addSourceAssumeCapacity(prefix);
2351 _ = a.variableChar(0, 1, digits);
2352}
2353
2354/// Does not include 'skip'. Does not add any token tag.
2355fn stringLiteralSingleInner(a: *AstSmith) SourceError!void {
2356 try a.addSourceByte('"');
2357 while (!a.smith.eosWeightedSimple(3, 1)) {
2358 try a.pegChar('"');
2359 }
2360 try a.addSourceByte('"');
2361}
2362
2363/// STRINGLITERALSINGLE <- ["] string_char* ["] skip
2364fn pegStringLiteralSingle(a: *AstSmith) SourceError!void {
2365 try a.addTokenTag(.string_literal);
2366 try a.stringLiteralSingleInner();
2367 try a.pegSkip();
2368}
2369
2370/// STRINGLITERAL
2371/// <- STRINGLITERALSINGLE
2372/// / (line_string skip)+
2373fn pegStringLiteral(a: *AstSmith) SourceError!void {
2374 if (a.smith.value(bool)) {
2375 try a.pegStringLiteralSingle();
2376 } else {
2377 while (true) {
2378 try a.pegLineString();
2379 try a.pegSkip();
2380 if (a.smith.eos()) break;
2381 }
2382 }
2383}
2384
2385const alphanumeric_weights: [4]Weight = .{
2386 .rangeAtMost(u8, '0', '9', 1),
2387 .rangeAtMost(u8, 'A', 'Z', 1),
2388 .rangeAtMost(u8, 'a', 'z', 1),
2389 .value(u8, '_', 1),
2390};
2391
2392/// IDENTIFIER
2393/// <- !keyword [A-Za-z_] [A-Za-z0-9_]* skip
2394/// / '@' STRINGLITERALSINGLE
2395fn pegIdentifier(a: *AstSmith) SourceError!void {
2396 const Kind = enum(u2) { underscore, regular_identifier, quoted_identifier, copy_identifier };
2397 const kind_weights: [4]Weight = .{
2398 .value(Kind, .underscore, 6),
2399 .value(Kind, .regular_identifier, 3),
2400 .value(Kind, .quoted_identifier, 1),
2401 .value(Kind, .copy_identifier, 6),
2402 };
2403 const n_weights = @as(usize, kind_weights.len) - @intFromBool(a.prev_ids_len == 0);
2404 const kind = a.smith.valueWeighted(Kind, kind_weights[0..n_weights]);
2405
2406 switch (kind) {
2407 .underscore => {
2408 try a.preservePegEndOfWord();
2409 try a.addTokenTag(.identifier);
2410 try a.addSourceByte('_');
2411 },
2412 .regular_identifier => {
2413 try a.preservePegEndOfWord();
2414 try a.addTokenTag(.identifier);
2415
2416 const start = a.source_len;
2417 try a.addSourceByte(a.smith.valueWeighted(u8, alphanumeric_weights[1..]));
2418 _ = a.variableChar(0, 0, &alphanumeric_weights);
2419
2420 if (Token.getKeyword(a.source_buf[start..a.source_len]) != null) {
2421 a.source_buf[start] = '_'; // No keywords start with '_'
2422 }
2423 },
2424 .quoted_identifier => {
2425 try a.addTokenTag(.identifier);
2426 try a.addSourceByte('@');
2427 try a.stringLiteralSingleInner();
2428 },
2429 .copy_identifier => {
2430 const n_prev = @min(a.prev_ids_len, a.prev_ids_buf.len);
2431 const prev_i = a.smith.valueRangeLessThan(u16, 0, n_prev);
2432 const prev = a.prev_ids_buf[prev_i];
2433
2434 if (a.source_buf[prev.start] != '@') try a.preservePegEndOfWord();
2435 try a.addTokenTag(.identifier);
2436 try a.addSource(a.source_buf[prev.start..][0..prev.len]);
2437 },
2438 }
2439 try a.pegSkip();
2440 if (kind != .copy_identifier) {
2441 const start = a.token_start_buf[a.tokens_len - 1];
2442 a.prev_ids_buf[a.prev_ids_len % a.prev_ids_buf.len] = .{
2443 .start = @intCast(start),
2444 .len = @intCast(a.source_len - start),
2445 };
2446 a.prev_ids_len += 1;
2447 }
2448}
2449
2450/// BUILTINIDENTIFIER <- '@'[A-Za-z_][A-Za-z0-9_]* skip
2451fn pegBuiltinIdentifier(a: *AstSmith) SourceError!void {
2452 try a.addTokenTag(.builtin);
2453 if (a.smith.boolWeighted(1, 31)) {
2454 if (a.smith.boolWeighted(1, 8)) {
2455 // Pointer cast (reordable with zig fmt)
2456 const ids = [_][]const u8{
2457 "@ptrCast",
2458 "@addrspaceCast",
2459 "@alignCast",
2460 "@constCast",
2461 "@volatileCast",
2462 };
2463 try a.addSource(ids[a.smith.index(ids.len)]);
2464 } else {
2465 const ids = std.zig.BuiltinFn.list.keys();
2466 try a.addSource(ids[a.smith.index(ids.len)]);
2467 }
2468 } else {
2469 try a.ensureSourceCapacity(2);
2470 a.addSourceByteAssumeCapacity('@');
2471 a.addSourceByteAssumeCapacity(a.smith.valueWeighted(u8, alphanumeric_weights[1..]));
2472 _ = a.variableChar(0, 0, &alphanumeric_weights);
2473 }
2474 try a.pegSkip();
2475}
2476
2477test AstSmith {
2478 try std.testing.fuzz({}, checkGenerated, .{});
2479}
2480
2481fn checkGenerated(_: void, smith: *Smith) !void {
2482 var a: AstSmith = .init(smith);
2483 try a.generateSource();
2484
2485 { // Check tokenization matches source
2486 errdefer a.logBadSource(null);
2487
2488 const token_tags = a.token_tag_buf[0..a.tokens_len];
2489 const token_starts = a.token_start_buf[0..a.tokens_len];
2490 try std.testing.expectEqual(Token.Tag.eof, token_tags[token_tags.len - 1]);
2491
2492 var tokenizer: std.zig.Tokenizer = .init(a.source());
2493 for (token_tags, token_starts) |tag, start| {
2494 const tok = tokenizer.next();
2495 try std.testing.expectEqual(tok.tag, tag);
2496 try std.testing.expectEqual(tok.loc.start, start);
2497 if (tag == .invalid) return error.InvalidToken;
2498 }
2499 }
2500
2501 var fba_buf: [1 << 18]u8 = undefined;
2502 var fba: std.heap.FixedBufferAllocator = .init(&fba_buf);
2503 const ast = std.zig.Ast.parseTokens(fba.allocator(), a.source(), a.tokens(), .zig) catch
2504 return error.SkipZigTest;
2505
2506 errdefer a.logBadSource(ast);
2507 try std.testing.expectEqual(0, ast.errors.len);
2508}
2509
2510fn logBadSource(a: *AstSmith, ast: ?std.zig.Ast) void {
2511 var buf: [256]u8 = undefined;
2512 const ls = std.debug.lockStderr(&buf);
2513 defer std.debug.unlockStderr();
2514 a.logBadSourceInner(ls.terminal(), ast) catch {};
2515}
2516
2517fn logBadSourceInner(a: *AstSmith, t: std.Io.Terminal, ast: ?std.zig.Ast) std.Io.Writer.Error!void {
2518 try a.logSourceInner(t);
2519 const w = t.writer;
2520
2521 if (ast) |bad_ast| {
2522 try w.writeAll("=== Parse Errors ===\n");
2523 for (bad_ast.errors) |err| {
2524 const loc = bad_ast.tokenLocation(0, err.token);
2525 try w.print("{}:{}: ", .{ loc.line + 1, loc.column + 1 });
2526 try bad_ast.renderError(err, w);
2527 try w.writeByte('\n');
2528 }
2529 } else {
2530 t.setColor(.dim) catch {};
2531 try w.writeAll("=== Tokens ===\n");
2532 t.setColor(.reset) catch {};
2533 for (
2534 0..,
2535 a.token_tag_buf[0..a.tokens_len],
2536 a.token_start_buf[0..a.tokens_len],
2537 ) |i, tag, start| {
2538 try w.print("#{} @{}: {t}\n", .{ i, start, tag });
2539 }
2540
2541 t.setColor(.dim) catch {};
2542 try w.writeAll("\n=== Expected Tokens ===\n");
2543 t.setColor(.reset) catch {};
2544
2545 var tokenizer: std.zig.Tokenizer = .init(a.source());
2546 var i: usize = 0;
2547 while (true) {
2548 const tok = tokenizer.next();
2549 try w.print("#{} @{}-{}: {t}\n", .{ i, tok.loc.start, tok.loc.end, tok.tag });
2550 i += 1;
2551 if (tok.tag == .invalid or tok.tag == .eof) break;
2552 }
2553 }
2554}
2555
2556pub fn logSource(a: *AstSmith) void {
2557 var buf: [256]u8 = undefined;
2558 const ls = std.debug.lockStderr(&buf);
2559 defer std.debug.unlockStderr();
2560 a.logSourceInner(ls.terminal()) catch {};
2561}
2562
2563fn logSourceInner(a: *AstSmith, t: std.Io.Terminal) std.Io.Writer.Error!void {
2564 const w = t.writer;
2565
2566 t.setColor(.dim) catch {};
2567 try w.writeAll("=== Source ===\n");
2568 t.setColor(.reset) catch {};
2569
2570 var line: usize = 1;
2571 try w.print("{: >5} ", .{line});
2572 for (a.source()) |c| switch (c) {
2573 ' '...0x7e => try w.writeByte(c),
2574 '\n' => {
2575 line += 1;
2576 try w.print("\n{: >5} ", .{line});
2577 },
2578 '\r' => {
2579 t.setColor(.cyan) catch {};
2580 try w.writeAll("\\r");
2581 t.setColor(.reset) catch {};
2582 },
2583 '\t' => {
2584 t.setColor(.cyan) catch {};
2585 try w.writeAll("\\t");
2586 t.setColor(.reset) catch {};
2587 },
2588 else => {
2589 t.setColor(.cyan) catch {};
2590 try w.print("\\x{x:0>2}", .{c});
2591 t.setColor(.reset) catch {};
2592 },
2593 };
2594 try w.writeByte('\n');
2595}
lib/std/zig/parser_test.zig-263
......@@ -7335,266 +7335,3 @@ fn fuzzTestOneParse(_: void, smith: *std.testing.Smith) !void {
73357335 var fba: std.heap.FixedBufferAllocator = .init(&fixed_buffer_mem);
73367336 _ = std.zig.Ast.parseTokens(fba.allocator(), tokens.source(), tokens.list(), mode) catch return;
73377337}
7338
7339test "zig fmt: fuzz" {
7340 try std.testing.fuzz({}, fuzzRender, .{});
7341}
7342
7343fn isRewritable(source: []const u8, tokens: std.zig.Ast.TokenList.Slice) !bool {
7344 @disableInstrumentation();
7345
7346 // Byte-order marker is stripped
7347 var maybe_rewritable = std.mem.startsWith(u8, source, "\xEF\xBB\xBF");
7348 // The above variable can not yet be replaced by returns since error.SkipZigTest still needs to
7349 // be checked for.
7350
7351 for (0.., tokens.items(.tag), tokens.items(.start)) |i, tag, start| switch (tag) {
7352 // Extra colons can be removed
7353 .keyword_asm,
7354 // Qualifiers can be reordered
7355 // keyword_const is intentionally excluded since it is used in other contexts and
7356 // having only one qualifier will never lead to reordering.
7357 .keyword_addrspace,
7358 .keyword_align,
7359 .keyword_allowzero,
7360 .keyword_callconv,
7361 .keyword_linksection,
7362 .keyword_volatile,
7363 => maybe_rewritable = true,
7364 .builtin,
7365 // Pointer casts can be reordered
7366 => for ([_][]const u8{
7367 "ptrCast",
7368 "alignCast",
7369 "addrSpaceCast",
7370 "constCast",
7371 "volatileCast",
7372 }) |id| {
7373 if (std.mem.startsWith(u8, source[start + 1 ..], id)) {
7374 maybe_rewritable = true;
7375 }
7376 },
7377 // Quoted identifiers can be unquoted
7378 .identifier => if (source[start] == '@') {
7379 maybe_rewritable = true;
7380 },
7381 else => {},
7382 // #23754
7383 .container_doc_comment,
7384 => if (std.mem.endsWith(Token.Tag, tokens.items(.tag)[0..i], &.{.l_brace})) {
7385 return error.SkipZigTest; // Can cause I.B.
7386 },
7387 // #24507
7388 .keyword_inline,
7389 .keyword_for,
7390 .keyword_while,
7391 .l_brace,
7392 => if (std.mem.endsWith(Token.Tag, tokens.items(.tag)[0..i], &.{ .identifier, .colon })) {
7393 return error.SkipZigTest; // Can cause I.B. due to double rendering of zig fmt on/off
7394 },
7395 };
7396
7397 return maybe_rewritable;
7398}
7399
7400/// Checks equivelence of non-whitespace characters.
7401/// If there are commas in `source`, then it is checked they are also present
7402/// in `rendered`. Extra commas in `rendered` are ignored.
7403fn isRewritten(source: [:0]const u8, rendered: [:0]const u8) bool {
7404 @disableInstrumentation();
7405 var i: usize = 0;
7406 for (source[0 .. source.len + 1]) |c| switch (c) {
7407 ' ', '\r', '\t', '\n' => {},
7408 else => while (true) {
7409 defer i += 1;
7410 switch (rendered[i]) {
7411 ' ', '\n' => {},
7412 ',' => if (c == ',') break,
7413 else => |r| if (c != r) return false else break,
7414 }
7415 },
7416 };
7417 std.debug.assert(i >= rendered.len);
7418 return false;
7419}
7420
7421/// Checks that no line ends in whitespace
7422fn checkBetweenTokens(src: []const u8, fmt_on: *bool) error{
7423 TrailingLineWhitespace,
7424 DoubleEmptyLine,
7425}!void {
7426 @disableInstrumentation();
7427 var pos: usize = 0;
7428 while (true) {
7429 const nl_pos = std.mem.indexOfScalarPos(u8, src, pos, '\n');
7430 var check_trailing = fmt_on.*;
7431
7432 const line = src[pos .. nl_pos orelse src.len];
7433 if (std.mem.indexOfScalar(u8, line, '/')) |comment_start| {
7434 const comment_content = line[comment_start..][2..];
7435 const trimmed_comment = std.mem.trim(u8, comment_content, &std.ascii.whitespace);
7436 if (std.mem.eql(u8, trimmed_comment, "zig fmt: off")) {
7437 fmt_on.* = false;
7438 } else if (std.mem.eql(u8, trimmed_comment, "zig fmt: on")) {
7439 fmt_on.* = true;
7440 check_trailing = true;
7441 }
7442 }
7443
7444 pos = nl_pos orelse break;
7445 if (check_trailing and pos != 0) switch (src[pos - 1]) {
7446 ' ', '\t', '\r' => return error.TrailingLineWhitespace,
7447 '\n' => if (pos != 1 and src[pos - 2] == '\n') return error.DoubleEmptyLine,
7448 else => {},
7449 };
7450 pos += 1;
7451 }
7452}
7453
7454/// Ignores extre `.comma` tokens in `rendered`
7455fn reparseTokens(
7456 fba: Allocator,
7457 rendered: [:0]const u8,
7458 expected_tags: [:.eof]const Token.Tag,
7459) error{
7460 OutOfMemory,
7461 SameLineMultilineStringLiteral,
7462 TrailingLineWhitespace,
7463 DoubleEmptyLine,
7464}!struct {
7465 toks: std.zig.Ast.TokenList,
7466 rewritten: bool,
7467} {
7468 @disableInstrumentation();
7469 var rewritten = false;
7470 var tokens: std.zig.Ast.TokenList = .{};
7471 var last_token_end: usize = 0;
7472 var fmt_on = true;
7473
7474 try tokens.ensureTotalCapacity(fba, expected_tags.len + 2); // 1 for EOF and 1 for maybe a comma
7475 var tokenizer: std.zig.Tokenizer = .init(rendered);
7476 var i: usize = 0;
7477 while (true) {
7478 const tok = tokenizer.next();
7479 try tokens.append(fba, .{
7480 .tag = tok.tag,
7481 .start = @intCast(tok.loc.start),
7482 });
7483
7484 const between = rendered[last_token_end..tok.loc.start];
7485 last_token_end = tok.loc.end;
7486 try checkBetweenTokens(between, &fmt_on);
7487 if (tok.tag == .multiline_string_literal_line and fmt_on) blk: {
7488 if (tokens.len == 1)
7489 break :blk; // first token
7490 if (std.mem.indexOfScalar(u8, between, '\n') == null)
7491 return error.SameLineMultilineStringLiteral;
7492 }
7493 if (tok.tag == expected_tags[i]) {
7494 if (tok.tag == .eof)
7495 break;
7496 i += 1;
7497 } else if (tok.tag != .comma or !fmt_on) {
7498 rewritten = true;
7499 }
7500 }
7501 std.debug.assert(i == expected_tags.len);
7502 try checkBetweenTokens(rendered[last_token_end..], &fmt_on);
7503
7504 return .{ .toks = tokens, .rewritten = rewritten };
7505}
7506
7507fn fuzzRender(_: void, smith: *std.testing.Smith) !void {
7508 @disableInstrumentation();
7509
7510 var ast_smith: std.zig.AstSmith = .init(smith);
7511 try ast_smith.generateSource();
7512 var fba_ctx = std.heap.FixedBufferAllocator.init(&fixed_buffer_mem);
7513 var opt_rendered: ?[]const u8 = null;
7514 fuzzRenderInner(&ast_smith, fba_ctx.allocator(), &opt_rendered) catch |e| switch (e) {
7515 error.SkipZigTest, error.OutOfMemory, error.WriteFailed => return error.SkipZigTest,
7516 else => |failure| {
7517 ast_smith.logSource();
7518 if (opt_rendered) |rendered| {
7519 logRenderedSource(rendered);
7520 }
7521 return failure;
7522 },
7523 };
7524}
7525
7526fn fuzzRenderInner(ast_smith: *std.zig.AstSmith, fba: Allocator, opt_rendered: *?[]const u8) !void {
7527 @disableInstrumentation();
7528
7529 const source = ast_smith.source();
7530 const src_rewritable = try isRewritable(source, ast_smith.tokens());
7531 const src_tree = try std.zig.Ast.parseTokens(fba, source, ast_smith.tokens(), .zig);
7532 std.debug.assert(src_tree.errors.len == 0);
7533 for (src_tree.nodes.items(.tag)) |tag| switch (tag) {
7534 // #24507 (`switch(x) { inline for (a) |a| a => {} }` to
7535 // `switch(x) { { inline for (a) |a| a => {} }` since
7536 // AST determines inline case token as one before the case expression's first)
7537 .switch_case_inline, .switch_case_inline_one => return error.SkipZigTest,
7538 else => {},
7539 };
7540
7541 var rendered_w: std.Io.Writer.Allocating = .init(fba);
7542 try rendered_w.ensureUnusedCapacity(source.len + source.len / 2);
7543 try src_tree.render(fba, &rendered_w.writer, .{});
7544 // `toOwnedSliceSentinel` is not used since it reallocates the entire
7545 // list to save space which is useless for fixed buffer allocators.
7546 try rendered_w.writer.writeByte(0);
7547 const rendered = rendered_w.written()[0 .. rendered_w.written().len - 1 :0];
7548 opt_rendered.* = rendered;
7549
7550 // First check that the non-whitespace characters match. This ensures that
7551 // identifier names, numbers, comments, et cetera are preserved.
7552 if (!src_rewritable and isRewritten(source, rendered))
7553 return error.Rewritten;
7554 // Next check that the tokens are the same since whitespace removal can change the tokens
7555 const src_tags = ast_smith.tokens().items(.tag);
7556 const rendered_toks = try reparseTokens(fba, rendered, src_tags[0 .. src_tags.len - 1 :.eof]);
7557 if (!src_rewritable and rendered_toks.rewritten)
7558 return error.Rewritten;
7559
7560 // Rerender the tree to check idempotency and that new commas
7561 // and whitespace changes did not create an AST error.
7562 const rendered_tree = try std.zig.Ast.parseTokens(fba, rendered, rendered_toks.toks.slice(), .zig);
7563 if (rendered_tree.errors.len != 0)
7564 return error.Rewritten;
7565 var rerendered_w: std.Io.Writer.Allocating = .init(fba);
7566 try rerendered_w.ensureUnusedCapacity(source.len);
7567 try rendered_tree.render(fba, &rerendered_w.writer, .{});
7568 try std.testing.expectEqualStrings(rendered, rerendered_w.written());
7569}
7570
7571fn logRenderedSource(source: []const u8) void {
7572 var buf: [256]u8 = undefined;
7573 const ls = std.debug.lockStderr(&buf);
7574 defer std.debug.unlockStderr();
7575 logRenderedSourceInner(source, ls.terminal()) catch {};
7576}
7577
7578fn logRenderedSourceInner(source: []const u8, t: std.Io.Terminal) std.Io.Writer.Error!void {
7579 const w = t.writer;
7580
7581 t.setColor(.dim) catch {};
7582 try w.writeAll("=== Rendered Source ===\n");
7583 t.setColor(.reset) catch {};
7584
7585 for (0.., source) |i, c| switch (c) {
7586 ' '...0x7e => try w.writeByte(c),
7587 '\n' => {
7588 if (i != 0 and source[i - 1] == ' ') {
7589 try w.writeAll("⏎");
7590 }
7591 try w.writeByte('\n');
7592 },
7593 else => {
7594 t.setColor(.cyan) catch {};
7595 try w.print("\\x{x:0>2}", .{c});
7596 t.setColor(.reset) catch {};
7597 },
7598 };
7599 try w.writeAll("␃\n");
7600}