authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-03 13:22:52+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-08 11:49:00+02:00
log3681c7c506829322a2980d5bd98e493968fdea68
treee2cfcf60efe2487a97038263b19b00ea34aadd85
parent04cb341cbfadcc9ed22202e0e48282e92bfd3f69
signaturelock-open Commit is signed but in an unrecognized format.

std.zig, grammar: agree on pointer modifier parsing

Currently the grammar only allows exactly one pointer modifier order but the Parse.zig implementation allows any order. However, the current Parse.zig behavior of allowing any order while forbidding duplicate modifiers would require a combinatorial explosion of grammar rules to specify (currently 5! i.e. 120). The simplest, most permissive grammar would remove the ordering requirement and allow duplicate pointer modifiers. However, this would unfortunately require a more complex AST data layout due to the child nodes of align() and addrspace() modifiers. Since these are the only pointer modifiers with child nodes and 2! is a perfectly reasonable number, forbid duplicate align() and addrspace() modifiers to keep the Ast data structure simple. In order to resolve the difference between the formal grammar and parser implementation, allow duplicate single-token modifiers (allowzero, const, volatile) in the grammar and move the compile error for that case to AstGen. Note that zig fmt will now silently remove duplicate single-token modifiers, which I think is a nice little UX improvement.

7 files changed, 143 insertions(+), 49 deletions(-)

doc/langref/grammar.peg+21-2
...@@ -364,10 +364,29 @@ PrefixOp...@@ -364,10 +364,29 @@ PrefixOp
364PrefixTypeOp364PrefixTypeOp
365 <- QUESTIONMARK365 <- QUESTIONMARK
366 / KEYWORD_anyframe MINUSRARROW366 / KEYWORD_anyframe MINUSRARROW
367 / (ManyPtrTypeStart / SliceTypeStart) KEYWORD_allowzero? ByteAlign? AddrSpace? KEYWORD_const? KEYWORD_volatile?367 / (ManyPtrTypeStart / SliceTypeStart) PtrMods
368 / SinglePtrTypeStart KEYWORD_allowzero? BitAlign? AddrSpace? KEYWORD_const? KEYWORD_volatile?368 / SinglePtrTypeStart SinglePtrMods
369 / ArrayTypeStart369 / ArrayTypeStart
370370
371# Forbid more than one align or addrspace pointer modifier since these
372# modifiers contain sub-expressions. This allows for a simpler AST data layout.
373# Allow duplicate single-token modifiers (allowzero/const/volatile) in the grammar
374# to avoid the combinatorial explosion of grammar rules necessary to forbid duplicates
375# while permitting arbitrary order. A compile error for duplicates single-token modifiers
376# is emitted during "AstGen" after parsing is complete.
377PtrMods
378 <- PtrMod* ByteAlign? PtrMod* AddrSpace? PtrMod*
379 / PtrMod* AddrSpace? PtrMod* ByteAlign? PtrMod*
380
381SinglePtrMods
382 <- PtrMod* BitAlign? PtrMod* AddrSpace? PtrMod*
383 / PtrMod* AddrSpace? PtrMod* BitAlign? PtrMod*
384
385PtrMod
386 <- KEYWORD_allowzero
387 / KEYWORD_const
388 / KEYWORD_volatile
389
371PrefixTypeOpPrefix390PrefixTypeOpPrefix
372 <- QUESTIONMARK391 <- QUESTIONMARK
373 / KEYWORD_anyframe MINUSRARROW392 / KEYWORD_anyframe MINUSRARROW
lib/std/zig/Ast.zig+20-15
...@@ -465,15 +465,6 @@ pub fn renderError(tree: Ast, parse_error: Error, w: *Writer) Writer.Error!void...@@ -465,15 +465,6 @@ pub fn renderError(tree: Ast, parse_error: Error, w: *Writer) Writer.Error!void
465 .extra_align_qualifier => {465 .extra_align_qualifier => {
466 return w.writeAll("extra align qualifier");466 return w.writeAll("extra align qualifier");
467 },467 },
468 .extra_allowzero_qualifier => {
469 return w.writeAll("extra allowzero qualifier");
470 },
471 .extra_const_qualifier => {
472 return w.writeAll("extra const qualifier");
473 },
474 .extra_volatile_qualifier => {
475 return w.writeAll("extra volatile qualifier");
476 },
477 .ptr_mod_on_array_child_type => {468 .ptr_mod_on_array_child_type => {
478 return w.print("pointer modifier '{s}' not allowed on array child type", .{469 return w.print("pointer modifier '{s}' not allowed on array child type", .{
479 tree.tokenTag(parse_error.token).symbol(),470 tree.tokenTag(parse_error.token).symbol(),
...@@ -2110,6 +2101,7 @@ fn fullPtrTypeComponents(tree: Ast, info: full.PtrType.Components) full.PtrType...@@ -2110,6 +2101,7 @@ fn fullPtrTypeComponents(tree: Ast, info: full.PtrType.Components) full.PtrType
2110 .allowzero_token = null,2101 .allowzero_token = null,
2111 .const_token = null,2102 .const_token = null,
2112 .volatile_token = null,2103 .volatile_token = null,
2104 .duplicate_token = null,
2113 .ast = info,2105 .ast = info,
2114 };2106 };
2115 // We need to be careful that we don't iterate over any sub-expressions2107 // We need to be careful that we don't iterate over any sub-expressions
...@@ -2125,9 +2117,24 @@ fn fullPtrTypeComponents(tree: Ast, info: full.PtrType.Components) full.PtrType...@@ -2125,9 +2117,24 @@ fn fullPtrTypeComponents(tree: Ast, info: full.PtrType.Components) full.PtrType
2125 const end = tree.firstToken(info.child_type);2117 const end = tree.firstToken(info.child_type);
2126 while (i < end) : (i += 1) {2118 while (i < end) : (i += 1) {
2127 switch (tree.tokenTag(i)) {2119 switch (tree.tokenTag(i)) {
2128 .keyword_allowzero => result.allowzero_token = i,2120 .keyword_allowzero => {
2129 .keyword_const => result.const_token = i,2121 if (result.allowzero_token != null) {
2130 .keyword_volatile => result.volatile_token = i,2122 result.duplicate_token = i;
2123 }
2124 result.allowzero_token = i;
2125 },
2126 .keyword_const => {
2127 if (result.const_token != null) {
2128 result.duplicate_token = i;
2129 }
2130 result.const_token = i;
2131 },
2132 .keyword_volatile => {
2133 if (result.volatile_token != null) {
2134 result.duplicate_token = i;
2135 }
2136 result.volatile_token = i;
2137 },
2131 .keyword_align => {2138 .keyword_align => {
2132 if (info.bit_range_end.unwrap()) |bit_range_end| {2139 if (info.bit_range_end.unwrap()) |bit_range_end| {
2133 assert(info.bit_range_start != .none);2140 assert(info.bit_range_start != .none);
...@@ -2724,6 +2731,7 @@ pub const full = struct {...@@ -2724,6 +2731,7 @@ pub const full = struct {
2724 allowzero_token: ?TokenIndex,2731 allowzero_token: ?TokenIndex,
2725 const_token: ?TokenIndex,2732 const_token: ?TokenIndex,
2726 volatile_token: ?TokenIndex,2733 volatile_token: ?TokenIndex,
2734 duplicate_token: ?TokenIndex,
2727 ast: Components,2735 ast: Components,
27282736
2729 pub const Components = struct {2737 pub const Components = struct {
...@@ -2857,9 +2865,6 @@ pub const Error = struct {...@@ -2857,9 +2865,6 @@ pub const Error = struct {
2857 extern_fn_body,2865 extern_fn_body,
2858 extra_addrspace_qualifier,2866 extra_addrspace_qualifier,
2859 extra_align_qualifier,2867 extra_align_qualifier,
2860 extra_allowzero_qualifier,
2861 extra_const_qualifier,
2862 extra_volatile_qualifier,
2863 ptr_mod_on_array_child_type,2868 ptr_mod_on_array_child_type,
2864 invalid_bit_range,2869 invalid_bit_range,
2865 same_line_doc_comment,2870 same_line_doc_comment,
lib/std/zig/AstGen.zig+3
...@@ -3745,6 +3745,9 @@ fn ptrType(...@@ -3745,6 +3745,9 @@ fn ptrType(
3745 if (ptr_info.size == .c and ptr_info.allowzero_token != null) {3745 if (ptr_info.size == .c and ptr_info.allowzero_token != null) {
3746 return gz.astgen.failTok(ptr_info.allowzero_token.?, "C pointers always allow address zero", .{});3746 return gz.astgen.failTok(ptr_info.allowzero_token.?, "C pointers always allow address zero", .{});
3747 }3747 }
3748 if (ptr_info.duplicate_token) |duplicate| {
3749 return gz.astgen.failTok(duplicate, "Extra pointer qualifier", .{});
3750 }
37483751
3749 const source_offset = gz.astgen.source_offset;3752 const source_offset = gz.astgen.source_offset;
3750 const source_line = gz.astgen.source_line;3753 const source_line = gz.astgen.source_line;
lib/std/zig/Parse.zig+22-28
...@@ -1691,8 +1691,8 @@ fn expectPrefixExpr(p: *Parse) Error!Node.Index {...@@ -1691,8 +1691,8 @@ fn expectPrefixExpr(p: *Parse) Error!Node.Index {
1691/// PrefixTypeOp1691/// PrefixTypeOp
1692/// <- QUESTIONMARK1692/// <- QUESTIONMARK
1693/// / KEYWORD_anyframe MINUSRARROW1693/// / KEYWORD_anyframe MINUSRARROW
1694/// / (ManyPtrTypeStart / SliceTypeStart) KEYWORD_allowzero? ByteAlign? AddrSpace? KEYWORD_const? KEYWORD_volatile?1694/// / (ManyPtrTypeStart / SliceTypeStart) PtrMods
1695/// / SinglePtrTypeStart KEYWORD_allowzero? BitAlign? AddrSpace? KEYWORD_const? KEYWORD_volatile?1695/// / SinglePtrTypeStart SinglePtrMods
1696/// / ArrayTypeStart1696/// / ArrayTypeStart
1697///1697///
1698/// PrefixTypeOpPrefix1698/// PrefixTypeOpPrefix
...@@ -1708,8 +1708,6 @@ fn expectPrefixExpr(p: *Parse) Error!Node.Index {...@@ -1708,8 +1708,6 @@ fn expectPrefixExpr(p: *Parse) Error!Node.Index {
1708/// ManyPtrTypeStart <- LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET1708/// ManyPtrTypeStart <- LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET
1709///1709///
1710/// ArrayTypeStart <- LBRACKET !ASTERISK Expr (COLON Expr)? RBRACKET1710/// ArrayTypeStart <- LBRACKET !ASTERISK Expr (COLON Expr)? RBRACKET
1711///
1712/// BitAlign <- KEYWORD_align LPAREN Expr (COLON Expr COLON Expr)? RPAREN
1713fn parseTypeExpr(p: *Parse) Error!?Node.Index {1711fn parseTypeExpr(p: *Parse) Error!?Node.Index {
1714 switch (p.tokenTag(p.tok_i)) {1712 switch (p.tokenTag(p.tok_i)) {
1715 .question_mark => return try p.addNode(.{1713 .question_mark => return try p.addNode(.{
...@@ -3005,6 +3003,22 @@ const PtrModifiers = struct {...@@ -3005,6 +3003,22 @@ const PtrModifiers = struct {
3005 bit_range_end: Node.OptionalIndex,3003 bit_range_end: Node.OptionalIndex,
3006};3004};
30073005
3006/// PtrMods
3007/// <- PtrMod* ByteAlign? PtrMod* AddrSpace? PtrMod*
3008/// / PtrMod* AddrSpace? PtrMod* ByteAlign? PtrMod*
3009///
3010/// SinglePtrMods
3011/// <- PtrMod* BitAlign? PtrMod* AddrSpace? PtrMod*
3012/// / PtrMod* AddrSpace? PtrMod* BitAlign? PtrMod*
3013///
3014/// PtrMod
3015/// <- KEYWORD_allowzero
3016/// / KEYWORD_const
3017/// / KEYWORD_volatile
3018///
3019/// AddrSpace <- KEYWORD_addrspace LPAREN Expr RPAREN
3020/// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN
3021/// BitAlign <- KEYWORD_align LPAREN Expr (COLON Expr COLON Expr)? RPAREN
3008fn parsePtrModifiers(p: *Parse) !PtrModifiers {3022fn parsePtrModifiers(p: *Parse) !PtrModifiers {
3009 var result: PtrModifiers = .{3023 var result: PtrModifiers = .{
3010 .align_node = .none,3024 .align_node = .none,
...@@ -3012,9 +3026,6 @@ fn parsePtrModifiers(p: *Parse) !PtrModifiers {...@@ -3012,9 +3026,6 @@ fn parsePtrModifiers(p: *Parse) !PtrModifiers {
3012 .bit_range_start = .none,3026 .bit_range_start = .none,
3013 .bit_range_end = .none,3027 .bit_range_end = .none,
3014 };3028 };
3015 var saw_const = false;
3016 var saw_volatile = false;
3017 var saw_allowzero = false;
3018 while (true) {3029 while (true) {
3019 switch (p.tokenTag(p.tok_i)) {3030 switch (p.tokenTag(p.tok_i)) {
3020 .keyword_align => {3031 .keyword_align => {
...@@ -3033,33 +3044,16 @@ fn parsePtrModifiers(p: *Parse) !PtrModifiers {...@@ -3033,33 +3044,16 @@ fn parsePtrModifiers(p: *Parse) !PtrModifiers {
30333044
3034 _ = try p.expectToken(.r_paren);3045 _ = try p.expectToken(.r_paren);
3035 },3046 },
3036 .keyword_const => {
3037 if (saw_const) {
3038 try p.warn(.extra_const_qualifier);
3039 }
3040 p.tok_i += 1;
3041 saw_const = true;
3042 },
3043 .keyword_volatile => {
3044 if (saw_volatile) {
3045 try p.warn(.extra_volatile_qualifier);
3046 }
3047 p.tok_i += 1;
3048 saw_volatile = true;
3049 },
3050 .keyword_allowzero => {
3051 if (saw_allowzero) {
3052 try p.warn(.extra_allowzero_qualifier);
3053 }
3054 p.tok_i += 1;
3055 saw_allowzero = true;
3056 },
3057 .keyword_addrspace => {3047 .keyword_addrspace => {
3058 if (result.addrspace_node != .none) {3048 if (result.addrspace_node != .none) {
3059 try p.warn(.extra_addrspace_qualifier);3049 try p.warn(.extra_addrspace_qualifier);
3060 }3050 }
3061 result.addrspace_node = .fromOptional(try p.parseAddrSpace());3051 result.addrspace_node = .fromOptional(try p.parseAddrSpace());
3062 },3052 },
3053 .keyword_allowzero,
3054 .keyword_const,
3055 .keyword_volatile,
3056 => p.tok_i += 1,
3063 else => return result,3057 else => return result,
3064 }3058 }
3065 }3059 }
lib/std/zig/parser_fuzz.zig+5
...@@ -98,6 +98,11 @@ test "dot question" {...@@ -98,6 +98,11 @@ test "dot question" {
98 try checkAgainstOracle("0. ?");98 try checkAgainstOracle("0. ?");
99}99}
100100
101// Found using AFL++
102test "volatile const" {
103 try checkAgainstOracle("*volatile\nconst\n0");
104}
105
101fn checkAgainstOracle(source: [:0]const u8) !void {106fn checkAgainstOracle(source: [:0]const u8) !void {
102 var fba_buf: [1 << 18]u8 = undefined;107 var fba_buf: [1 << 18]u8 = undefined;
103 var fba: std.heap.FixedBufferAllocator = .init(&fba_buf);108 var fba: std.heap.FixedBufferAllocator = .init(&fba_buf);
lib/std/zig/parser_generated_oracle.zig+70-2
...@@ -1751,15 +1751,83 @@ const Parser = struct {...@@ -1751,15 +1751,83 @@ const Parser = struct {
1751 if (p.parseSliceTypeStart()) break :blk_2 true;1751 if (p.parseSliceTypeStart()) break :blk_2 true;
1752 p.i = pos_2;1752 p.i = pos_2;
1753 break :blk_2 false;1753 break :blk_2 false;
1754 } and (p.parseKEYWORD_allowzero() or true) and (p.parseByteAlign() or true) and (p.parseAddrSpace() or true) and (p.parseKEYWORD_const() or true) and (p.parseKEYWORD_volatile() or true)) break :blk_0 true;1754 } and p.parsePtrMods()) break :blk_0 true;
1755 p.i = pos_0;1755 p.i = pos_0;
1756 if (p.parseSinglePtrTypeStart() and (p.parseKEYWORD_allowzero() or true) and (p.parseBitAlign() or true) and (p.parseAddrSpace() or true) and (p.parseKEYWORD_const() or true) and (p.parseKEYWORD_volatile() or true)) break :blk_0 true;1756 if (p.parseSinglePtrTypeStart() and p.parseSinglePtrMods()) break :blk_0 true;
1757 p.i = pos_0;1757 p.i = pos_0;
1758 if (p.parseArrayTypeStart()) break :blk_0 true;1758 if (p.parseArrayTypeStart()) break :blk_0 true;
1759 p.i = pos_0;1759 p.i = pos_0;
1760 break :blk_0 false;1760 break :blk_0 false;
1761 };1761 };
1762 }1762 }
1763 pub fn parsePtrMods(p: *Parser) bool {
1764 return blk_0: {
1765 const pos_0 = p.i;
1766 if (blk_1: {
1767 while (p.parsePtrMod()) {}
1768 break :blk_1 true;
1769 } and (p.parseByteAlign() or true) and blk_1: {
1770 while (p.parsePtrMod()) {}
1771 break :blk_1 true;
1772 } and (p.parseAddrSpace() or true) and blk_1: {
1773 while (p.parsePtrMod()) {}
1774 break :blk_1 true;
1775 }) break :blk_0 true;
1776 p.i = pos_0;
1777 if (blk_1: {
1778 while (p.parsePtrMod()) {}
1779 break :blk_1 true;
1780 } and (p.parseAddrSpace() or true) and blk_1: {
1781 while (p.parsePtrMod()) {}
1782 break :blk_1 true;
1783 } and (p.parseByteAlign() or true) and blk_1: {
1784 while (p.parsePtrMod()) {}
1785 break :blk_1 true;
1786 }) break :blk_0 true;
1787 p.i = pos_0;
1788 break :blk_0 false;
1789 };
1790 }
1791 pub fn parseSinglePtrMods(p: *Parser) bool {
1792 return blk_0: {
1793 const pos_0 = p.i;
1794 if (blk_1: {
1795 while (p.parsePtrMod()) {}
1796 break :blk_1 true;
1797 } and (p.parseBitAlign() or true) and blk_1: {
1798 while (p.parsePtrMod()) {}
1799 break :blk_1 true;
1800 } and (p.parseAddrSpace() or true) and blk_1: {
1801 while (p.parsePtrMod()) {}
1802 break :blk_1 true;
1803 }) break :blk_0 true;
1804 p.i = pos_0;
1805 if (blk_1: {
1806 while (p.parsePtrMod()) {}
1807 break :blk_1 true;
1808 } and (p.parseAddrSpace() or true) and blk_1: {
1809 while (p.parsePtrMod()) {}
1810 break :blk_1 true;
1811 } and (p.parseBitAlign() or true) and blk_1: {
1812 while (p.parsePtrMod()) {}
1813 break :blk_1 true;
1814 }) break :blk_0 true;
1815 p.i = pos_0;
1816 break :blk_0 false;
1817 };
1818 }
1819 pub fn parsePtrMod(p: *Parser) bool {
1820 return blk_0: {
1821 const pos_0 = p.i;
1822 if (p.parseKEYWORD_allowzero()) break :blk_0 true;
1823 p.i = pos_0;
1824 if (p.parseKEYWORD_const()) break :blk_0 true;
1825 p.i = pos_0;
1826 if (p.parseKEYWORD_volatile()) break :blk_0 true;
1827 p.i = pos_0;
1828 break :blk_0 false;
1829 };
1830 }
1763 pub fn parsePrefixTypeOpPrefix(p: *Parser) bool {1831 pub fn parsePrefixTypeOpPrefix(p: *Parser) bool {
1764 return blk_0: {1832 return blk_0: {
1765 const pos_0 = p.i;1833 const pos_0 = p.i;
lib/std/zig/parser_test.zig+2-2
...@@ -6992,10 +6992,10 @@ test "recovery: non-associative operators" {...@@ -6992,10 +6992,10 @@ test "recovery: non-associative operators" {
69926992
6993test "recovery: extra qualifier" {6993test "recovery: extra qualifier" {
6994 try testError(6994 try testError(
6995 \\const a: *const const u8;6995 \\const a: *align(4) align(8) u8;
6996 \\test ""6996 \\test ""
6997 , &[_]Error{6997 , &[_]Error{
6998 .extra_const_qualifier,6998 .extra_align_qualifier,
6999 .expected_block,6999 .expected_block,
7000 });7000 });
7001}7001}