authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-06-11 11:01:27+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-08 11:48:57+02:00
log6101a1372086d374dab7fa930834c5c80f87e4f9
tree918d10c2c54fec7dfe0964e2aab0dc3ad3c09e70
parent25dee862b47e235a6b1e86ea596f91ed57c83f85
signaturelock-open Commit is signed but in an unrecognized format.

grammar: match tokenizer number literal behavior


2 files changed, 112 insertions(+), 291 deletions(-)

doc/langref/grammar.peg+12-25
......@@ -134,17 +134,16 @@ PrimaryTypeExpr
134134 / DOT IDENTIFIER
135135 / DOT InitList
136136 / ErrorSetDecl
137 / FLOAT
138137 / FnProto
139138 / GroupedExpr
140139 / LabeledTypeExpr
141140 / IDENTIFIER !(COLON LabelableExpr)
142141 / IfTypeExpr
143 / INTEGER
144142 / KEYWORD_comptime TypeExpr !ExprSuffix
145143 / KEYWORD_error DOT IDENTIFIER
146144 / KEYWORD_anyframe
147145 / KEYWORD_unreachable
146 / NUMBERLITERAL
148147 / STRINGLITERAL
149148
150149ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto
......@@ -369,19 +368,6 @@ ExprList <- (Expr COMMA)* Expr?
369368
370369# *** Tokens ***
371370eof <- !.
372bin <- [01]
373bin_ <- '_'? bin
374oct <- [0-7]
375oct_ <- '_'? oct
376hex <- [0-9a-fA-F]
377hex_ <- '_'? hex
378dec <- [0-9]
379dec_ <- '_'? dec
380
381bin_int <- bin bin_*
382oct_int <- oct oct_*
383dec_int <- dec dec_*
384hex_int <- hex hex_*
385371
386372ox80_oxBF <- [\200-\277]
387373oxF4 <- '\364'
......@@ -422,6 +408,9 @@ multibyte_utf8 <-
422408non_control_ascii <- [\040-\176]
423409non_control_utf8 <- [\040-\377]
424410
411hex <- [0-9a-fA-F]
412hex_ <- '_'? hex
413
425414char_escape
426415 <- "\\x" hex hex
427416 / "\\u{" hex+ "}"
......@@ -443,16 +432,14 @@ line_string <- '\\\\' non_control_utf8* [ \n]*
443432skip <- ([ \n] / line_comment)*
444433
445434CHAR_LITERAL <- ['] char_char ['] skip
446FLOAT
447 <- '0x' hex_int '.' hex_int ([pP] [-+]? dec_int)? skip
448 / dec_int '.' dec_int ([eE] [-+]? dec_int)? skip
449 / '0x' hex_int [pP] [-+]? dec_int skip
450 / dec_int [eE] [-+]? dec_int skip
451INTEGER
452 <- '0b' bin_int skip
453 / '0o' oct_int skip
454 / '0x' hex_int skip
455 / dec_int skip
435
436digit <- [_0-9A-DF-OQ-Za-df-oq-z]
437digit_int <- digit / [eEpP]
438digit_float <- digit / [eEpP] [-+]?
439NUMBERLITERAL
440 <- [0-9] digit_int* '.' digit_float+ skip
441 / [0-9] digit_float* skip
442
456443STRINGLITERALSINGLE <- ["] string_char* ["] skip
457444STRINGLITERAL
458445 <- STRINGLITERALSINGLE
lib/std/zig/parser_generated_oracle.zig+100-266
......@@ -921,8 +921,6 @@ const Parser = struct {
921921 p.i = pos_0;
922922 if (p.parseErrorSetDecl()) break :blk_0 true;
923923 p.i = pos_0;
924 if (p.parseFLOAT()) break :blk_0 true;
925 p.i = pos_0;
926924 if (p.parseFnProto()) break :blk_0 true;
927925 p.i = pos_0;
928926 if (p.parseGroupedExpr()) break :blk_0 true;
......@@ -943,8 +941,6 @@ const Parser = struct {
943941 p.i = pos_0;
944942 if (p.parseIfTypeExpr()) break :blk_0 true;
945943 p.i = pos_0;
946 if (p.parseINTEGER()) break :blk_0 true;
947 p.i = pos_0;
948944 if (p.parseKEYWORD_comptime() and p.parseTypeExpr() and blk_1: {
949945 const pos_1 = p.i;
950946 const match_1 = p.parseExprSuffix();
......@@ -958,6 +954,8 @@ const Parser = struct {
958954 p.i = pos_0;
959955 if (p.parseKEYWORD_unreachable()) break :blk_0 true;
960956 p.i = pos_0;
957 if (p.parseNUMBERLITERAL()) break :blk_0 true;
958 p.i = pos_0;
961959 if (p.parseSTRINGLITERAL()) break :blk_0 true;
962960 p.i = pos_0;
963961 break :blk_0 false;
......@@ -1880,169 +1878,6 @@ const Parser = struct {
18801878 break :blk_0 false;
18811879 };
18821880 }
1883 pub fn parsebin(p: *Parser) bool {
1884 return blk_0: {
1885 const pos_0 = p.i;
1886 if ((p.i < p.source.len and switch (p.source[p.i]) {
1887 '0'...'0',
1888 '1'...'1',
1889 => blk_1: {
1890 p.i += 1;
1891 break :blk_1 true;
1892 },
1893 else => false,
1894 })) break :blk_0 true;
1895 p.i = pos_0;
1896 break :blk_0 false;
1897 };
1898 }
1899 pub fn parsebin_(p: *Parser) bool {
1900 return blk_0: {
1901 const pos_0 = p.i;
1902 if ((blk_2: {
1903 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
1904 p.i += 1;
1905 break :blk_2 true;
1906 }
1907 break :blk_2 false;
1908 } or true) and p.parsebin()) break :blk_0 true;
1909 p.i = pos_0;
1910 break :blk_0 false;
1911 };
1912 }
1913 pub fn parseoct(p: *Parser) bool {
1914 return blk_0: {
1915 const pos_0 = p.i;
1916 if ((p.i < p.source.len and switch (p.source[p.i]) {
1917 '0'...'7',
1918 => blk_1: {
1919 p.i += 1;
1920 break :blk_1 true;
1921 },
1922 else => false,
1923 })) break :blk_0 true;
1924 p.i = pos_0;
1925 break :blk_0 false;
1926 };
1927 }
1928 pub fn parseoct_(p: *Parser) bool {
1929 return blk_0: {
1930 const pos_0 = p.i;
1931 if ((blk_2: {
1932 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
1933 p.i += 1;
1934 break :blk_2 true;
1935 }
1936 break :blk_2 false;
1937 } or true) and p.parseoct()) break :blk_0 true;
1938 p.i = pos_0;
1939 break :blk_0 false;
1940 };
1941 }
1942 pub fn parsehex(p: *Parser) bool {
1943 return blk_0: {
1944 const pos_0 = p.i;
1945 if ((p.i < p.source.len and switch (p.source[p.i]) {
1946 '0'...'9',
1947 'a'...'f',
1948 'A'...'F',
1949 => blk_1: {
1950 p.i += 1;
1951 break :blk_1 true;
1952 },
1953 else => false,
1954 })) break :blk_0 true;
1955 p.i = pos_0;
1956 break :blk_0 false;
1957 };
1958 }
1959 pub fn parsehex_(p: *Parser) bool {
1960 return blk_0: {
1961 const pos_0 = p.i;
1962 if ((blk_2: {
1963 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
1964 p.i += 1;
1965 break :blk_2 true;
1966 }
1967 break :blk_2 false;
1968 } or true) and p.parsehex()) break :blk_0 true;
1969 p.i = pos_0;
1970 break :blk_0 false;
1971 };
1972 }
1973 pub fn parsedec(p: *Parser) bool {
1974 return blk_0: {
1975 const pos_0 = p.i;
1976 if ((p.i < p.source.len and switch (p.source[p.i]) {
1977 '0'...'9',
1978 => blk_1: {
1979 p.i += 1;
1980 break :blk_1 true;
1981 },
1982 else => false,
1983 })) break :blk_0 true;
1984 p.i = pos_0;
1985 break :blk_0 false;
1986 };
1987 }
1988 pub fn parsedec_(p: *Parser) bool {
1989 return blk_0: {
1990 const pos_0 = p.i;
1991 if ((blk_2: {
1992 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
1993 p.i += 1;
1994 break :blk_2 true;
1995 }
1996 break :blk_2 false;
1997 } or true) and p.parsedec()) break :blk_0 true;
1998 p.i = pos_0;
1999 break :blk_0 false;
2000 };
2001 }
2002 pub fn parsebin_int(p: *Parser) bool {
2003 return blk_0: {
2004 const pos_0 = p.i;
2005 if (p.parsebin() and blk_1: {
2006 while (p.parsebin_()) {}
2007 break :blk_1 true;
2008 }) break :blk_0 true;
2009 p.i = pos_0;
2010 break :blk_0 false;
2011 };
2012 }
2013 pub fn parseoct_int(p: *Parser) bool {
2014 return blk_0: {
2015 const pos_0 = p.i;
2016 if (p.parseoct() and blk_1: {
2017 while (p.parseoct_()) {}
2018 break :blk_1 true;
2019 }) break :blk_0 true;
2020 p.i = pos_0;
2021 break :blk_0 false;
2022 };
2023 }
2024 pub fn parsedec_int(p: *Parser) bool {
2025 return blk_0: {
2026 const pos_0 = p.i;
2027 if (p.parsedec() and blk_1: {
2028 while (p.parsedec_()) {}
2029 break :blk_1 true;
2030 }) break :blk_0 true;
2031 p.i = pos_0;
2032 break :blk_0 false;
2033 };
2034 }
2035 pub fn parsehex_int(p: *Parser) bool {
2036 return blk_0: {
2037 const pos_0 = p.i;
2038 if (p.parsehex() and blk_1: {
2039 while (p.parsehex_()) {}
2040 break :blk_1 true;
2041 }) break :blk_0 true;
2042 p.i = pos_0;
2043 break :blk_0 false;
2044 };
2045 }
20461881 pub fn parseox80_oxBF(p: *Parser) bool {
20471882 return blk_0: {
20481883 const pos_0 = p.i;
......@@ -2286,6 +2121,37 @@ const Parser = struct {
22862121 break :blk_0 false;
22872122 };
22882123 }
2124 pub fn parsehex(p: *Parser) bool {
2125 return blk_0: {
2126 const pos_0 = p.i;
2127 if ((p.i < p.source.len and switch (p.source[p.i]) {
2128 '0'...'9',
2129 'a'...'f',
2130 'A'...'F',
2131 => blk_1: {
2132 p.i += 1;
2133 break :blk_1 true;
2134 },
2135 else => false,
2136 })) break :blk_0 true;
2137 p.i = pos_0;
2138 break :blk_0 false;
2139 };
2140 }
2141 pub fn parsehex_(p: *Parser) bool {
2142 return blk_0: {
2143 const pos_0 = p.i;
2144 if ((blk_2: {
2145 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
2146 p.i += 1;
2147 break :blk_2 true;
2148 }
2149 break :blk_2 false;
2150 } or true) and p.parsehex()) break :blk_0 true;
2151 p.i = pos_0;
2152 break :blk_0 false;
2153 };
2154 }
22892155 pub fn parsechar_escape(p: *Parser) bool {
22902156 return blk_0: {
22912157 const pos_0 = p.i;
......@@ -2587,80 +2453,36 @@ const Parser = struct {
25872453 break :blk_0 false;
25882454 };
25892455 }
2590 pub fn parseFLOAT(p: *Parser) bool {
2456 pub fn parsedigit(p: *Parser) bool {
25912457 return blk_0: {
25922458 const pos_0 = p.i;
2593 if (blk_1: {
2594 if (std.mem.startsWith(u8, p.source[p.i..], "0x")) {
2595 p.i += 2;
2596 break :blk_1 true;
2597 }
2598 break :blk_1 false;
2599 } and p.parsehex_int() and blk_1: {
2600 if (std.mem.startsWith(u8, p.source[p.i..], ".")) {
2459 if ((p.i < p.source.len and switch (p.source[p.i]) {
2460 '_'...'_',
2461 '0'...'9',
2462 'A'...'D',
2463 'F'...'O',
2464 'Q'...'Z',
2465 'a'...'d',
2466 'f'...'o',
2467 'q'...'z',
2468 => blk_1: {
26012469 p.i += 1;
26022470 break :blk_1 true;
2603 }
2604 break :blk_1 false;
2605 } and p.parsehex_int() and (blk_3: {
2606 const pos_3 = p.i;
2607 if ((p.i < p.source.len and switch (p.source[p.i]) {
2608 'p'...'p',
2609 'P'...'P',
2610 => blk_4: {
2611 p.i += 1;
2612 break :blk_4 true;
2613 },
2614 else => false,
2615 }) and ((p.i < p.source.len and switch (p.source[p.i]) {
2616 '-'...'-',
2617 '+'...'+',
2618 => blk_5: {
2619 p.i += 1;
2620 break :blk_5 true;
2621 },
2622 else => false,
2623 }) or true) and p.parsedec_int()) break :blk_3 true;
2624 p.i = pos_3;
2625 break :blk_3 false;
2626 } or true) and p.parseskip()) break :blk_0 true;
2471 },
2472 else => false,
2473 })) break :blk_0 true;
26272474 p.i = pos_0;
2628 if (p.parsedec_int() and blk_1: {
2629 if (std.mem.startsWith(u8, p.source[p.i..], ".")) {
2630 p.i += 1;
2631 break :blk_1 true;
2632 }
2633 break :blk_1 false;
2634 } and p.parsedec_int() and (blk_3: {
2635 const pos_3 = p.i;
2636 if ((p.i < p.source.len and switch (p.source[p.i]) {
2637 'e'...'e',
2638 'E'...'E',
2639 => blk_4: {
2640 p.i += 1;
2641 break :blk_4 true;
2642 },
2643 else => false,
2644 }) and ((p.i < p.source.len and switch (p.source[p.i]) {
2645 '-'...'-',
2646 '+'...'+',
2647 => blk_5: {
2648 p.i += 1;
2649 break :blk_5 true;
2650 },
2651 else => false,
2652 }) or true) and p.parsedec_int()) break :blk_3 true;
2653 p.i = pos_3;
2654 break :blk_3 false;
2655 } or true) and p.parseskip()) break :blk_0 true;
2475 break :blk_0 false;
2476 };
2477 }
2478 pub fn parsedigit_int(p: *Parser) bool {
2479 return blk_0: {
2480 const pos_0 = p.i;
2481 if (p.parsedigit()) break :blk_0 true;
26562482 p.i = pos_0;
2657 if (blk_1: {
2658 if (std.mem.startsWith(u8, p.source[p.i..], "0x")) {
2659 p.i += 2;
2660 break :blk_1 true;
2661 }
2662 break :blk_1 false;
2663 } and p.parsehex_int() and (p.i < p.source.len and switch (p.source[p.i]) {
2483 if ((p.i < p.source.len and switch (p.source[p.i]) {
2484 'e'...'e',
2485 'E'...'E',
26642486 'p'...'p',
26652487 'P'...'P',
26662488 => blk_1: {
......@@ -2668,19 +2490,21 @@ const Parser = struct {
26682490 break :blk_1 true;
26692491 },
26702492 else => false,
2671 }) and ((p.i < p.source.len and switch (p.source[p.i]) {
2672 '-'...'-',
2673 '+'...'+',
2674 => blk_2: {
2675 p.i += 1;
2676 break :blk_2 true;
2677 },
2678 else => false,
2679 }) or true) and p.parsedec_int() and p.parseskip()) break :blk_0 true;
2493 })) break :blk_0 true;
2494 p.i = pos_0;
2495 break :blk_0 false;
2496 };
2497 }
2498 pub fn parsedigit_float(p: *Parser) bool {
2499 return blk_0: {
2500 const pos_0 = p.i;
2501 if (p.parsedigit()) break :blk_0 true;
26802502 p.i = pos_0;
2681 if (p.parsedec_int() and (p.i < p.source.len and switch (p.source[p.i]) {
2503 if ((p.i < p.source.len and switch (p.source[p.i]) {
26822504 'e'...'e',
26832505 'E'...'E',
2506 'p'...'p',
2507 'P'...'P',
26842508 => blk_1: {
26852509 p.i += 1;
26862510 break :blk_1 true;
......@@ -2694,39 +2518,49 @@ const Parser = struct {
26942518 break :blk_2 true;
26952519 },
26962520 else => false,
2697 }) or true) and p.parsedec_int() and p.parseskip()) break :blk_0 true;
2521 }) or true)) break :blk_0 true;
26982522 p.i = pos_0;
26992523 break :blk_0 false;
27002524 };
27012525 }
2702 pub fn parseINTEGER(p: *Parser) bool {
2526 pub fn parseNUMBERLITERAL(p: *Parser) bool {
27032527 return blk_0: {
27042528 const pos_0 = p.i;
2705 if (blk_1: {
2706 if (std.mem.startsWith(u8, p.source[p.i..], "0b")) {
2707 p.i += 2;
2529 if ((p.i < p.source.len and switch (p.source[p.i]) {
2530 '0'...'9',
2531 => blk_1: {
2532 p.i += 1;
27082533 break :blk_1 true;
2709 }
2710 break :blk_1 false;
2711 } and p.parsebin_int() and p.parseskip()) break :blk_0 true;
2712 p.i = pos_0;
2713 if (blk_1: {
2714 if (std.mem.startsWith(u8, p.source[p.i..], "0o")) {
2715 p.i += 2;
2534 },
2535 else => false,
2536 }) and blk_1: {
2537 while (p.parsedigit_int()) {}
2538 break :blk_1 true;
2539 } and blk_1: {
2540 if (std.mem.startsWith(u8, p.source[p.i..], ".")) {
2541 p.i += 1;
27162542 break :blk_1 true;
27172543 }
27182544 break :blk_1 false;
2719 } and p.parseoct_int() and p.parseskip()) break :blk_0 true;
2720 p.i = pos_0;
2721 if (blk_1: {
2722 if (std.mem.startsWith(u8, p.source[p.i..], "0x")) {
2723 p.i += 2;
2724 break :blk_1 true;
2545 } and blk_1: {
2546 var match_1 = false;
2547 while (p.parsedigit_float()) {
2548 match_1 = true;
27252549 }
2726 break :blk_1 false;
2727 } and p.parsehex_int() and p.parseskip()) break :blk_0 true;
2550 break :blk_1 match_1;
2551 } and p.parseskip()) break :blk_0 true;
27282552 p.i = pos_0;
2729 if (p.parsedec_int() and p.parseskip()) break :blk_0 true;
2553 if ((p.i < p.source.len and switch (p.source[p.i]) {
2554 '0'...'9',
2555 => blk_1: {
2556 p.i += 1;
2557 break :blk_1 true;
2558 },
2559 else => false,
2560 }) and blk_1: {
2561 while (p.parsedigit_float()) {}
2562 break :blk_1 true;
2563 } and p.parseskip()) break :blk_0 true;
27302564 p.i = pos_0;
27312565 break :blk_0 false;
27322566 };