authorgravatar for 57862114+momumi@users.noreply.github.commomumi <57862114+momumi@users.noreply.github.com> 2020-03-22 10:35:19+10:00
committergravatar for 57862114+momumi@users.noreply.github.commomumi <57862114+momumi@users.noreply.github.com> 2020-03-22 10:35:19+10:00
log138dab45248c718b8d38fa0a8eefe56f40fe617c
tree6e5a2bf5cf1e6c94fed77bb6425366ec7ef049e0
parent47f7e6658077fb4f55c36e9c62ed5012ef8aace2

add number `_` separators for stage 2 tokenizer

1 files changed, 368 insertions(+), 47 deletions(-)

lib/std/zig/tokenizer.zig+368-47
......@@ -387,17 +387,23 @@ pub const Tokenizer = struct {
387387 DocComment,
388388 ContainerDocComment,
389389 Zero,
390 IntegerLiteral,
391 IntegerLiteralWithRadix,
392 IntegerLiteralWithRadixHex,
393 NumberDot,
390 IntegerLiteralDec,
391 IntegerLiteralDecNoUnderscore,
392 IntegerLiteralBin,
393 IntegerLiteralBinNoUnderscore,
394 IntegerLiteralOct,
395 IntegerLiteralOctNoUnderscore,
396 IntegerLiteralHex,
397 IntegerLiteralHexNoUnderscore,
398 NumberDotDec,
394399 NumberDotHex,
395 FloatFraction,
400 FloatFractionDec,
401 FloatFractionDecNoUnderscore,
396402 FloatFractionHex,
403 FloatFractionHexNoUnderscore,
397404 FloatExponentUnsigned,
398 FloatExponentUnsignedHex,
399405 FloatExponentNumber,
400 FloatExponentNumberHex,
406 FloatExponentNumberNoUnderscore,
401407 Ampersand,
402408 Caret,
403409 Percent,
......@@ -550,7 +556,7 @@ pub const Tokenizer = struct {
550556 result.id = Token.Id.IntegerLiteral;
551557 },
552558 '1'...'9' => {
553 state = State.IntegerLiteral;
559 state = State.IntegerLiteralDec;
554560 result.id = Token.Id.IntegerLiteral;
555561 },
556562 else => {
......@@ -1048,55 +1054,122 @@ pub const Tokenizer = struct {
10481054 else => self.checkLiteralCharacter(),
10491055 },
10501056 State.Zero => switch (c) {
1051 'b', 'o' => {
1052 state = State.IntegerLiteralWithRadix;
1057 'b' => {
1058 state = State.IntegerLiteralBinNoUnderscore;
1059 },
1060 'o' => {
1061 state = State.IntegerLiteralOctNoUnderscore;
10531062 },
10541063 'x' => {
1055 state = State.IntegerLiteralWithRadixHex;
1064 state = State.IntegerLiteralHexNoUnderscore;
10561065 },
10571066 else => {
10581067 // reinterpret as a normal number
10591068 self.index -= 1;
1060 state = State.IntegerLiteral;
1069 state = State.IntegerLiteralDec;
10611070 },
10621071 },
1063 State.IntegerLiteral => switch (c) {
1064 '.' => {
1065 state = State.NumberDot;
1072 State.IntegerLiteralBinNoUnderscore => switch (c) {
1073 '0'...'1' => {
1074 state = State.IntegerLiteralBin;
10661075 },
1067 'p', 'P', 'e', 'E' => {
1068 state = State.FloatExponentUnsigned;
1076 else => {
1077 result.id = Token.Id.Invalid;
1078 },
1079 },
1080 State.IntegerLiteralBin => switch (c) {
1081 '_' => {
1082 state = State.IntegerLiteralBinNoUnderscore;
1083 },
1084 '0'...'1' => {},
1085 '2'...'9', 'a'...'z', 'A'...'Z' => {
1086 result.id = Token.Id.Invalid;
10691087 },
1070 '0'...'9' => {},
10711088 else => break,
10721089 },
1073 State.IntegerLiteralWithRadix => switch (c) {
1090 State.IntegerLiteralOctNoUnderscore => switch (c) {
1091 '0'...'7' => {
1092 state = State.IntegerLiteralOct;
1093 },
1094 else => {
1095 result.id = Token.Id.Invalid;
1096 },
1097 },
1098 State.IntegerLiteralOct => switch (c) {
1099 '_' => {
1100 state = State.IntegerLiteralOctNoUnderscore;
1101 },
1102 '0'...'7' => {},
1103 '8'...'9', 'a'...'z', 'A'...'Z' => {
1104 result.id = Token.Id.Invalid;
1105 },
1106 else => break,
1107 },
1108 State.IntegerLiteralDecNoUnderscore => switch (c) {
1109 '0'...'9' => {
1110 state = State.IntegerLiteralDec;
1111 },
1112 else => {
1113 result.id = Token.Id.Invalid;
1114 },
1115 },
1116 State.IntegerLiteralDec => switch (c) {
1117 '_' => {
1118 state = State.IntegerLiteralDecNoUnderscore;
1119 },
10741120 '.' => {
1075 state = State.NumberDot;
1121 state = State.NumberDotDec;
1122 result.id = Token.Id.FloatLiteral;
1123 },
1124 'e', 'E' => {
1125 state = State.FloatExponentUnsigned;
1126 result.id = Token.Id.FloatLiteral;
10761127 },
10771128 '0'...'9' => {},
1129 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
1130 result.id = Token.Id.Invalid;
1131 },
10781132 else => break,
10791133 },
1080 State.IntegerLiteralWithRadixHex => switch (c) {
1134 State.IntegerLiteralHexNoUnderscore => switch (c) {
1135 '0'...'9', 'a'...'f', 'A'...'F' => {
1136 state = State.IntegerLiteralHex;
1137 },
1138 else => {
1139 result.id = Token.Id.Invalid;
1140 },
1141 },
1142 State.IntegerLiteralHex => switch (c) {
1143 '_' => {
1144 state = State.IntegerLiteralHexNoUnderscore;
1145 },
10811146 '.' => {
10821147 state = State.NumberDotHex;
1148 result.id = Token.Id.FloatLiteral;
10831149 },
10841150 'p', 'P' => {
1085 state = State.FloatExponentUnsignedHex;
1151 state = State.FloatExponentUnsigned;
1152 result.id = Token.Id.FloatLiteral;
10861153 },
10871154 '0'...'9', 'a'...'f', 'A'...'F' => {},
1155 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
1156 result.id = Token.Id.Invalid;
1157 },
10881158 else => break,
10891159 },
1090 State.NumberDot => switch (c) {
1160 State.NumberDotDec => switch (c) {
10911161 '.' => {
10921162 self.index -= 1;
10931163 state = State.Start;
10941164 break;
10951165 },
1166 'e', 'E' => {
1167 state = State.FloatExponentUnsigned;
1168 },
10961169 else => {
10971170 self.index -= 1;
10981171 result.id = Token.Id.FloatLiteral;
1099 state = State.FloatFraction;
1172 state = State.FloatFractionDecNoUnderscore;
11001173 },
11011174 },
11021175 State.NumberDotHex => switch (c) {
......@@ -1105,65 +1178,98 @@ pub const Tokenizer = struct {
11051178 state = State.Start;
11061179 break;
11071180 },
1181 'p', 'P' => {
1182 state = State.FloatExponentUnsigned;
1183 },
11081184 else => {
11091185 self.index -= 1;
11101186 result.id = Token.Id.FloatLiteral;
1111 state = State.FloatFractionHex;
1187 state = State.FloatFractionHexNoUnderscore;
1188 },
1189 },
1190 State.FloatFractionDecNoUnderscore => switch (c) {
1191 '0'...'9' => {
1192 state = State.FloatFractionDec;
1193 },
1194 else => {
1195 result.id = Token.Id.Invalid;
11121196 },
11131197 },
1114 State.FloatFraction => switch (c) {
1198 State.FloatFractionDec => switch (c) {
1199 '_' => {
1200 state = State.FloatFractionDecNoUnderscore;
1201 },
11151202 'e', 'E' => {
11161203 state = State.FloatExponentUnsigned;
11171204 },
11181205 '0'...'9' => {},
1206 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
1207 result.id = Token.Id.Invalid;
1208 },
11191209 else => break,
11201210 },
1211 State.FloatFractionHexNoUnderscore => switch (c) {
1212 '0'...'9', 'a'...'f', 'A'...'F' => {
1213 state = State.FloatFractionHex;
1214 },
1215 else => {
1216 result.id = Token.Id.Invalid;
1217 },
1218 },
11211219 State.FloatFractionHex => switch (c) {
1220 '_' => {
1221 state = State.FloatFractionHexNoUnderscore;
1222 },
11221223 'p', 'P' => {
1123 state = State.FloatExponentUnsignedHex;
1224 state = State.FloatExponentUnsigned;
11241225 },
11251226 '0'...'9', 'a'...'f', 'A'...'F' => {},
1227 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
1228 result.id = Token.Id.Invalid;
1229 },
11261230 else => break,
11271231 },
11281232 State.FloatExponentUnsigned => switch (c) {
11291233 '+', '-' => {
1130 state = State.FloatExponentNumber;
1234 state = State.FloatExponentNumberNoUnderscore;
11311235 },
11321236 else => {
11331237 // reinterpret as a normal exponent number
11341238 self.index -= 1;
1135 state = State.FloatExponentNumber;
1239 state = State.FloatExponentNumberNoUnderscore;
11361240 },
11371241 },
1138 State.FloatExponentUnsignedHex => switch (c) {
1139 '+', '-' => {
1140 state = State.FloatExponentNumberHex;
1242 State.FloatExponentNumberNoUnderscore => switch (c) {
1243 '0'...'9' => {
1244 state = State.FloatExponentNumber;
11411245 },
11421246 else => {
1143 // reinterpret as a normal exponent number
1144 self.index -= 1;
1145 state = State.FloatExponentNumberHex;
1247 result.id = Token.Id.Invalid;
11461248 },
11471249 },
11481250 State.FloatExponentNumber => switch (c) {
1251 '_' => {
1252 state = State.FloatExponentNumberNoUnderscore;
1253 },
11491254 '0'...'9' => {},
1150 else => break,
1151 },
1152 State.FloatExponentNumberHex => switch (c) {
1153 '0'...'9', 'a'...'f', 'A'...'F' => {},
1255 'a'...'z', 'A'...'Z' => {
1256 result.id = Token.Id.Invalid;
1257 },
11541258 else => break,
11551259 },
11561260 }
11571261 } else if (self.index == self.buffer.len) {
11581262 switch (state) {
11591263 State.Start,
1160 State.IntegerLiteral,
1161 State.IntegerLiteralWithRadix,
1162 State.IntegerLiteralWithRadixHex,
1163 State.FloatFraction,
1264 State.IntegerLiteralDec,
1265 State.IntegerLiteralBin,
1266 State.IntegerLiteralOct,
1267 State.IntegerLiteralHex,
1268 State.NumberDotDec,
1269 State.NumberDotHex,
1270 State.FloatFractionDec,
11641271 State.FloatFractionHex,
11651272 State.FloatExponentNumber,
1166 State.FloatExponentNumberHex,
11671273 State.StringLiteral, // find this error later
11681274 State.MultilineStringLiteralLine,
11691275 State.Builtin,
......@@ -1184,10 +1290,14 @@ pub const Tokenizer = struct {
11841290 result.id = Token.Id.ContainerDocComment;
11851291 },
11861292
1187 State.NumberDot,
1188 State.NumberDotHex,
1293 State.IntegerLiteralDecNoUnderscore,
1294 State.IntegerLiteralBinNoUnderscore,
1295 State.IntegerLiteralOctNoUnderscore,
1296 State.IntegerLiteralHexNoUnderscore,
1297 State.FloatFractionDecNoUnderscore,
1298 State.FloatFractionHexNoUnderscore,
1299 State.FloatExponentNumberNoUnderscore,
11891300 State.FloatExponentUnsigned,
1190 State.FloatExponentUnsignedHex,
11911301 State.SawAtSign,
11921302 State.Backslash,
11931303 State.CharLiteral,
......@@ -1585,6 +1695,217 @@ test "correctly parse pointer assignment" {
15851695 });
15861696}
15871697
1698test "tokenizer - number literals decimal" {
1699 testTokenize("1", &[_]Token.Id{ .IntegerLiteral });
1700 testTokenize("2", &[_]Token.Id{ .IntegerLiteral });
1701 testTokenize("3", &[_]Token.Id{ .IntegerLiteral });
1702 testTokenize("4", &[_]Token.Id{ .IntegerLiteral });
1703 testTokenize("5", &[_]Token.Id{ .IntegerLiteral });
1704 testTokenize("6", &[_]Token.Id{ .IntegerLiteral });
1705 testTokenize("7", &[_]Token.Id{ .IntegerLiteral });
1706 testTokenize("8", &[_]Token.Id{ .IntegerLiteral });
1707 testTokenize("9", &[_]Token.Id{ .IntegerLiteral });
1708 testTokenize("0a", &[_]Token.Id{ .Invalid });
1709 testTokenize("9b", &[_]Token.Id{ .Invalid });
1710 testTokenize("1z", &[_]Token.Id{ .Invalid });
1711 testTokenize("1z_1", &[_]Token.Id{ .Invalid });
1712 testTokenize("9z3", &[_]Token.Id{ .Invalid });
1713
1714 testTokenize("0_0", &[_]Token.Id{ .IntegerLiteral });
1715 testTokenize("0001", &[_]Token.Id{ .IntegerLiteral });
1716 testTokenize("01234567890", &[_]Token.Id{ .IntegerLiteral });
1717 testTokenize("012_345_6789_0", &[_]Token.Id{ .IntegerLiteral });
1718 testTokenize("0_1_2_3_4_5_6_7_8_9_0", &[_]Token.Id{ .IntegerLiteral });
1719
1720 testTokenize("00_", &[_]Token.Id{ .Invalid });
1721 testTokenize("0_0_", &[_]Token.Id{ .Invalid });
1722 testTokenize("0__0", &[_]Token.Id{ .Invalid });
1723 testTokenize("0_0f", &[_]Token.Id{ .Invalid });
1724 testTokenize("0_0_f", &[_]Token.Id{ .Invalid });
1725 testTokenize("1_,", &[_]Token.Id{ .Invalid });
1726
1727 testTokenize("1.", &[_]Token.Id{ .FloatLiteral });
1728 testTokenize("0.0", &[_]Token.Id{ .FloatLiteral });
1729 testTokenize("1.0", &[_]Token.Id{ .FloatLiteral });
1730 testTokenize("10.0", &[_]Token.Id{ .FloatLiteral });
1731 testTokenize("1e0", &[_]Token.Id{ .FloatLiteral });
1732 testTokenize("1e100", &[_]Token.Id{ .FloatLiteral });
1733 testTokenize("1.e100", &[_]Token.Id{ .FloatLiteral });
1734 testTokenize("1.0e100", &[_]Token.Id{ .FloatLiteral });
1735 testTokenize("1.0e+100", &[_]Token.Id{ .FloatLiteral });
1736 testTokenize("1.0e-100", &[_]Token.Id{ .FloatLiteral });
1737 testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{ .FloatLiteral });
1738
1739 testTokenize("1e", &[_]Token.Id{ .Invalid });
1740 testTokenize("1.0e1f0", &[_]Token.Id{ .Invalid });
1741 testTokenize("1.0p100", &[_]Token.Id{ .Invalid });
1742 testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Minus, .IntegerLiteral });
1743 testTokenize("1.0p1f0", &[_]Token.Id{ .Invalid });
1744 testTokenize("1.0_,", &[_]Token.Id{ .Invalid });
1745 testTokenize("1.0e,", &[_]Token.Id{ .Invalid });
1746}
1747
1748
1749test "tokenizer - number literals binary" {
1750 testTokenize("0b0", &[_]Token.Id{ .IntegerLiteral });
1751 testTokenize("0b1", &[_]Token.Id{ .IntegerLiteral });
1752 testTokenize("0b2", &[_]Token.Id{ .Invalid });
1753 testTokenize("0b3", &[_]Token.Id{ .Invalid });
1754 testTokenize("0b4", &[_]Token.Id{ .Invalid });
1755 testTokenize("0b5", &[_]Token.Id{ .Invalid });
1756 testTokenize("0b6", &[_]Token.Id{ .Invalid });
1757 testTokenize("0b7", &[_]Token.Id{ .Invalid });
1758 testTokenize("0b8", &[_]Token.Id{ .Invalid });
1759 testTokenize("0b9", &[_]Token.Id{ .Invalid });
1760 testTokenize("0ba", &[_]Token.Id{ .Invalid });
1761 testTokenize("0bb", &[_]Token.Id{ .Invalid });
1762 testTokenize("0bc", &[_]Token.Id{ .Invalid });
1763 testTokenize("0bd", &[_]Token.Id{ .Invalid });
1764 testTokenize("0be", &[_]Token.Id{ .Invalid });
1765 testTokenize("0bf", &[_]Token.Id{ .Invalid });
1766 testTokenize("0bz", &[_]Token.Id{ .Invalid });
1767
1768 testTokenize("0b0000_0000", &[_]Token.Id{ .IntegerLiteral });
1769 testTokenize("0b1111_1111", &[_]Token.Id{ .IntegerLiteral });
1770 testTokenize("0b10_10_10_10", &[_]Token.Id{ .IntegerLiteral });
1771 testTokenize("0b0_1_0_1_0_1_0_1", &[_]Token.Id{ .IntegerLiteral });
1772 testTokenize("0b1.", &[_]Token.Id{ .IntegerLiteral, .Period });
1773 testTokenize("0b1.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
1774
1775 testTokenize("0B0", &[_]Token.Id{ .Invalid });
1776 testTokenize("0b_", &[_]Token.Id{ .Invalid });
1777 testTokenize("0b_0", &[_]Token.Id{ .Invalid });
1778 testTokenize("0b1_", &[_]Token.Id{ .Invalid });
1779 testTokenize("0b0__1", &[_]Token.Id{ .Invalid });
1780 testTokenize("0b0_1_", &[_]Token.Id{ .Invalid });
1781 testTokenize("0b1e", &[_]Token.Id{ .Invalid });
1782 testTokenize("0b1p", &[_]Token.Id{ .Invalid });
1783 testTokenize("0b1e0", &[_]Token.Id{ .Invalid });
1784 testTokenize("0b1p0", &[_]Token.Id{ .Invalid });
1785 testTokenize("0b1_,", &[_]Token.Id{ .Invalid });
1786}
1787
1788test "tokenizer - number literals octal" {
1789 testTokenize("0o0", &[_]Token.Id{ .IntegerLiteral });
1790 testTokenize("0o1", &[_]Token.Id{ .IntegerLiteral });
1791 testTokenize("0o2", &[_]Token.Id{ .IntegerLiteral });
1792 testTokenize("0o3", &[_]Token.Id{ .IntegerLiteral });
1793 testTokenize("0o4", &[_]Token.Id{ .IntegerLiteral });
1794 testTokenize("0o5", &[_]Token.Id{ .IntegerLiteral });
1795 testTokenize("0o6", &[_]Token.Id{ .IntegerLiteral });
1796 testTokenize("0o7", &[_]Token.Id{ .IntegerLiteral });
1797 testTokenize("0o8", &[_]Token.Id{ .Invalid });
1798 testTokenize("0o9", &[_]Token.Id{ .Invalid });
1799 testTokenize("0oa", &[_]Token.Id{ .Invalid });
1800 testTokenize("0ob", &[_]Token.Id{ .Invalid });
1801 testTokenize("0oc", &[_]Token.Id{ .Invalid });
1802 testTokenize("0od", &[_]Token.Id{ .Invalid });
1803 testTokenize("0oe", &[_]Token.Id{ .Invalid });
1804 testTokenize("0of", &[_]Token.Id{ .Invalid });
1805 testTokenize("0oz", &[_]Token.Id{ .Invalid });
1806
1807 testTokenize("0o01234567", &[_]Token.Id{ .IntegerLiteral });
1808 testTokenize("0o0123_4567", &[_]Token.Id{ .IntegerLiteral });
1809 testTokenize("0o01_23_45_67", &[_]Token.Id{ .IntegerLiteral });
1810 testTokenize("0o0_1_2_3_4_5_6_7", &[_]Token.Id{ .IntegerLiteral });
1811 testTokenize("0o7.", &[_]Token.Id{ .IntegerLiteral, .Period });
1812 testTokenize("0o7.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
1813
1814 testTokenize("0O0", &[_]Token.Id{ .Invalid });
1815 testTokenize("0o_", &[_]Token.Id{ .Invalid });
1816 testTokenize("0o_0", &[_]Token.Id{ .Invalid });
1817 testTokenize("0o1_", &[_]Token.Id{ .Invalid });
1818 testTokenize("0o0__1", &[_]Token.Id{ .Invalid });
1819 testTokenize("0o0_1_", &[_]Token.Id{ .Invalid });
1820 testTokenize("0o1e", &[_]Token.Id{ .Invalid });
1821 testTokenize("0o1p", &[_]Token.Id{ .Invalid });
1822 testTokenize("0o1e0", &[_]Token.Id{ .Invalid });
1823 testTokenize("0o1p0", &[_]Token.Id{ .Invalid });
1824 testTokenize("0o_,", &[_]Token.Id{ .Invalid });
1825}
1826
1827test "tokenizer - number literals hexadeciaml" {
1828 testTokenize("0x0", &[_]Token.Id{ .IntegerLiteral });
1829 testTokenize("0x1", &[_]Token.Id{ .IntegerLiteral });
1830 testTokenize("0x2", &[_]Token.Id{ .IntegerLiteral });
1831 testTokenize("0x3", &[_]Token.Id{ .IntegerLiteral });
1832 testTokenize("0x4", &[_]Token.Id{ .IntegerLiteral });
1833 testTokenize("0x5", &[_]Token.Id{ .IntegerLiteral });
1834 testTokenize("0x6", &[_]Token.Id{ .IntegerLiteral });
1835 testTokenize("0x7", &[_]Token.Id{ .IntegerLiteral });
1836 testTokenize("0x8", &[_]Token.Id{ .IntegerLiteral });
1837 testTokenize("0x9", &[_]Token.Id{ .IntegerLiteral });
1838 testTokenize("0xa", &[_]Token.Id{ .IntegerLiteral });
1839 testTokenize("0xb", &[_]Token.Id{ .IntegerLiteral });
1840 testTokenize("0xc", &[_]Token.Id{ .IntegerLiteral });
1841 testTokenize("0xd", &[_]Token.Id{ .IntegerLiteral });
1842 testTokenize("0xe", &[_]Token.Id{ .IntegerLiteral });
1843 testTokenize("0xf", &[_]Token.Id{ .IntegerLiteral });
1844 testTokenize("0xA", &[_]Token.Id{ .IntegerLiteral });
1845 testTokenize("0xB", &[_]Token.Id{ .IntegerLiteral });
1846 testTokenize("0xC", &[_]Token.Id{ .IntegerLiteral });
1847 testTokenize("0xD", &[_]Token.Id{ .IntegerLiteral });
1848 testTokenize("0xE", &[_]Token.Id{ .IntegerLiteral });
1849 testTokenize("0xF", &[_]Token.Id{ .IntegerLiteral });
1850 testTokenize("0x0z", &[_]Token.Id{ .Invalid });
1851 testTokenize("0xz", &[_]Token.Id{ .Invalid });
1852
1853 testTokenize("0x0123456789ABCDEF", &[_]Token.Id{ .IntegerLiteral });
1854 testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{ .IntegerLiteral });
1855 testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{ .IntegerLiteral });
1856 testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{ .IntegerLiteral });
1857
1858 testTokenize("0X0", &[_]Token.Id{ .Invalid });
1859 testTokenize("0x_", &[_]Token.Id{ .Invalid });
1860 testTokenize("0x_1", &[_]Token.Id{ .Invalid });
1861 testTokenize("0x1_", &[_]Token.Id{ .Invalid });
1862 testTokenize("0x0__1", &[_]Token.Id{ .Invalid });
1863 testTokenize("0x0_1_", &[_]Token.Id{ .Invalid });
1864 testTokenize("0x_,", &[_]Token.Id{ .Invalid });
1865
1866 testTokenize("0x1.", &[_]Token.Id{ .FloatLiteral });
1867 testTokenize("0x1.0", &[_]Token.Id{ .FloatLiteral });
1868 testTokenize("0xF.", &[_]Token.Id{ .FloatLiteral });
1869 testTokenize("0xF.0", &[_]Token.Id{ .FloatLiteral });
1870 testTokenize("0xF.F", &[_]Token.Id{ .FloatLiteral });
1871 testTokenize("0xF.Fp0", &[_]Token.Id{ .FloatLiteral });
1872 testTokenize("0xF.FP0", &[_]Token.Id{ .FloatLiteral });
1873 testTokenize("0x1p0", &[_]Token.Id{ .FloatLiteral });
1874 testTokenize("0xfp0", &[_]Token.Id{ .FloatLiteral });
1875
1876 testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{ .FloatLiteral });
1877 testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{ .FloatLiteral });
1878 testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{ .FloatLiteral });
1879 testTokenize("0x0.0p0", &[_]Token.Id{ .FloatLiteral });
1880 testTokenize("0xff.ffp10", &[_]Token.Id{ .FloatLiteral });
1881 testTokenize("0xff.ffP10", &[_]Token.Id{ .FloatLiteral });
1882 testTokenize("0xff.p10", &[_]Token.Id{ .FloatLiteral });
1883 testTokenize("0xffp10", &[_]Token.Id{ .FloatLiteral });
1884 testTokenize("0xff_ff.ff_ffp1_0_0_0", &[_]Token.Id{ .FloatLiteral });
1885 testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &[_]Token.Id{ .FloatLiteral });
1886 testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &[_]Token.Id{ .FloatLiteral });
1887
1888 testTokenize("0x1e", &[_]Token.Id{ .IntegerLiteral });
1889 testTokenize("0x1e0", &[_]Token.Id{ .IntegerLiteral });
1890 testTokenize("0x1p", &[_]Token.Id{ .Invalid });
1891 testTokenize("0xfp0z1", &[_]Token.Id{ .Invalid });
1892 testTokenize("0xff.ffpff", &[_]Token.Id{ .Invalid });
1893 testTokenize("0x0_.0", &[_]Token.Id{ .Invalid });
1894 testTokenize("0x0._0", &[_]Token.Id{ .Invalid });
1895 testTokenize("0x0.0_", &[_]Token.Id{ .Invalid });
1896 testTokenize("0x0_p0", &[_]Token.Id{ .Invalid });
1897 testTokenize("0x0_.p0", &[_]Token.Id{ .Invalid });
1898 testTokenize("0x0._p0", &[_]Token.Id{ .Invalid });
1899 testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid });
1900 testTokenize("0x0._0p0", &[_]Token.Id{ .Invalid });
1901 testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid });
1902 testTokenize("0x0.0p_0", &[_]Token.Id{ .Invalid });
1903 testTokenize("0x0.0p+_0", &[_]Token.Id{ .Invalid });
1904 testTokenize("0x0.0p-_0", &[_]Token.Id{ .Invalid });
1905 testTokenize("0x0.0p0_", &[_]Token.Id{ .Invalid });
1906}
1907
1908
15881909fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
15891910 var tokenizer = Tokenizer.init(source);
15901911 for (expected_tokens) |expected_token_id| {