authorgravatar for 57862114+momumi@users.noreply.github.commomumi <57862114+momumi@users.noreply.github.com> 2020-03-22 10:35:19+10:00
committergravatar for 57862114+momumi@users.noreply.github.commomumi <57862114+momumi@users.noreply.github.com> 2020-03-22 10:35:19+10:00
log138dab45248c718b8d38fa0a8eefe56f40fe617c
tree6e5a2bf5cf1e6c94fed77bb6425366ec7ef049e0
parent47f7e6658077fb4f55c36e9c62ed5012ef8aace2

add number `_` separators for stage 2 tokenizer

1 files changed, 368 insertions(+), 47 deletions(-)

lib/std/zig/tokenizer.zig+368-47
...@@ -387,17 +387,23 @@ pub const Tokenizer = struct {...@@ -387,17 +387,23 @@ pub const Tokenizer = struct {
387 DocComment,387 DocComment,
388 ContainerDocComment,388 ContainerDocComment,
389 Zero,389 Zero,
390 IntegerLiteral,390 IntegerLiteralDec,
391 IntegerLiteralWithRadix,391 IntegerLiteralDecNoUnderscore,
392 IntegerLiteralWithRadixHex,392 IntegerLiteralBin,
393 NumberDot,393 IntegerLiteralBinNoUnderscore,
394 IntegerLiteralOct,
395 IntegerLiteralOctNoUnderscore,
396 IntegerLiteralHex,
397 IntegerLiteralHexNoUnderscore,
398 NumberDotDec,
394 NumberDotHex,399 NumberDotHex,
395 FloatFraction,400 FloatFractionDec,
401 FloatFractionDecNoUnderscore,
396 FloatFractionHex,402 FloatFractionHex,
403 FloatFractionHexNoUnderscore,
397 FloatExponentUnsigned,404 FloatExponentUnsigned,
398 FloatExponentUnsignedHex,
399 FloatExponentNumber,405 FloatExponentNumber,
400 FloatExponentNumberHex,406 FloatExponentNumberNoUnderscore,
401 Ampersand,407 Ampersand,
402 Caret,408 Caret,
403 Percent,409 Percent,
...@@ -550,7 +556,7 @@ pub const Tokenizer = struct {...@@ -550,7 +556,7 @@ pub const Tokenizer = struct {
550 result.id = Token.Id.IntegerLiteral;556 result.id = Token.Id.IntegerLiteral;
551 },557 },
552 '1'...'9' => {558 '1'...'9' => {
553 state = State.IntegerLiteral;559 state = State.IntegerLiteralDec;
554 result.id = Token.Id.IntegerLiteral;560 result.id = Token.Id.IntegerLiteral;
555 },561 },
556 else => {562 else => {
...@@ -1048,55 +1054,122 @@ pub const Tokenizer = struct {...@@ -1048,55 +1054,122 @@ pub const Tokenizer = struct {
1048 else => self.checkLiteralCharacter(),1054 else => self.checkLiteralCharacter(),
1049 },1055 },
1050 State.Zero => switch (c) {1056 State.Zero => switch (c) {
1051 'b', 'o' => {1057 'b' => {
1052 state = State.IntegerLiteralWithRadix;1058 state = State.IntegerLiteralBinNoUnderscore;
1059 },
1060 'o' => {
1061 state = State.IntegerLiteralOctNoUnderscore;
1053 },1062 },
1054 'x' => {1063 'x' => {
1055 state = State.IntegerLiteralWithRadixHex;1064 state = State.IntegerLiteralHexNoUnderscore;
1056 },1065 },
1057 else => {1066 else => {
1058 // reinterpret as a normal number1067 // reinterpret as a normal number
1059 self.index -= 1;1068 self.index -= 1;
1060 state = State.IntegerLiteral;1069 state = State.IntegerLiteralDec;
1061 },1070 },
1062 },1071 },
1063 State.IntegerLiteral => switch (c) {1072 State.IntegerLiteralBinNoUnderscore => switch (c) {
1064 '.' => {1073 '0'...'1' => {
1065 state = State.NumberDot;1074 state = State.IntegerLiteralBin;
1066 },1075 },
1067 'p', 'P', 'e', 'E' => {1076 else => {
1068 state = State.FloatExponentUnsigned;1077 result.id = Token.Id.Invalid;
1078 },
1079 },
1080 State.IntegerLiteralBin => switch (c) {
1081 '_' => {
1082 state = State.IntegerLiteralBinNoUnderscore;
1083 },
1084 '0'...'1' => {},
1085 '2'...'9', 'a'...'z', 'A'...'Z' => {
1086 result.id = Token.Id.Invalid;
1069 },1087 },
1070 '0'...'9' => {},
1071 else => break,1088 else => break,
1072 },1089 },
1073 State.IntegerLiteralWithRadix => switch (c) {1090 State.IntegerLiteralOctNoUnderscore => switch (c) {
1091 '0'...'7' => {
1092 state = State.IntegerLiteralOct;
1093 },
1094 else => {
1095 result.id = Token.Id.Invalid;
1096 },
1097 },
1098 State.IntegerLiteralOct => switch (c) {
1099 '_' => {
1100 state = State.IntegerLiteralOctNoUnderscore;
1101 },
1102 '0'...'7' => {},
1103 '8'...'9', 'a'...'z', 'A'...'Z' => {
1104 result.id = Token.Id.Invalid;
1105 },
1106 else => break,
1107 },
1108 State.IntegerLiteralDecNoUnderscore => switch (c) {
1109 '0'...'9' => {
1110 state = State.IntegerLiteralDec;
1111 },
1112 else => {
1113 result.id = Token.Id.Invalid;
1114 },
1115 },
1116 State.IntegerLiteralDec => switch (c) {
1117 '_' => {
1118 state = State.IntegerLiteralDecNoUnderscore;
1119 },
1074 '.' => {1120 '.' => {
1075 state = State.NumberDot;1121 state = State.NumberDotDec;
1122 result.id = Token.Id.FloatLiteral;
1123 },
1124 'e', 'E' => {
1125 state = State.FloatExponentUnsigned;
1126 result.id = Token.Id.FloatLiteral;
1076 },1127 },
1077 '0'...'9' => {},1128 '0'...'9' => {},
1129 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
1130 result.id = Token.Id.Invalid;
1131 },
1078 else => break,1132 else => break,
1079 },1133 },
1080 State.IntegerLiteralWithRadixHex => switch (c) {1134 State.IntegerLiteralHexNoUnderscore => switch (c) {
1135 '0'...'9', 'a'...'f', 'A'...'F' => {
1136 state = State.IntegerLiteralHex;
1137 },
1138 else => {
1139 result.id = Token.Id.Invalid;
1140 },
1141 },
1142 State.IntegerLiteralHex => switch (c) {
1143 '_' => {
1144 state = State.IntegerLiteralHexNoUnderscore;
1145 },
1081 '.' => {1146 '.' => {
1082 state = State.NumberDotHex;1147 state = State.NumberDotHex;
1148 result.id = Token.Id.FloatLiteral;
1083 },1149 },
1084 'p', 'P' => {1150 'p', 'P' => {
1085 state = State.FloatExponentUnsignedHex;1151 state = State.FloatExponentUnsigned;
1152 result.id = Token.Id.FloatLiteral;
1086 },1153 },
1087 '0'...'9', 'a'...'f', 'A'...'F' => {},1154 '0'...'9', 'a'...'f', 'A'...'F' => {},
1155 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
1156 result.id = Token.Id.Invalid;
1157 },
1088 else => break,1158 else => break,
1089 },1159 },
1090 State.NumberDot => switch (c) {1160 State.NumberDotDec => switch (c) {
1091 '.' => {1161 '.' => {
1092 self.index -= 1;1162 self.index -= 1;
1093 state = State.Start;1163 state = State.Start;
1094 break;1164 break;
1095 },1165 },
1166 'e', 'E' => {
1167 state = State.FloatExponentUnsigned;
1168 },
1096 else => {1169 else => {
1097 self.index -= 1;1170 self.index -= 1;
1098 result.id = Token.Id.FloatLiteral;1171 result.id = Token.Id.FloatLiteral;
1099 state = State.FloatFraction;1172 state = State.FloatFractionDecNoUnderscore;
1100 },1173 },
1101 },1174 },
1102 State.NumberDotHex => switch (c) {1175 State.NumberDotHex => switch (c) {
...@@ -1105,65 +1178,98 @@ pub const Tokenizer = struct {...@@ -1105,65 +1178,98 @@ pub const Tokenizer = struct {
1105 state = State.Start;1178 state = State.Start;
1106 break;1179 break;
1107 },1180 },
1181 'p', 'P' => {
1182 state = State.FloatExponentUnsigned;
1183 },
1108 else => {1184 else => {
1109 self.index -= 1;1185 self.index -= 1;
1110 result.id = Token.Id.FloatLiteral;1186 result.id = Token.Id.FloatLiteral;
1111 state = State.FloatFractionHex;1187 state = State.FloatFractionHexNoUnderscore;
1188 },
1189 },
1190 State.FloatFractionDecNoUnderscore => switch (c) {
1191 '0'...'9' => {
1192 state = State.FloatFractionDec;
1193 },
1194 else => {
1195 result.id = Token.Id.Invalid;
1112 },1196 },
1113 },1197 },
1114 State.FloatFraction => switch (c) {1198 State.FloatFractionDec => switch (c) {
1199 '_' => {
1200 state = State.FloatFractionDecNoUnderscore;
1201 },
1115 'e', 'E' => {1202 'e', 'E' => {
1116 state = State.FloatExponentUnsigned;1203 state = State.FloatExponentUnsigned;
1117 },1204 },
1118 '0'...'9' => {},1205 '0'...'9' => {},
1206 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
1207 result.id = Token.Id.Invalid;
1208 },
1119 else => break,1209 else => break,
1120 },1210 },
1211 State.FloatFractionHexNoUnderscore => switch (c) {
1212 '0'...'9', 'a'...'f', 'A'...'F' => {
1213 state = State.FloatFractionHex;
1214 },
1215 else => {
1216 result.id = Token.Id.Invalid;
1217 },
1218 },
1121 State.FloatFractionHex => switch (c) {1219 State.FloatFractionHex => switch (c) {
1220 '_' => {
1221 state = State.FloatFractionHexNoUnderscore;
1222 },
1122 'p', 'P' => {1223 'p', 'P' => {
1123 state = State.FloatExponentUnsignedHex;1224 state = State.FloatExponentUnsigned;
1124 },1225 },
1125 '0'...'9', 'a'...'f', 'A'...'F' => {},1226 '0'...'9', 'a'...'f', 'A'...'F' => {},
1227 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
1228 result.id = Token.Id.Invalid;
1229 },
1126 else => break,1230 else => break,
1127 },1231 },
1128 State.FloatExponentUnsigned => switch (c) {1232 State.FloatExponentUnsigned => switch (c) {
1129 '+', '-' => {1233 '+', '-' => {
1130 state = State.FloatExponentNumber;1234 state = State.FloatExponentNumberNoUnderscore;
1131 },1235 },
1132 else => {1236 else => {
1133 // reinterpret as a normal exponent number1237 // reinterpret as a normal exponent number
1134 self.index -= 1;1238 self.index -= 1;
1135 state = State.FloatExponentNumber;1239 state = State.FloatExponentNumberNoUnderscore;
1136 },1240 },
1137 },1241 },
1138 State.FloatExponentUnsignedHex => switch (c) {1242 State.FloatExponentNumberNoUnderscore => switch (c) {
1139 '+', '-' => {1243 '0'...'9' => {
1140 state = State.FloatExponentNumberHex;1244 state = State.FloatExponentNumber;
1141 },1245 },
1142 else => {1246 else => {
1143 // reinterpret as a normal exponent number1247 result.id = Token.Id.Invalid;
1144 self.index -= 1;
1145 state = State.FloatExponentNumberHex;
1146 },1248 },
1147 },1249 },
1148 State.FloatExponentNumber => switch (c) {1250 State.FloatExponentNumber => switch (c) {
1251 '_' => {
1252 state = State.FloatExponentNumberNoUnderscore;
1253 },
1149 '0'...'9' => {},1254 '0'...'9' => {},
1150 else => break,1255 'a'...'z', 'A'...'Z' => {
1151 },1256 result.id = Token.Id.Invalid;
1152 State.FloatExponentNumberHex => switch (c) {1257 },
1153 '0'...'9', 'a'...'f', 'A'...'F' => {},
1154 else => break,1258 else => break,
1155 },1259 },
1156 }1260 }
1157 } else if (self.index == self.buffer.len) {1261 } else if (self.index == self.buffer.len) {
1158 switch (state) {1262 switch (state) {
1159 State.Start,1263 State.Start,
1160 State.IntegerLiteral,1264 State.IntegerLiteralDec,
1161 State.IntegerLiteralWithRadix,1265 State.IntegerLiteralBin,
1162 State.IntegerLiteralWithRadixHex,1266 State.IntegerLiteralOct,
1163 State.FloatFraction,1267 State.IntegerLiteralHex,
1268 State.NumberDotDec,
1269 State.NumberDotHex,
1270 State.FloatFractionDec,
1164 State.FloatFractionHex,1271 State.FloatFractionHex,
1165 State.FloatExponentNumber,1272 State.FloatExponentNumber,
1166 State.FloatExponentNumberHex,
1167 State.StringLiteral, // find this error later1273 State.StringLiteral, // find this error later
1168 State.MultilineStringLiteralLine,1274 State.MultilineStringLiteralLine,
1169 State.Builtin,1275 State.Builtin,
...@@ -1184,10 +1290,14 @@ pub const Tokenizer = struct {...@@ -1184,10 +1290,14 @@ pub const Tokenizer = struct {
1184 result.id = Token.Id.ContainerDocComment;1290 result.id = Token.Id.ContainerDocComment;
1185 },1291 },
11861292
1187 State.NumberDot,1293 State.IntegerLiteralDecNoUnderscore,
1188 State.NumberDotHex,1294 State.IntegerLiteralBinNoUnderscore,
1295 State.IntegerLiteralOctNoUnderscore,
1296 State.IntegerLiteralHexNoUnderscore,
1297 State.FloatFractionDecNoUnderscore,
1298 State.FloatFractionHexNoUnderscore,
1299 State.FloatExponentNumberNoUnderscore,
1189 State.FloatExponentUnsigned,1300 State.FloatExponentUnsigned,
1190 State.FloatExponentUnsignedHex,
1191 State.SawAtSign,1301 State.SawAtSign,
1192 State.Backslash,1302 State.Backslash,
1193 State.CharLiteral,1303 State.CharLiteral,
...@@ -1585,6 +1695,217 @@ test "correctly parse pointer assignment" {...@@ -1585,6 +1695,217 @@ test "correctly parse pointer assignment" {
1585 });1695 });
1586}1696}
15871697
1698test "tokenizer - number literals decimal" {
1699 testTokenize("1", &[_]Token.Id{ .IntegerLiteral });
1700 testTokenize("2", &[_]Token.Id{ .IntegerLiteral });
1701 testTokenize("3", &[_]Token.Id{ .IntegerLiteral });
1702 testTokenize("4", &[_]Token.Id{ .IntegerLiteral });
1703 testTokenize("5", &[_]Token.Id{ .IntegerLiteral });
1704 testTokenize("6", &[_]Token.Id{ .IntegerLiteral });
1705 testTokenize("7", &[_]Token.Id{ .IntegerLiteral });
1706 testTokenize("8", &[_]Token.Id{ .IntegerLiteral });
1707 testTokenize("9", &[_]Token.Id{ .IntegerLiteral });
1708 testTokenize("0a", &[_]Token.Id{ .Invalid });
1709 testTokenize("9b", &[_]Token.Id{ .Invalid });
1710 testTokenize("1z", &[_]Token.Id{ .Invalid });
1711 testTokenize("1z_1", &[_]Token.Id{ .Invalid });
1712 testTokenize("9z3", &[_]Token.Id{ .Invalid });
1713
1714 testTokenize("0_0", &[_]Token.Id{ .IntegerLiteral });
1715 testTokenize("0001", &[_]Token.Id{ .IntegerLiteral });
1716 testTokenize("01234567890", &[_]Token.Id{ .IntegerLiteral });
1717 testTokenize("012_345_6789_0", &[_]Token.Id{ .IntegerLiteral });
1718 testTokenize("0_1_2_3_4_5_6_7_8_9_0", &[_]Token.Id{ .IntegerLiteral });
1719
1720 testTokenize("00_", &[_]Token.Id{ .Invalid });
1721 testTokenize("0_0_", &[_]Token.Id{ .Invalid });
1722 testTokenize("0__0", &[_]Token.Id{ .Invalid });
1723 testTokenize("0_0f", &[_]Token.Id{ .Invalid });
1724 testTokenize("0_0_f", &[_]Token.Id{ .Invalid });
1725 testTokenize("1_,", &[_]Token.Id{ .Invalid });
1726
1727 testTokenize("1.", &[_]Token.Id{ .FloatLiteral });
1728 testTokenize("0.0", &[_]Token.Id{ .FloatLiteral });
1729 testTokenize("1.0", &[_]Token.Id{ .FloatLiteral });
1730 testTokenize("10.0", &[_]Token.Id{ .FloatLiteral });
1731 testTokenize("1e0", &[_]Token.Id{ .FloatLiteral });
1732 testTokenize("1e100", &[_]Token.Id{ .FloatLiteral });
1733 testTokenize("1.e100", &[_]Token.Id{ .FloatLiteral });
1734 testTokenize("1.0e100", &[_]Token.Id{ .FloatLiteral });
1735 testTokenize("1.0e+100", &[_]Token.Id{ .FloatLiteral });
1736 testTokenize("1.0e-100", &[_]Token.Id{ .FloatLiteral });
1737 testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{ .FloatLiteral });
1738
1739 testTokenize("1e", &[_]Token.Id{ .Invalid });
1740 testTokenize("1.0e1f0", &[_]Token.Id{ .Invalid });
1741 testTokenize("1.0p100", &[_]Token.Id{ .Invalid });
1742 testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Minus, .IntegerLiteral });
1743 testTokenize("1.0p1f0", &[_]Token.Id{ .Invalid });
1744 testTokenize("1.0_,", &[_]Token.Id{ .Invalid });
1745 testTokenize("1.0e,", &[_]Token.Id{ .Invalid });
1746}
1747
1748
1749test "tokenizer - number literals binary" {
1750 testTokenize("0b0", &[_]Token.Id{ .IntegerLiteral });
1751 testTokenize("0b1", &[_]Token.Id{ .IntegerLiteral });
1752 testTokenize("0b2", &[_]Token.Id{ .Invalid });
1753 testTokenize("0b3", &[_]Token.Id{ .Invalid });
1754 testTokenize("0b4", &[_]Token.Id{ .Invalid });
1755 testTokenize("0b5", &[_]Token.Id{ .Invalid });
1756 testTokenize("0b6", &[_]Token.Id{ .Invalid });
1757 testTokenize("0b7", &[_]Token.Id{ .Invalid });
1758 testTokenize("0b8", &[_]Token.Id{ .Invalid });
1759 testTokenize("0b9", &[_]Token.Id{ .Invalid });
1760 testTokenize("0ba", &[_]Token.Id{ .Invalid });
1761 testTokenize("0bb", &[_]Token.Id{ .Invalid });
1762 testTokenize("0bc", &[_]Token.Id{ .Invalid });
1763 testTokenize("0bd", &[_]Token.Id{ .Invalid });
1764 testTokenize("0be", &[_]Token.Id{ .Invalid });
1765 testTokenize("0bf", &[_]Token.Id{ .Invalid });
1766 testTokenize("0bz", &[_]Token.Id{ .Invalid });
1767
1768 testTokenize("0b0000_0000", &[_]Token.Id{ .IntegerLiteral });
1769 testTokenize("0b1111_1111", &[_]Token.Id{ .IntegerLiteral });
1770 testTokenize("0b10_10_10_10", &[_]Token.Id{ .IntegerLiteral });
1771 testTokenize("0b0_1_0_1_0_1_0_1", &[_]Token.Id{ .IntegerLiteral });
1772 testTokenize("0b1.", &[_]Token.Id{ .IntegerLiteral, .Period });
1773 testTokenize("0b1.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
1774
1775 testTokenize("0B0", &[_]Token.Id{ .Invalid });
1776 testTokenize("0b_", &[_]Token.Id{ .Invalid });
1777 testTokenize("0b_0", &[_]Token.Id{ .Invalid });
1778 testTokenize("0b1_", &[_]Token.Id{ .Invalid });
1779 testTokenize("0b0__1", &[_]Token.Id{ .Invalid });
1780 testTokenize("0b0_1_", &[_]Token.Id{ .Invalid });
1781 testTokenize("0b1e", &[_]Token.Id{ .Invalid });
1782 testTokenize("0b1p", &[_]Token.Id{ .Invalid });
1783 testTokenize("0b1e0", &[_]Token.Id{ .Invalid });
1784 testTokenize("0b1p0", &[_]Token.Id{ .Invalid });
1785 testTokenize("0b1_,", &[_]Token.Id{ .Invalid });
1786}
1787
1788test "tokenizer - number literals octal" {
1789 testTokenize("0o0", &[_]Token.Id{ .IntegerLiteral });
1790 testTokenize("0o1", &[_]Token.Id{ .IntegerLiteral });
1791 testTokenize("0o2", &[_]Token.Id{ .IntegerLiteral });
1792 testTokenize("0o3", &[_]Token.Id{ .IntegerLiteral });
1793 testTokenize("0o4", &[_]Token.Id{ .IntegerLiteral });
1794 testTokenize("0o5", &[_]Token.Id{ .IntegerLiteral });
1795 testTokenize("0o6", &[_]Token.Id{ .IntegerLiteral });
1796 testTokenize("0o7", &[_]Token.Id{ .IntegerLiteral });
1797 testTokenize("0o8", &[_]Token.Id{ .Invalid });
1798 testTokenize("0o9", &[_]Token.Id{ .Invalid });
1799 testTokenize("0oa", &[_]Token.Id{ .Invalid });
1800 testTokenize("0ob", &[_]Token.Id{ .Invalid });
1801 testTokenize("0oc", &[_]Token.Id{ .Invalid });
1802 testTokenize("0od", &[_]Token.Id{ .Invalid });
1803 testTokenize("0oe", &[_]Token.Id{ .Invalid });
1804 testTokenize("0of", &[_]Token.Id{ .Invalid });
1805 testTokenize("0oz", &[_]Token.Id{ .Invalid });
1806
1807 testTokenize("0o01234567", &[_]Token.Id{ .IntegerLiteral });
1808 testTokenize("0o0123_4567", &[_]Token.Id{ .IntegerLiteral });
1809 testTokenize("0o01_23_45_67", &[_]Token.Id{ .IntegerLiteral });
1810 testTokenize("0o0_1_2_3_4_5_6_7", &[_]Token.Id{ .IntegerLiteral });
1811 testTokenize("0o7.", &[_]Token.Id{ .IntegerLiteral, .Period });
1812 testTokenize("0o7.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
1813
1814 testTokenize("0O0", &[_]Token.Id{ .Invalid });
1815 testTokenize("0o_", &[_]Token.Id{ .Invalid });
1816 testTokenize("0o_0", &[_]Token.Id{ .Invalid });
1817 testTokenize("0o1_", &[_]Token.Id{ .Invalid });
1818 testTokenize("0o0__1", &[_]Token.Id{ .Invalid });
1819 testTokenize("0o0_1_", &[_]Token.Id{ .Invalid });
1820 testTokenize("0o1e", &[_]Token.Id{ .Invalid });
1821 testTokenize("0o1p", &[_]Token.Id{ .Invalid });
1822 testTokenize("0o1e0", &[_]Token.Id{ .Invalid });
1823 testTokenize("0o1p0", &[_]Token.Id{ .Invalid });
1824 testTokenize("0o_,", &[_]Token.Id{ .Invalid });
1825}
1826
1827test "tokenizer - number literals hexadeciaml" {
1828 testTokenize("0x0", &[_]Token.Id{ .IntegerLiteral });
1829 testTokenize("0x1", &[_]Token.Id{ .IntegerLiteral });
1830 testTokenize("0x2", &[_]Token.Id{ .IntegerLiteral });
1831 testTokenize("0x3", &[_]Token.Id{ .IntegerLiteral });
1832 testTokenize("0x4", &[_]Token.Id{ .IntegerLiteral });
1833 testTokenize("0x5", &[_]Token.Id{ .IntegerLiteral });
1834 testTokenize("0x6", &[_]Token.Id{ .IntegerLiteral });
1835 testTokenize("0x7", &[_]Token.Id{ .IntegerLiteral });
1836 testTokenize("0x8", &[_]Token.Id{ .IntegerLiteral });
1837 testTokenize("0x9", &[_]Token.Id{ .IntegerLiteral });
1838 testTokenize("0xa", &[_]Token.Id{ .IntegerLiteral });
1839 testTokenize("0xb", &[_]Token.Id{ .IntegerLiteral });
1840 testTokenize("0xc", &[_]Token.Id{ .IntegerLiteral });
1841 testTokenize("0xd", &[_]Token.Id{ .IntegerLiteral });
1842 testTokenize("0xe", &[_]Token.Id{ .IntegerLiteral });
1843 testTokenize("0xf", &[_]Token.Id{ .IntegerLiteral });
1844 testTokenize("0xA", &[_]Token.Id{ .IntegerLiteral });
1845 testTokenize("0xB", &[_]Token.Id{ .IntegerLiteral });
1846 testTokenize("0xC", &[_]Token.Id{ .IntegerLiteral });
1847 testTokenize("0xD", &[_]Token.Id{ .IntegerLiteral });
1848 testTokenize("0xE", &[_]Token.Id{ .IntegerLiteral });
1849 testTokenize("0xF", &[_]Token.Id{ .IntegerLiteral });
1850 testTokenize("0x0z", &[_]Token.Id{ .Invalid });
1851 testTokenize("0xz", &[_]Token.Id{ .Invalid });
1852
1853 testTokenize("0x0123456789ABCDEF", &[_]Token.Id{ .IntegerLiteral });
1854 testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{ .IntegerLiteral });
1855 testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{ .IntegerLiteral });
1856 testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{ .IntegerLiteral });
1857
1858 testTokenize("0X0", &[_]Token.Id{ .Invalid });
1859 testTokenize("0x_", &[_]Token.Id{ .Invalid });
1860 testTokenize("0x_1", &[_]Token.Id{ .Invalid });
1861 testTokenize("0x1_", &[_]Token.Id{ .Invalid });
1862 testTokenize("0x0__1", &[_]Token.Id{ .Invalid });
1863 testTokenize("0x0_1_", &[_]Token.Id{ .Invalid });
1864 testTokenize("0x_,", &[_]Token.Id{ .Invalid });
1865
1866 testTokenize("0x1.", &[_]Token.Id{ .FloatLiteral });
1867 testTokenize("0x1.0", &[_]Token.Id{ .FloatLiteral });
1868 testTokenize("0xF.", &[_]Token.Id{ .FloatLiteral });
1869 testTokenize("0xF.0", &[_]Token.Id{ .FloatLiteral });
1870 testTokenize("0xF.F", &[_]Token.Id{ .FloatLiteral });
1871 testTokenize("0xF.Fp0", &[_]Token.Id{ .FloatLiteral });
1872 testTokenize("0xF.FP0", &[_]Token.Id{ .FloatLiteral });
1873 testTokenize("0x1p0", &[_]Token.Id{ .FloatLiteral });
1874 testTokenize("0xfp0", &[_]Token.Id{ .FloatLiteral });
1875
1876 testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{ .FloatLiteral });
1877 testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{ .FloatLiteral });
1878 testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{ .FloatLiteral });
1879 testTokenize("0x0.0p0", &[_]Token.Id{ .FloatLiteral });
1880 testTokenize("0xff.ffp10", &[_]Token.Id{ .FloatLiteral });
1881 testTokenize("0xff.ffP10", &[_]Token.Id{ .FloatLiteral });
1882 testTokenize("0xff.p10", &[_]Token.Id{ .FloatLiteral });
1883 testTokenize("0xffp10", &[_]Token.Id{ .FloatLiteral });
1884 testTokenize("0xff_ff.ff_ffp1_0_0_0", &[_]Token.Id{ .FloatLiteral });
1885 testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &[_]Token.Id{ .FloatLiteral });
1886 testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &[_]Token.Id{ .FloatLiteral });
1887
1888 testTokenize("0x1e", &[_]Token.Id{ .IntegerLiteral });
1889 testTokenize("0x1e0", &[_]Token.Id{ .IntegerLiteral });
1890 testTokenize("0x1p", &[_]Token.Id{ .Invalid });
1891 testTokenize("0xfp0z1", &[_]Token.Id{ .Invalid });
1892 testTokenize("0xff.ffpff", &[_]Token.Id{ .Invalid });
1893 testTokenize("0x0_.0", &[_]Token.Id{ .Invalid });
1894 testTokenize("0x0._0", &[_]Token.Id{ .Invalid });
1895 testTokenize("0x0.0_", &[_]Token.Id{ .Invalid });
1896 testTokenize("0x0_p0", &[_]Token.Id{ .Invalid });
1897 testTokenize("0x0_.p0", &[_]Token.Id{ .Invalid });
1898 testTokenize("0x0._p0", &[_]Token.Id{ .Invalid });
1899 testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid });
1900 testTokenize("0x0._0p0", &[_]Token.Id{ .Invalid });
1901 testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid });
1902 testTokenize("0x0.0p_0", &[_]Token.Id{ .Invalid });
1903 testTokenize("0x0.0p+_0", &[_]Token.Id{ .Invalid });
1904 testTokenize("0x0.0p-_0", &[_]Token.Id{ .Invalid });
1905 testTokenize("0x0.0p0_", &[_]Token.Id{ .Invalid });
1906}
1907
1908
1588fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {1909fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
1589 var tokenizer = Tokenizer.init(source);1910 var tokenizer = Tokenizer.init(source);
1590 for (expected_tokens) |expected_token_id| {1911 for (expected_tokens) |expected_token_id| {