authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2021-03-03 11:58:09+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-03-03 11:33:14-08:00
log904f774563c69feae9f73a8ee9637e3b760ecd3a
treeabad7a0f88543637be874bd5497c5c02a5f19334
parent7fbe9e7d60fa17e49177f52df922bbf295ecc619

translate-c: fix c tokenizer giving invalid tokens

3 files changed, 33 insertions(+), 3 deletions(-)

lib/std/c/tokenizer.zig+30-2
......@@ -401,7 +401,9 @@ pub const Tokenizer = struct {
401401 Zero,
402402 IntegerLiteralOct,
403403 IntegerLiteralBinary,
404 IntegerLiteralBinaryFirst,
404405 IntegerLiteralHex,
406 IntegerLiteralHexFirst,
405407 IntegerLiteral,
406408 IntegerSuffix,
407409 IntegerSuffixU,
......@@ -1046,10 +1048,10 @@ pub const Tokenizer = struct {
10461048 state = .IntegerLiteralOct;
10471049 },
10481050 'b', 'B' => {
1049 state = .IntegerLiteralBinary;
1051 state = .IntegerLiteralBinaryFirst;
10501052 },
10511053 'x', 'X' => {
1052 state = .IntegerLiteralHex;
1054 state = .IntegerLiteralHexFirst;
10531055 },
10541056 '.' => {
10551057 state = .FloatFraction;
......@@ -1066,6 +1068,13 @@ pub const Tokenizer = struct {
10661068 self.index -= 1;
10671069 },
10681070 },
1071 .IntegerLiteralBinaryFirst => switch (c) {
1072 '0'...'7' => state = .IntegerLiteralBinary,
1073 else => {
1074 result.id = .Invalid;
1075 break;
1076 },
1077 },
10691078 .IntegerLiteralBinary => switch (c) {
10701079 '0', '1' => {},
10711080 else => {
......@@ -1073,6 +1082,19 @@ pub const Tokenizer = struct {
10731082 self.index -= 1;
10741083 },
10751084 },
1085 .IntegerLiteralHexFirst => switch (c) {
1086 '0'...'9', 'a'...'f', 'A'...'F' => state = .IntegerLiteralHex,
1087 '.' => {
1088 state = .FloatFractionHex;
1089 },
1090 'p', 'P' => {
1091 state = .FloatExponent;
1092 },
1093 else => {
1094 result.id = .Invalid;
1095 break;
1096 },
1097 },
10761098 .IntegerLiteralHex => switch (c) {
10771099 '0'...'9', 'a'...'f', 'A'...'F' => {},
10781100 '.' => {
......@@ -1238,6 +1260,8 @@ pub const Tokenizer = struct {
12381260 .MultiLineCommentAsterisk,
12391261 .FloatExponent,
12401262 .MacroString,
1263 .IntegerLiteralBinaryFirst,
1264 .IntegerLiteralHexFirst,
12411265 => result.id = .Invalid,
12421266
12431267 .FloatExponentDigits => result.id = if (counter == 0) .Invalid else .{ .FloatLiteral = .none },
......@@ -1523,6 +1547,7 @@ test "num suffixes" {
15231547 \\ 1.0f 1.0L 1.0 .0 1.
15241548 \\ 0l 0lu 0ll 0llu 0
15251549 \\ 1u 1ul 1ull 1
1550 \\ 0x 0b
15261551 \\
15271552 , &[_]Token.Id{
15281553 .{ .FloatLiteral = .f },
......@@ -1542,6 +1567,9 @@ test "num suffixes" {
15421567 .{ .IntegerLiteral = .llu },
15431568 .{ .IntegerLiteral = .none },
15441569 .Nl,
1570 .Invalid,
1571 .Invalid,
1572 .Nl,
15451573 });
15461574}
15471575
lib/std/multi_array_list.zig+1-1
......@@ -136,7 +136,7 @@ pub fn MultiArrayList(comptime S: type) type {
136136 const slices = self.slice();
137137 var result: S = undefined;
138138 inline for (fields) |field_info, i| {
139 @field(elem, field_info.name) = slices.items(@intToEnum(Field, i))[index];
139 @field(result, field_info.name) = slices.items(@intToEnum(Field, i))[index];
140140 }
141141 return result;
142142 }
test/run_translated_c.zig+2
......@@ -27,6 +27,8 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void {
2727 \\#define FOO =
2828 \\#define PtrToPtr64(p) ((void *POINTER_64) p)
2929 \\#define STRUC_ALIGNED_STACK_COPY(t,s) ((CONST t *)(s))
30 \\#define bar = 0x
31 \\#define baz = 0b
3032 \\int main(void) {}
3133 , "");
3234