authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2021-03-03 11:58:09+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-03-03 11:33:14-08:00
log904f774563c69feae9f73a8ee9637e3b760ecd3a
treeabad7a0f88543637be874bd5497c5c02a5f19334
parent7fbe9e7d60fa17e49177f52df922bbf295ecc619

translate-c: fix c tokenizer giving invalid tokens

3 files changed, 33 insertions(+), 3 deletions(-)

lib/std/c/tokenizer.zig+30-2
...@@ -401,7 +401,9 @@ pub const Tokenizer = struct {...@@ -401,7 +401,9 @@ pub const Tokenizer = struct {
401 Zero,401 Zero,
402 IntegerLiteralOct,402 IntegerLiteralOct,
403 IntegerLiteralBinary,403 IntegerLiteralBinary,
404 IntegerLiteralBinaryFirst,
404 IntegerLiteralHex,405 IntegerLiteralHex,
406 IntegerLiteralHexFirst,
405 IntegerLiteral,407 IntegerLiteral,
406 IntegerSuffix,408 IntegerSuffix,
407 IntegerSuffixU,409 IntegerSuffixU,
...@@ -1046,10 +1048,10 @@ pub const Tokenizer = struct {...@@ -1046,10 +1048,10 @@ pub const Tokenizer = struct {
1046 state = .IntegerLiteralOct;1048 state = .IntegerLiteralOct;
1047 },1049 },
1048 'b', 'B' => {1050 'b', 'B' => {
1049 state = .IntegerLiteralBinary;1051 state = .IntegerLiteralBinaryFirst;
1050 },1052 },
1051 'x', 'X' => {1053 'x', 'X' => {
1052 state = .IntegerLiteralHex;1054 state = .IntegerLiteralHexFirst;
1053 },1055 },
1054 '.' => {1056 '.' => {
1055 state = .FloatFraction;1057 state = .FloatFraction;
...@@ -1066,6 +1068,13 @@ pub const Tokenizer = struct {...@@ -1066,6 +1068,13 @@ pub const Tokenizer = struct {
1066 self.index -= 1;1068 self.index -= 1;
1067 },1069 },
1068 },1070 },
1071 .IntegerLiteralBinaryFirst => switch (c) {
1072 '0'...'7' => state = .IntegerLiteralBinary,
1073 else => {
1074 result.id = .Invalid;
1075 break;
1076 },
1077 },
1069 .IntegerLiteralBinary => switch (c) {1078 .IntegerLiteralBinary => switch (c) {
1070 '0', '1' => {},1079 '0', '1' => {},
1071 else => {1080 else => {
...@@ -1073,6 +1082,19 @@ pub const Tokenizer = struct {...@@ -1073,6 +1082,19 @@ pub const Tokenizer = struct {
1073 self.index -= 1;1082 self.index -= 1;
1074 },1083 },
1075 },1084 },
1085 .IntegerLiteralHexFirst => switch (c) {
1086 '0'...'9', 'a'...'f', 'A'...'F' => state = .IntegerLiteralHex,
1087 '.' => {
1088 state = .FloatFractionHex;
1089 },
1090 'p', 'P' => {
1091 state = .FloatExponent;
1092 },
1093 else => {
1094 result.id = .Invalid;
1095 break;
1096 },
1097 },
1076 .IntegerLiteralHex => switch (c) {1098 .IntegerLiteralHex => switch (c) {
1077 '0'...'9', 'a'...'f', 'A'...'F' => {},1099 '0'...'9', 'a'...'f', 'A'...'F' => {},
1078 '.' => {1100 '.' => {
...@@ -1238,6 +1260,8 @@ pub const Tokenizer = struct {...@@ -1238,6 +1260,8 @@ pub const Tokenizer = struct {
1238 .MultiLineCommentAsterisk,1260 .MultiLineCommentAsterisk,
1239 .FloatExponent,1261 .FloatExponent,
1240 .MacroString,1262 .MacroString,
1263 .IntegerLiteralBinaryFirst,
1264 .IntegerLiteralHexFirst,
1241 => result.id = .Invalid,1265 => result.id = .Invalid,
12421266
1243 .FloatExponentDigits => result.id = if (counter == 0) .Invalid else .{ .FloatLiteral = .none },1267 .FloatExponentDigits => result.id = if (counter == 0) .Invalid else .{ .FloatLiteral = .none },
...@@ -1523,6 +1547,7 @@ test "num suffixes" {...@@ -1523,6 +1547,7 @@ test "num suffixes" {
1523 \\ 1.0f 1.0L 1.0 .0 1.1547 \\ 1.0f 1.0L 1.0 .0 1.
1524 \\ 0l 0lu 0ll 0llu 01548 \\ 0l 0lu 0ll 0llu 0
1525 \\ 1u 1ul 1ull 11549 \\ 1u 1ul 1ull 1
1550 \\ 0x 0b
1526 \\1551 \\
1527 , &[_]Token.Id{1552 , &[_]Token.Id{
1528 .{ .FloatLiteral = .f },1553 .{ .FloatLiteral = .f },
...@@ -1542,6 +1567,9 @@ test "num suffixes" {...@@ -1542,6 +1567,9 @@ test "num suffixes" {
1542 .{ .IntegerLiteral = .llu },1567 .{ .IntegerLiteral = .llu },
1543 .{ .IntegerLiteral = .none },1568 .{ .IntegerLiteral = .none },
1544 .Nl,1569 .Nl,
1570 .Invalid,
1571 .Invalid,
1572 .Nl,
1545 });1573 });
1546}1574}
15471575
lib/std/multi_array_list.zig+1-1
...@@ -136,7 +136,7 @@ pub fn MultiArrayList(comptime S: type) type {...@@ -136,7 +136,7 @@ pub fn MultiArrayList(comptime S: type) type {
136 const slices = self.slice();136 const slices = self.slice();
137 var result: S = undefined;137 var result: S = undefined;
138 inline for (fields) |field_info, i| {138 inline for (fields) |field_info, i| {
139 @field(elem, field_info.name) = slices.items(@intToEnum(Field, i))[index];139 @field(result, field_info.name) = slices.items(@intToEnum(Field, i))[index];
140 }140 }
141 return result;141 return result;
142 }142 }
test/run_translated_c.zig+2
...@@ -27,6 +27,8 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void {...@@ -27,6 +27,8 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void {
27 \\#define FOO =27 \\#define FOO =
28 \\#define PtrToPtr64(p) ((void *POINTER_64) p)28 \\#define PtrToPtr64(p) ((void *POINTER_64) p)
29 \\#define STRUC_ALIGNED_STACK_COPY(t,s) ((CONST t *)(s))29 \\#define STRUC_ALIGNED_STACK_COPY(t,s) ((CONST t *)(s))
30 \\#define bar = 0x
31 \\#define baz = 0b
30 \\int main(void) {}32 \\int main(void) {}
31 , "");33 , "");
3234