authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 11:23:19+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:50+02:00
loga5d1fb1e49891c70fd384e1cf38e9d2f4eac6ee9
tree28042a348e454853a8e02af4d204e6b52c5a7620
parentc221593d7d6d441c04c9332aaa6d2be8b3d24bc0
signaturelock-open Commit is signed but in an unrecognized format.

std-c tokenizer line continuation, tests and fixes

1 files changed, 135 insertions(+), 6 deletions(-)

lib/std/c/tokenizer.zig+135-6
......@@ -265,13 +265,17 @@ pub const Tokenizer = struct {
265265 var state: enum {
266266 Start,
267267 Cr,
268 BackSlash,
269 BackSlashCr,
268270 u,
269271 u8,
270272 U,
271273 L,
272274 StringLiteral,
275 CharLiteralStart,
273276 CharLiteral,
274277 EscapeSequence,
278 CrEscape,
275279 OctalEscape,
276280 HexEscape,
277281 UnicodeEscape,
......@@ -344,7 +348,7 @@ pub const Tokenizer = struct {
344348 },
345349 '\'' => {
346350 result.id = .{ .CharLiteral = .None };
347 state = .CharLiteral;
351 state = .CharLiteralStart;
348352 },
349353 'u' => {
350354 state = .u;
......@@ -464,6 +468,9 @@ pub const Tokenizer = struct {
464468 '1'...'9' => {
465469 state = .IntegerLiteral;
466470 },
471 '\\' => {
472 state = .BackSlash;
473 },
467474 else => {
468475 result.start = self.index + 1;
469476 },
......@@ -480,13 +487,34 @@ pub const Tokenizer = struct {
480487 break;
481488 },
482489 },
490 .BackSlash => switch (c) {
491 '\n' => {
492 state = .Start;
493 },
494 '\r' => {
495 state = .BackSlashCr;
496 },
497 else => {
498 result.id = .Invalid;
499 break;
500 },
501 },
502 .BackSlashCr => switch (c) {
503 '\n' => {
504 state = .Start;
505 },
506 else => {
507 result.id = .Invalid;
508 break;
509 },
510 },
483511 .u => switch (c) {
484512 '8' => {
485513 state = .u8;
486514 },
487515 '\'' => {
488516 result.id = .{ .CharLiteral = .Utf16 };
489 state = .CharLiteral;
517 state = .CharLiteralStart;
490518 },
491519 '\"' => {
492520 result.id = .{ .StringLiteral = .Utf16 };
......@@ -508,7 +536,7 @@ pub const Tokenizer = struct {
508536 .U => switch (c) {
509537 '\'' => {
510538 result.id = .{ .CharLiteral = .Utf32 };
511 state = .CharLiteral;
539 state = .CharLiteralStart;
512540 },
513541 '\"' => {
514542 result.id = .{ .StringLiteral = .Utf32 };
......@@ -521,7 +549,7 @@ pub const Tokenizer = struct {
521549 .L => switch (c) {
522550 '\'' => {
523551 result.id = .{ .CharLiteral = .Wide };
524 state = .CharLiteral;
552 state = .CharLiteralStart;
525553 },
526554 '\"' => {
527555 result.id = .{ .StringLiteral = .Wide };
......@@ -546,7 +574,7 @@ pub const Tokenizer = struct {
546574 },
547575 else => {},
548576 },
549 .CharLiteral => switch (c) {
577 .CharLiteralStart => switch (c) {
550578 '\\' => {
551579 string = false;
552580 state = .EscapeSequence;
......@@ -555,10 +583,32 @@ pub const Tokenizer = struct {
555583 result.id = .Invalid;
556584 break;
557585 },
586 else => {
587 state = .CharLiteral;
588 },
589 },
590 .CharLiteral => switch (c) {
591 '\\' => {
592 string = false;
593 state = .EscapeSequence;
594 },
595 '\'' => {
596 self.index += 1;
597 break;
598 },
599 '\n' => {
600 result.id = .Invalid;
601 break;
602 },
558603 else => {},
559604 },
560605 .EscapeSequence => switch (c) {
561 '\'', '"', '?', '\\', 'a', 'b', 'f', 'n', 'r', 't', 'v' => {},
606 '\'', '"', '?', '\\', 'a', 'b', 'f', 'n', 'r', 't', 'v', '\n' => {
607 state = if (string) .StringLiteral else .CharLiteral;
608 },
609 '\r' => {
610 state = .CrEscape;
611 },
562612 '0'...'7' => {
563613 counter = 1;
564614 state = .OctalEscape;
......@@ -579,6 +629,15 @@ pub const Tokenizer = struct {
579629 break;
580630 },
581631 },
632 .CrEscape => switch (c) {
633 '\n' => {
634 state = if (string) .StringLiteral else .CharLiteral;
635 },
636 else => {
637 result.id = .Invalid;
638 break;
639 },
640 },
582641 .OctalEscape => switch (c) {
583642 '0'...'7' => {
584643 counter += 1;
......@@ -1056,10 +1115,14 @@ pub const Tokenizer = struct {
10561115 },
10571116
10581117 .Cr,
1118 .BackSlash,
1119 .BackSlashCr,
10591120 .Period2,
10601121 .StringLiteral,
1122 .CharLiteralStart,
10611123 .CharLiteral,
10621124 .EscapeSequence,
1125 .CrEscape,
10631126 .OctalEscape,
10641127 .HexEscape,
10651128 .UnicodeEscape,
......@@ -1269,6 +1332,72 @@ test "preprocessor keywords" {
12691332 });
12701333}
12711334
1335test "line continuation" {
1336 expectTokens(
1337 \\#define foo \
1338 \\ bar
1339 \\"foo\
1340 \\ bar"
1341 \\
1342 , &[_]Token.Id{
1343 .Hash,
1344 .Keyword_define,
1345 .Identifier,
1346 .Identifier,
1347 .Nl,
1348 .{ .StringLiteral = .None },
1349 });
1350}
1351
1352test "string prefix" {
1353 expectTokens(
1354 \\"foo"
1355 \\u"foo"
1356 \\u8"foo"
1357 \\U"foo"
1358 \\L"foo"
1359 \\'foo'
1360 \\u'foo'
1361 \\U'foo'
1362 \\L'foo'
1363 \\
1364 , &[_]Token.Id{
1365 .{ .StringLiteral = .None },
1366 .{ .StringLiteral = .Utf16 },
1367 .{ .StringLiteral = .Utf8 },
1368 .{ .StringLiteral = .Utf32 },
1369 .{ .StringLiteral = .Wide },
1370 .{ .CharLiteral = .None },
1371 .{ .CharLiteral = .Utf16 },
1372 .{ .CharLiteral = .Utf32 },
1373 .{ .CharLiteral = .Wide },
1374 });
1375}
1376
1377test "num suffixes" {
1378 expectTokens(
1379 \\ 1.0f 1.0L 1.0 .0 1.
1380 \\ 0l 0lu 0ll 0llu 0
1381 \\ 1u 1ul 1ull 1
1382 \\
1383 , &[_]Token.Id{
1384 .{ .FloatLiteral = .F },
1385 .{ .FloatLiteral = .L },
1386 .{ .FloatLiteral = .None },
1387 .{ .FloatLiteral = .None },
1388 .{ .FloatLiteral = .None },
1389 .{ .IntegerLiteral = .L },
1390 .{ .IntegerLiteral = .LU },
1391 .{ .IntegerLiteral = .LL },
1392 .{ .IntegerLiteral = .LLU },
1393 .{ .IntegerLiteral = .None },
1394 .{ .IntegerLiteral = .U },
1395 .{ .IntegerLiteral = .LU },
1396 .{ .IntegerLiteral = .LLU },
1397 .{ .IntegerLiteral = .None },
1398 });
1399}
1400
12721401fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void {
12731402 var tokenizer = Tokenizer{
12741403 .source = &Source{