authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 11:23:19+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:50+02:00
loga5d1fb1e49891c70fd384e1cf38e9d2f4eac6ee9
tree28042a348e454853a8e02af4d204e6b52c5a7620
parentc221593d7d6d441c04c9332aaa6d2be8b3d24bc0
signaturelock-open Commit is signed but in an unrecognized format.

std-c tokenizer line continuation, tests and fixes

1 files changed, 135 insertions(+), 6 deletions(-)

lib/std/c/tokenizer.zig+135-6
...@@ -265,13 +265,17 @@ pub const Tokenizer = struct {...@@ -265,13 +265,17 @@ pub const Tokenizer = struct {
265 var state: enum {265 var state: enum {
266 Start,266 Start,
267 Cr,267 Cr,
268 BackSlash,
269 BackSlashCr,
268 u,270 u,
269 u8,271 u8,
270 U,272 U,
271 L,273 L,
272 StringLiteral,274 StringLiteral,
275 CharLiteralStart,
273 CharLiteral,276 CharLiteral,
274 EscapeSequence,277 EscapeSequence,
278 CrEscape,
275 OctalEscape,279 OctalEscape,
276 HexEscape,280 HexEscape,
277 UnicodeEscape,281 UnicodeEscape,
...@@ -344,7 +348,7 @@ pub const Tokenizer = struct {...@@ -344,7 +348,7 @@ pub const Tokenizer = struct {
344 },348 },
345 '\'' => {349 '\'' => {
346 result.id = .{ .CharLiteral = .None };350 result.id = .{ .CharLiteral = .None };
347 state = .CharLiteral;351 state = .CharLiteralStart;
348 },352 },
349 'u' => {353 'u' => {
350 state = .u;354 state = .u;
...@@ -464,6 +468,9 @@ pub const Tokenizer = struct {...@@ -464,6 +468,9 @@ pub const Tokenizer = struct {
464 '1'...'9' => {468 '1'...'9' => {
465 state = .IntegerLiteral;469 state = .IntegerLiteral;
466 },470 },
471 '\\' => {
472 state = .BackSlash;
473 },
467 else => {474 else => {
468 result.start = self.index + 1;475 result.start = self.index + 1;
469 },476 },
...@@ -480,13 +487,34 @@ pub const Tokenizer = struct {...@@ -480,13 +487,34 @@ pub const Tokenizer = struct {
480 break;487 break;
481 },488 },
482 },489 },
490 .BackSlash => switch (c) {
491 '\n' => {
492 state = .Start;
493 },
494 '\r' => {
495 state = .BackSlashCr;
496 },
497 else => {
498 result.id = .Invalid;
499 break;
500 },
501 },
502 .BackSlashCr => switch (c) {
503 '\n' => {
504 state = .Start;
505 },
506 else => {
507 result.id = .Invalid;
508 break;
509 },
510 },
483 .u => switch (c) {511 .u => switch (c) {
484 '8' => {512 '8' => {
485 state = .u8;513 state = .u8;
486 },514 },
487 '\'' => {515 '\'' => {
488 result.id = .{ .CharLiteral = .Utf16 };516 result.id = .{ .CharLiteral = .Utf16 };
489 state = .CharLiteral;517 state = .CharLiteralStart;
490 },518 },
491 '\"' => {519 '\"' => {
492 result.id = .{ .StringLiteral = .Utf16 };520 result.id = .{ .StringLiteral = .Utf16 };
...@@ -508,7 +536,7 @@ pub const Tokenizer = struct {...@@ -508,7 +536,7 @@ pub const Tokenizer = struct {
508 .U => switch (c) {536 .U => switch (c) {
509 '\'' => {537 '\'' => {
510 result.id = .{ .CharLiteral = .Utf32 };538 result.id = .{ .CharLiteral = .Utf32 };
511 state = .CharLiteral;539 state = .CharLiteralStart;
512 },540 },
513 '\"' => {541 '\"' => {
514 result.id = .{ .StringLiteral = .Utf32 };542 result.id = .{ .StringLiteral = .Utf32 };
...@@ -521,7 +549,7 @@ pub const Tokenizer = struct {...@@ -521,7 +549,7 @@ pub const Tokenizer = struct {
521 .L => switch (c) {549 .L => switch (c) {
522 '\'' => {550 '\'' => {
523 result.id = .{ .CharLiteral = .Wide };551 result.id = .{ .CharLiteral = .Wide };
524 state = .CharLiteral;552 state = .CharLiteralStart;
525 },553 },
526 '\"' => {554 '\"' => {
527 result.id = .{ .StringLiteral = .Wide };555 result.id = .{ .StringLiteral = .Wide };
...@@ -546,7 +574,7 @@ pub const Tokenizer = struct {...@@ -546,7 +574,7 @@ pub const Tokenizer = struct {
546 },574 },
547 else => {},575 else => {},
548 },576 },
549 .CharLiteral => switch (c) {577 .CharLiteralStart => switch (c) {
550 '\\' => {578 '\\' => {
551 string = false;579 string = false;
552 state = .EscapeSequence;580 state = .EscapeSequence;
...@@ -555,10 +583,32 @@ pub const Tokenizer = struct {...@@ -555,10 +583,32 @@ pub const Tokenizer = struct {
555 result.id = .Invalid;583 result.id = .Invalid;
556 break;584 break;
557 },585 },
586 else => {
587 state = .CharLiteral;
588 },
589 },
590 .CharLiteral => switch (c) {
591 '\\' => {
592 string = false;
593 state = .EscapeSequence;
594 },
595 '\'' => {
596 self.index += 1;
597 break;
598 },
599 '\n' => {
600 result.id = .Invalid;
601 break;
602 },
558 else => {},603 else => {},
559 },604 },
560 .EscapeSequence => switch (c) {605 .EscapeSequence => switch (c) {
561 '\'', '"', '?', '\\', 'a', 'b', 'f', 'n', 'r', 't', 'v' => {},606 '\'', '"', '?', '\\', 'a', 'b', 'f', 'n', 'r', 't', 'v', '\n' => {
607 state = if (string) .StringLiteral else .CharLiteral;
608 },
609 '\r' => {
610 state = .CrEscape;
611 },
562 '0'...'7' => {612 '0'...'7' => {
563 counter = 1;613 counter = 1;
564 state = .OctalEscape;614 state = .OctalEscape;
...@@ -579,6 +629,15 @@ pub const Tokenizer = struct {...@@ -579,6 +629,15 @@ pub const Tokenizer = struct {
579 break;629 break;
580 },630 },
581 },631 },
632 .CrEscape => switch (c) {
633 '\n' => {
634 state = if (string) .StringLiteral else .CharLiteral;
635 },
636 else => {
637 result.id = .Invalid;
638 break;
639 },
640 },
582 .OctalEscape => switch (c) {641 .OctalEscape => switch (c) {
583 '0'...'7' => {642 '0'...'7' => {
584 counter += 1;643 counter += 1;
...@@ -1056,10 +1115,14 @@ pub const Tokenizer = struct {...@@ -1056,10 +1115,14 @@ pub const Tokenizer = struct {
1056 },1115 },
10571116
1058 .Cr,1117 .Cr,
1118 .BackSlash,
1119 .BackSlashCr,
1059 .Period2,1120 .Period2,
1060 .StringLiteral,1121 .StringLiteral,
1122 .CharLiteralStart,
1061 .CharLiteral,1123 .CharLiteral,
1062 .EscapeSequence,1124 .EscapeSequence,
1125 .CrEscape,
1063 .OctalEscape,1126 .OctalEscape,
1064 .HexEscape,1127 .HexEscape,
1065 .UnicodeEscape,1128 .UnicodeEscape,
...@@ -1269,6 +1332,72 @@ test "preprocessor keywords" {...@@ -1269,6 +1332,72 @@ test "preprocessor keywords" {
1269 });1332 });
1270}1333}
12711334
1335test "line continuation" {
1336 expectTokens(
1337 \\#define foo \
1338 \\ bar
1339 \\"foo\
1340 \\ bar"
1341 \\
1342 , &[_]Token.Id{
1343 .Hash,
1344 .Keyword_define,
1345 .Identifier,
1346 .Identifier,
1347 .Nl,
1348 .{ .StringLiteral = .None },
1349 });
1350}
1351
1352test "string prefix" {
1353 expectTokens(
1354 \\"foo"
1355 \\u"foo"
1356 \\u8"foo"
1357 \\U"foo"
1358 \\L"foo"
1359 \\'foo'
1360 \\u'foo'
1361 \\U'foo'
1362 \\L'foo'
1363 \\
1364 , &[_]Token.Id{
1365 .{ .StringLiteral = .None },
1366 .{ .StringLiteral = .Utf16 },
1367 .{ .StringLiteral = .Utf8 },
1368 .{ .StringLiteral = .Utf32 },
1369 .{ .StringLiteral = .Wide },
1370 .{ .CharLiteral = .None },
1371 .{ .CharLiteral = .Utf16 },
1372 .{ .CharLiteral = .Utf32 },
1373 .{ .CharLiteral = .Wide },
1374 });
1375}
1376
1377test "num suffixes" {
1378 expectTokens(
1379 \\ 1.0f 1.0L 1.0 .0 1.
1380 \\ 0l 0lu 0ll 0llu 0
1381 \\ 1u 1ul 1ull 1
1382 \\
1383 , &[_]Token.Id{
1384 .{ .FloatLiteral = .F },
1385 .{ .FloatLiteral = .L },
1386 .{ .FloatLiteral = .None },
1387 .{ .FloatLiteral = .None },
1388 .{ .FloatLiteral = .None },
1389 .{ .IntegerLiteral = .L },
1390 .{ .IntegerLiteral = .LU },
1391 .{ .IntegerLiteral = .LL },
1392 .{ .IntegerLiteral = .LLU },
1393 .{ .IntegerLiteral = .None },
1394 .{ .IntegerLiteral = .U },
1395 .{ .IntegerLiteral = .LU },
1396 .{ .IntegerLiteral = .LLU },
1397 .{ .IntegerLiteral = .None },
1398 });
1399}
1400
1272fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void {1401fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void {
1273 var tokenizer = Tokenizer{1402 var tokenizer = Tokenizer{
1274 .source = &Source{1403 .source = &Source{