authorgravatar for frmdstryr@protonmail.comfrmdstryr <frmdstryr@protonmail.com> 2019-12-29 10:41:26-05:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-29 19:50:58+02:00
log6df9e9fe47ad79f362a57279751374b6966782ed
treef6cb4e698b2fdc43d05ae17deea75a3a58bfe0fa
parentfcc82a219ab20eb4b5e26a6e46219e564ee87810
signature Commit is signed but in an unrecognized format.

Cleanup c_tokenizer.zig tests

1 files changed, 143 insertions(+), 72 deletions(-)

src-self-hosted/c_tokenizer.zig+143-72
...@@ -24,21 +24,26 @@ pub const CToken = struct {...@@ -24,21 +24,26 @@ pub const CToken = struct {
24 RParen,24 RParen,
25 Eof,25 Eof,
26 Dot,26 Dot,
27 Asterisk,27 Asterisk, // *
28 Ampersand,28 Ampersand, // &
29 And,29 And, // &&
30 Or,30 Assign, // =
31 Bang,31 Or, // ||
32 Tilde,32 Bang, // !
33 Shl,33 Tilde, // ~
34 Shr,34 Shl, // <<
35 Lt,35 Shr, // >>
36 Gt,36 Lt, // <
37 Increment,37 Lte, // <=
38 Decrement,38 Gt, // >
39 Gte, // >=
40 Eq, // ==
41 Ne, // !=
42 Increment, // ++
43 Decrement, // --
39 Comma,44 Comma,
40 Fn,45 Fn,
41 Arrow,46 Arrow, // ->
42 LBrace,47 LBrace,
43 RBrace,48 RBrace,
44 Pipe,49 Pipe,
...@@ -239,6 +244,8 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -239,6 +244,8 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
239 GotMinus,244 GotMinus,
240 GotAmpersand,245 GotAmpersand,
241 GotPipe,246 GotPipe,
247 GotBang,
248 GotEq,
242 CharLit,249 CharLit,
243 OpenComment,250 OpenComment,
244 Comment,251 Comment,
...@@ -298,6 +305,8 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -298,6 +305,8 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
298 .GotPlus,305 .GotPlus,
299 .GotAmpersand,306 .GotAmpersand,
300 .GotPipe,307 .GotPipe,
308 .GotBang,
309 .GotEq,
301 => {310 => {
302 return result;311 return result;
303 },312 },
...@@ -387,12 +396,16 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -387,12 +396,16 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
387 },396 },
388 '!' => {397 '!' => {
389 result.id = .Bang;398 result.id = .Bang;
390 state = .Done;399 state = .GotBang;
391 },400 },
392 '~' => {401 '~' => {
393 result.id = .Tilde;402 result.id = .Tilde;
394 state = .Done;403 state = .Done;
395 },404 },
405 '=' => {
406 result.id = .Assign;
407 state = .GotEq;
408 },
396 ',' => {409 ',' => {
397 result.id = .Comma;410 result.id = .Comma;
398 state = .Done;411 state = .Done;
...@@ -438,9 +451,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -438,9 +451,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
438 result.id = .Decrement;451 result.id = .Decrement;
439 state = .Done;452 state = .Done;
440 },453 },
441 else => {454 else => return result,
442 return result;
443 },
444 }455 }
445 },456 },
446 .GotPlus => {457 .GotPlus => {
...@@ -449,9 +460,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -449,9 +460,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
449 result.id = .Increment;460 result.id = .Increment;
450 state = .Done;461 state = .Done;
451 },462 },
452 else => {463 else => return result,
453 return result;
454 },
455 }464 }
456 },465 },
457 .GotLt => {466 .GotLt => {
...@@ -460,9 +469,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -460,9 +469,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
460 result.id = .Shl;469 result.id = .Shl;
461 state = .Done;470 state = .Done;
462 },471 },
463 else => {472 '=' => {
464 return result;473 result.id = .Lte;
474 state = .Done;
465 },475 },
476 else => return result,
466 }477 }
467 },478 },
468 .GotGt => {479 .GotGt => {
...@@ -471,9 +482,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -471,9 +482,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
471 result.id = .Shr;482 result.id = .Shr;
472 state = .Done;483 state = .Done;
473 },484 },
474 else => {485 '=' => {
475 return result;486 result.id = .Gte;
487 state = .Done;
476 },488 },
489 else => return result,
477 }490 }
478 },491 },
479 .GotPipe => {492 .GotPipe => {
...@@ -482,9 +495,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -482,9 +495,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
482 result.id = .Or;495 result.id = .Or;
483 state = .Done;496 state = .Done;
484 },497 },
485 else => {498 else => return result,
486 return result;
487 },
488 }499 }
489 },500 },
490 .GotAmpersand => {501 .GotAmpersand => {
...@@ -493,9 +504,25 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -493,9 +504,25 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
493 result.id = .And;504 result.id = .And;
494 state = .Done;505 state = .Done;
495 },506 },
496 else => {507 else => return result,
497 return result;508 }
509 },
510 .GotBang => {
511 switch (c) {
512 '=' => {
513 result.id = .Ne;
514 state = .Done;
515 },
516 else => return result,
517 }
518 },
519 .GotEq => {
520 switch (c) {
521 '=' => {
522 result.id = .Eq;
523 state = .Done;
498 },524 },
525 else => return result,
499 }526 }
500 },527 },
501 .Float => {528 .Float => {
...@@ -802,57 +829,101 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -802,57 +829,101 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
802 unreachable;829 unreachable;
803}830}
804831
832
833fn expectTokens(tl: *TokenList, src: [*:0]const u8, expected: []CToken) void {
834 tokenizeCMacro(tl, src) catch unreachable;
835 var it = tl.iterator(0);
836 for (expected) |t| {
837 var tok = it.next().?;
838 std.testing.expectEqual(t.id, tok.id);
839 if (t.bytes.len > 0) {
840 //std.debug.warn(" {} = {}\n", .{tok.bytes, t.bytes});
841 std.testing.expectEqualSlices(u8, tok.bytes, t.bytes);
842 }
843 if (t.num_lit_suffix != .None) {
844 std.testing.expectEqual(t.num_lit_suffix, tok.num_lit_suffix);
845 }
846 }
847 std.testing.expect(it.next() == null);
848 tl.shrink(0);
849}
850
851
805test "tokenize macro" {852test "tokenize macro" {
806 var tl = TokenList.init(std.heap.page_allocator);853 var tl = TokenList.init(std.heap.page_allocator);
807 defer tl.deinit();854 defer tl.deinit();
808855
809 const src = "TEST(0\n";856 expectTokens(&tl, "TEST(0\n", &[_]CToken{
810 try tokenizeCMacro(&tl, src);857 ctoken(.Identifier, "TEST"),
811 var it = tl.iterator(0);858 ctoken(.Fn, ""),
812 expect(it.next().?.id == .Identifier);859 ctoken(.LParen, ""),
813 expect(it.next().?.id == .Fn);860 ctoken(.NumLitInt, "0"),
814 expect(it.next().?.id == .LParen);861 ctoken(.Eof, ""),
815 expect(std.mem.eql(u8, it.next().?.bytes, "0"));862 });
816 expect(it.next().?.id == .Eof);
817 expect(it.next() == null);
818 tl.shrink(0);
819863
820 const src2 = "__FLT_MIN_10_EXP__ -37\n";864 expectTokens(&tl, "__FLT_MIN_10_EXP__ -37\n", &[_]CToken{
821 try tokenizeCMacro(&tl, src2);865 ctoken(.Identifier, "__FLT_MIN_10_EXP__"),
822 it = tl.iterator(0);866 ctoken(.Minus, ""),
823 expect(std.mem.eql(u8, it.next().?.bytes, "__FLT_MIN_10_EXP__"));867 ctoken(.NumLitInt, "37"),
824 expect(it.next().?.id == .Minus);868 ctoken(.Eof, ""),
825 expect(std.mem.eql(u8, it.next().?.bytes, "37"));869 });
826 expect(it.next().?.id == .Eof);
827 expect(it.next() == null);
828 tl.shrink(0);
829870
830 const src3 = "__llvm__ 1\n#define";871 expectTokens(&tl, "__llvm__ 1\n#define", &[_]CToken{
831 try tokenizeCMacro(&tl, src3);872 ctoken(.Identifier, "__llvm__"),
832 it = tl.iterator(0);873 ctoken(.NumLitInt, "1"),
833 expect(std.mem.eql(u8, it.next().?.bytes, "__llvm__"));874 ctoken(.Eof, ""),
834 expect(std.mem.eql(u8, it.next().?.bytes, "1"));
835 expect(it.next().?.id == .Eof);
836 expect(it.next() == null);
837 tl.shrink(0);
838875
839 const src4 = "TEST 2";876 });
840 try tokenizeCMacro(&tl, src4);
841 it = tl.iterator(0);
842 expect(it.next().?.id == .Identifier);
843 expect(std.mem.eql(u8, it.next().?.bytes, "2"));
844 expect(it.next().?.id == .Eof);
845 expect(it.next() == null);
846 tl.shrink(0);
847877
848 const src5 = "FOO 0ull";878 expectTokens(&tl, "TEST 2", &[_]CToken{
849 try tokenizeCMacro(&tl, src5);879 ctoken(.Identifier, "TEST"),
850 it = tl.iterator(0);880 ctoken(.NumLitInt, "2"),
851 expect(it.next().?.id == .Identifier);881 ctoken(.Eof, ""),
852 expect(std.mem.eql(u8, it.next().?.bytes, "0"));882
853 expect(it.next().?.id == .Eof);883 });
854 expect(it.next() == null);884
855 tl.shrink(0);885 expectTokens(&tl, "FOO 0ull", &[_]CToken{
886 ctoken(.Identifier, "FOO"),
887 cnumtoken(.LLU, "0"),
888 ctoken(.Eof, ""),
889
890 });
891
892}
893
894
895
896test "tokenize macro ops" {
897 var tl = TokenList.init(std.heap.page_allocator);
898 defer tl.deinit();
899
900 expectTokens(&tl, "ADD A + B", &[_]CToken{
901 ctoken(.Identifier, "ADD"),
902 ctoken(.Identifier, "A"),
903 ctoken(.Plus, ""),
904 ctoken(.Identifier, "B"),
905 ctoken(.Eof, ""),
906 });
907
908 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
909 ctoken(.Identifier, "ADD"),
910 ctoken(.LParen, ""),
911 ctoken(.Identifier, "A"),
912 ctoken(.RParen, ""),
913 ctoken(.Plus, ""),
914 ctoken(.Identifier, "B"),
915 ctoken(.Eof, ""),
916 });
917
918 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
919 ctoken(.Identifier, "ADD"),
920 ctoken(.LParen, ""),
921 ctoken(.Identifier, "A"),
922 ctoken(.RParen, ""),
923 ctoken(.Plus, ""),
924 ctoken(.Identifier, "B"),
925 ctoken(.Eof, ""),
926 });
856}927}
857928
858test "escape sequences" {929test "escape sequences" {