authorgravatar for frmdstryr@protonmail.comfrmdstryr <frmdstryr@protonmail.com> 2019-12-29 10:41:26-05:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-29 19:50:58+02:00
log6df9e9fe47ad79f362a57279751374b6966782ed
treef6cb4e698b2fdc43d05ae17deea75a3a58bfe0fa
parentfcc82a219ab20eb4b5e26a6e46219e564ee87810
signature Commit is signed but in an unrecognized format.

Cleanup c_tokenizer.zig tests

1 files changed, 143 insertions(+), 72 deletions(-)

src-self-hosted/c_tokenizer.zig+143-72
......@@ -24,21 +24,26 @@ pub const CToken = struct {
2424 RParen,
2525 Eof,
2626 Dot,
27 Asterisk,
28 Ampersand,
29 And,
30 Or,
31 Bang,
32 Tilde,
33 Shl,
34 Shr,
35 Lt,
36 Gt,
37 Increment,
38 Decrement,
27 Asterisk, // *
28 Ampersand, // &
29 And, // &&
30 Assign, // =
31 Or, // ||
32 Bang, // !
33 Tilde, // ~
34 Shl, // <<
35 Shr, // >>
36 Lt, // <
37 Lte, // <=
38 Gt, // >
39 Gte, // >=
40 Eq, // ==
41 Ne, // !=
42 Increment, // ++
43 Decrement, // --
3944 Comma,
4045 Fn,
41 Arrow,
46 Arrow, // ->
4247 LBrace,
4348 RBrace,
4449 Pipe,
......@@ -239,6 +244,8 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
239244 GotMinus,
240245 GotAmpersand,
241246 GotPipe,
247 GotBang,
248 GotEq,
242249 CharLit,
243250 OpenComment,
244251 Comment,
......@@ -298,6 +305,8 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
298305 .GotPlus,
299306 .GotAmpersand,
300307 .GotPipe,
308 .GotBang,
309 .GotEq,
301310 => {
302311 return result;
303312 },
......@@ -387,12 +396,16 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
387396 },
388397 '!' => {
389398 result.id = .Bang;
390 state = .Done;
399 state = .GotBang;
391400 },
392401 '~' => {
393402 result.id = .Tilde;
394403 state = .Done;
395404 },
405 '=' => {
406 result.id = .Assign;
407 state = .GotEq;
408 },
396409 ',' => {
397410 result.id = .Comma;
398411 state = .Done;
......@@ -438,9 +451,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
438451 result.id = .Decrement;
439452 state = .Done;
440453 },
441 else => {
442 return result;
443 },
454 else => return result,
444455 }
445456 },
446457 .GotPlus => {
......@@ -449,9 +460,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
449460 result.id = .Increment;
450461 state = .Done;
451462 },
452 else => {
453 return result;
454 },
463 else => return result,
455464 }
456465 },
457466 .GotLt => {
......@@ -460,9 +469,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
460469 result.id = .Shl;
461470 state = .Done;
462471 },
463 else => {
464 return result;
472 '=' => {
473 result.id = .Lte;
474 state = .Done;
465475 },
476 else => return result,
466477 }
467478 },
468479 .GotGt => {
......@@ -471,9 +482,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
471482 result.id = .Shr;
472483 state = .Done;
473484 },
474 else => {
475 return result;
485 '=' => {
486 result.id = .Gte;
487 state = .Done;
476488 },
489 else => return result,
477490 }
478491 },
479492 .GotPipe => {
......@@ -482,9 +495,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
482495 result.id = .Or;
483496 state = .Done;
484497 },
485 else => {
486 return result;
487 },
498 else => return result,
488499 }
489500 },
490501 .GotAmpersand => {
......@@ -493,9 +504,25 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
493504 result.id = .And;
494505 state = .Done;
495506 },
496 else => {
497 return result;
507 else => return result,
508 }
509 },
510 .GotBang => {
511 switch (c) {
512 '=' => {
513 result.id = .Ne;
514 state = .Done;
515 },
516 else => return result,
517 }
518 },
519 .GotEq => {
520 switch (c) {
521 '=' => {
522 result.id = .Eq;
523 state = .Done;
498524 },
525 else => return result,
499526 }
500527 },
501528 .Float => {
......@@ -802,57 +829,101 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
802829 unreachable;
803830}
804831
832
833fn expectTokens(tl: *TokenList, src: [*:0]const u8, expected: []CToken) void {
834 tokenizeCMacro(tl, src) catch unreachable;
835 var it = tl.iterator(0);
836 for (expected) |t| {
837 var tok = it.next().?;
838 std.testing.expectEqual(t.id, tok.id);
839 if (t.bytes.len > 0) {
840 //std.debug.warn(" {} = {}\n", .{tok.bytes, t.bytes});
841 std.testing.expectEqualSlices(u8, tok.bytes, t.bytes);
842 }
843 if (t.num_lit_suffix != .None) {
844 std.testing.expectEqual(t.num_lit_suffix, tok.num_lit_suffix);
845 }
846 }
847 std.testing.expect(it.next() == null);
848 tl.shrink(0);
849}
850
851
805852test "tokenize macro" {
806853 var tl = TokenList.init(std.heap.page_allocator);
807854 defer tl.deinit();
808855
809 const src = "TEST(0\n";
810 try tokenizeCMacro(&tl, src);
811 var it = tl.iterator(0);
812 expect(it.next().?.id == .Identifier);
813 expect(it.next().?.id == .Fn);
814 expect(it.next().?.id == .LParen);
815 expect(std.mem.eql(u8, it.next().?.bytes, "0"));
816 expect(it.next().?.id == .Eof);
817 expect(it.next() == null);
818 tl.shrink(0);
856 expectTokens(&tl, "TEST(0\n", &[_]CToken{
857 ctoken(.Identifier, "TEST"),
858 ctoken(.Fn, ""),
859 ctoken(.LParen, ""),
860 ctoken(.NumLitInt, "0"),
861 ctoken(.Eof, ""),
862 });
819863
820 const src2 = "__FLT_MIN_10_EXP__ -37\n";
821 try tokenizeCMacro(&tl, src2);
822 it = tl.iterator(0);
823 expect(std.mem.eql(u8, it.next().?.bytes, "__FLT_MIN_10_EXP__"));
824 expect(it.next().?.id == .Minus);
825 expect(std.mem.eql(u8, it.next().?.bytes, "37"));
826 expect(it.next().?.id == .Eof);
827 expect(it.next() == null);
828 tl.shrink(0);
864 expectTokens(&tl, "__FLT_MIN_10_EXP__ -37\n", &[_]CToken{
865 ctoken(.Identifier, "__FLT_MIN_10_EXP__"),
866 ctoken(.Minus, ""),
867 ctoken(.NumLitInt, "37"),
868 ctoken(.Eof, ""),
869 });
829870
830 const src3 = "__llvm__ 1\n#define";
831 try tokenizeCMacro(&tl, src3);
832 it = tl.iterator(0);
833 expect(std.mem.eql(u8, it.next().?.bytes, "__llvm__"));
834 expect(std.mem.eql(u8, it.next().?.bytes, "1"));
835 expect(it.next().?.id == .Eof);
836 expect(it.next() == null);
837 tl.shrink(0);
871 expectTokens(&tl, "__llvm__ 1\n#define", &[_]CToken{
872 ctoken(.Identifier, "__llvm__"),
873 ctoken(.NumLitInt, "1"),
874 ctoken(.Eof, ""),
838875
839 const src4 = "TEST 2";
840 try tokenizeCMacro(&tl, src4);
841 it = tl.iterator(0);
842 expect(it.next().?.id == .Identifier);
843 expect(std.mem.eql(u8, it.next().?.bytes, "2"));
844 expect(it.next().?.id == .Eof);
845 expect(it.next() == null);
846 tl.shrink(0);
876 });
847877
848 const src5 = "FOO 0ull";
849 try tokenizeCMacro(&tl, src5);
850 it = tl.iterator(0);
851 expect(it.next().?.id == .Identifier);
852 expect(std.mem.eql(u8, it.next().?.bytes, "0"));
853 expect(it.next().?.id == .Eof);
854 expect(it.next() == null);
855 tl.shrink(0);
878 expectTokens(&tl, "TEST 2", &[_]CToken{
879 ctoken(.Identifier, "TEST"),
880 ctoken(.NumLitInt, "2"),
881 ctoken(.Eof, ""),
882
883 });
884
885 expectTokens(&tl, "FOO 0ull", &[_]CToken{
886 ctoken(.Identifier, "FOO"),
887 cnumtoken(.LLU, "0"),
888 ctoken(.Eof, ""),
889
890 });
891
892}
893
894
895
896test "tokenize macro ops" {
897 var tl = TokenList.init(std.heap.page_allocator);
898 defer tl.deinit();
899
900 expectTokens(&tl, "ADD A + B", &[_]CToken{
901 ctoken(.Identifier, "ADD"),
902 ctoken(.Identifier, "A"),
903 ctoken(.Plus, ""),
904 ctoken(.Identifier, "B"),
905 ctoken(.Eof, ""),
906 });
907
908 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
909 ctoken(.Identifier, "ADD"),
910 ctoken(.LParen, ""),
911 ctoken(.Identifier, "A"),
912 ctoken(.RParen, ""),
913 ctoken(.Plus, ""),
914 ctoken(.Identifier, "B"),
915 ctoken(.Eof, ""),
916 });
917
918 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
919 ctoken(.Identifier, "ADD"),
920 ctoken(.LParen, ""),
921 ctoken(.Identifier, "A"),
922 ctoken(.RParen, ""),
923 ctoken(.Plus, ""),
924 ctoken(.Identifier, "B"),
925 ctoken(.Eof, ""),
926 });
856927}
857928
858929test "escape sequences" {