authorgravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-03-23 22:03:17-07:00
committergravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-03-25 12:25:43-07:00
log7f64f7c9259ba5f67adb386ba55473d4b2b74607
tree77eb3a4eb8bacc3d0b1b6ffa5fdd6e8902e9b8c7
parent1c33ea2c35e9260babedb116ad527256e0a4ef5e

Add rudimentary compile error test file support

This brings two quality-of-life improvements for folks working on compile error test cases: - test cases can be added/changed without re-building Zig - wrapping the source in a multi-line string literal is not necessary I decided to keep things as simple as possible for this initial implementation. The test "manifest" is a contiguous comment block at the end of the test file: 1. The first line is the test case name 2. The second line is a blank comment 2. The following lines are expected errors Here's an example: ```zig const U = union(enum(u2)) { A: u8, B: u8, C: u8, D: u8, E: u8, }; export fn entry() void { _ = U{ .E = 1 }; } // union with too small explicit unsigned tag type // // tmp.zig:1:22: error: specified integer tag type cannot represent every field // tmp.zig:1:22: note: type u2 cannot fit values in range 0...4 ``` The mode of the test (obj/exe/test), as well as the target (stage1/stage2) is determined based on the directory containing the test. We'll probably eventually want to support embedding this information in the test files themselves, similar to the arocc test runner, but that enhancement can be tackled later.

2 files changed, 135 insertions(+), 0 deletions(-)

src/test.zig+99
......@@ -20,6 +20,7 @@ const assert = std.debug.assert;
2020
2121const zig_h = link.File.C.zig_h;
2222
23var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){};
2324const hr = "=" ** 80;
2425
2526test {
......@@ -594,6 +595,104 @@ pub const TestContext = struct {
594595 case.compiles(fixed_src);
595596 }
596597
598 /// Adds a compile-error test for each file in the provided directory, using the
599 /// selected backend and output mode. If `one_test_case_per_file` is true, a new
600 /// test case is created for each file. Otherwise, a single test case is used for
601 /// all tests.
602 ///
603 /// Each file should include a test manifest as a contiguous block of comments at
604 /// the end of the file. The first line should be the test case name, followed by
605 /// a blank line, then one expected errors on each line in the form
606 /// `:line:column: error: message`
607 pub fn addErrorCasesFromDir(
608 ctx: *TestContext,
609 name: []const u8,
610 dir: std.fs.Dir,
611 backend: Backend,
612 output_mode: std.builtin.OutputMode,
613 is_test: bool,
614 one_test_case_per_file: bool,
615 ) !void {
616 if (skip_compile_errors) return;
617
618 const gpa = general_purpose_allocator.allocator();
619 var case: ?*Case = null;
620
621 var it = dir.iterate();
622 while (try it.next()) |entry| {
623 if (entry.kind != .File) continue;
624
625 var contents = try dir.readFileAlloc(gpa, entry.name, std.math.maxInt(u32));
626 defer gpa.free(contents);
627
628 // The manifest is the last contiguous block of comments in the file
629 // We scan for the beginning by searching backward for the first non-empty line that does not start with "//"
630 var manifest_start: ?usize = null;
631 var manifest_end: usize = contents.len;
632 if (contents.len > 0) {
633 var cursor: usize = contents.len - 1;
634 while (true) {
635 // Move to beginning of line
636 while (cursor > 0 and contents[cursor - 1] != '\n') cursor -= 1;
637
638 // Check if line is non-empty and does not start with "//"
639 if (cursor + 1 < contents.len and contents[cursor + 1] != '\n' and contents[cursor + 1] != '\r') {
640 if (std.mem.startsWith(u8, contents[cursor..], "//")) {
641 manifest_start = cursor;
642 } else {
643 break;
644 }
645 } else manifest_end = cursor;
646
647 // Move to previous line
648 if (cursor != 0) cursor -= 1 else break;
649 }
650 }
651
652 var errors = std.ArrayList([]const u8).init(gpa);
653 defer errors.deinit();
654
655 if (manifest_start) |start| {
656 // Due to the above processing, we know that this is a contiguous block of comments
657 var manifest_it = std.mem.tokenize(u8, contents[start..manifest_end], "\r\n");
658
659 // First line is the test case name
660 const first_line = manifest_it.next() orelse return error.InvalidFile;
661 const case_name = try std.mem.concat(gpa, u8, &.{ name, ": ", std.mem.trim(u8, first_line[2..], " \t") });
662
663 // If the second line is present, it should be blank
664 if (manifest_it.next()) |second_line| {
665 if (std.mem.trim(u8, second_line[2..], " \t").len != 0) return error.InvalidFile;
666 }
667
668 // All following lines are expected error messages
669 while (manifest_it.next()) |line| try errors.append(try gpa.dupe(u8, std.mem.trim(u8, line[2..], " \t")));
670
671 // The entire file contents is the source, including the manifest
672 const src = try gpa.dupeZ(u8, contents);
673
674 // Create a new test case, if necessary
675 case = if (one_test_case_per_file or case == null) blk: {
676 ctx.cases.append(TestContext.Case{
677 .name = if (one_test_case_per_file) case_name else name,
678 .target = .{},
679 .backend = backend,
680 .updates = std.ArrayList(TestContext.Update).init(ctx.cases.allocator),
681 .is_test = is_test,
682 .output_mode = output_mode,
683 .files = std.ArrayList(TestContext.File).init(ctx.cases.allocator),
684 }) catch @panic("out of memory");
685 break :blk &ctx.cases.items[ctx.cases.items.len - 1];
686 } else case.?;
687
688 // Add our update + expected errors
689 case.?.addError(src, errors.items);
690 } else {
691 return error.InvalidFile; // Manifests are currently mandatory
692 }
693 }
694 }
695
597696 fn init() TestContext {
598697 const allocator = std.heap.page_allocator;
599698 return .{ .cases = std.ArrayList(Case).init(allocator) };
test/compile_errors.zig+36
......@@ -3,6 +3,42 @@ const builtin = @import("builtin");
33const TestContext = @import("../src/test.zig").TestContext;
44
55pub fn addCases(ctx: *TestContext) !void {
6 var parent_dir = try std.fs.cwd().openDir(std.fs.path.dirname(@src().file).?, .{ .no_follow = true });
7 defer parent_dir.close();
8
9 var compile_errors_dir = try parent_dir.openDir("compile_errors", .{ .no_follow = true });
10 defer compile_errors_dir.close();
11
12 {
13 var stage2_dir = try compile_errors_dir.openDir("stage2", .{ .iterate = true, .no_follow = true });
14 defer stage2_dir.close();
15
16 const one_test_case_per_file = false;
17 try ctx.addErrorCasesFromDir("stage2 compile errors", stage2_dir, .stage2, .Obj, false, one_test_case_per_file);
18 }
19
20 {
21 var stage1_dir = try compile_errors_dir.openDir("stage1", .{ .no_follow = true });
22 defer stage1_dir.close();
23 {
24 const one_test_case_per_file = true;
25
26 var obj_dir = try stage1_dir.openDir("obj", .{ .iterate = true, .no_follow = true });
27 defer obj_dir.close();
28
29 try ctx.addErrorCasesFromDir("stage1", obj_dir, .stage1, .Obj, false, one_test_case_per_file);
30
31 var exe_dir = try stage1_dir.openDir("exe", .{ .iterate = true, .no_follow = true });
32 defer exe_dir.close();
33
34 try ctx.addErrorCasesFromDir("stage1", exe_dir, .stage1, .Exe, false, one_test_case_per_file);
35
36 var test_dir = try stage1_dir.openDir("test", .{ .iterate = true, .no_follow = true });
37 defer test_dir.close();
38
39 try ctx.addErrorCasesFromDir("stage1", test_dir, .stage1, .Exe, true, one_test_case_per_file);
40 }
41 }
642 {
743 var case = ctx.obj("stage2 compile errors", .{});
844