authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-03-31 15:06:44-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-03-31 15:10:31-07:00
log243afdcdf57d74a184784551aebe58062e5afc03
tree7e637a4c50b8abf6beb1354c346fc08055c5f0e7
parentdf1ba38a88a255286af4d939d427f8a4ee667485

test harness improvements

* `-Dskip-compile-errors` is removed; `-Dskip-stage1` is added. * Use `std.testing.allocator` instead of a new instance of GPA. - Fix the memory leaks this revealed. * Show the file name when it is not parsed correctly such as when the manifest is missing. - Better error messages when test files are not parsed correctly. * Ignore unknown files such as swap files. * Move logic from declarative file to the test harness implementation. * Move stage1 tests to stage2 tests where appropriate.

11 files changed, 184 insertions(+), 152 deletions(-)

build.zig+2-2
......@@ -54,7 +54,7 @@ pub fn build(b: *Builder) !void {
5454 const skip_release_safe = b.option(bool, "skip-release-safe", "Main test suite skips release-safe builds") orelse skip_release;
5555 const skip_non_native = b.option(bool, "skip-non-native", "Main test suite skips non-native builds") orelse false;
5656 const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false;
57 const skip_compile_errors = b.option(bool, "skip-compile-errors", "Main test suite skips compile error tests") orelse false;
57 const skip_stage1 = b.option(bool, "skip-stage1", "Main test suite skips stage1 compile error tests") orelse false;
5858 const skip_run_translated_c = b.option(bool, "skip-run-translated-c", "Main test suite skips run-translated-c tests") orelse false;
5959 const skip_stage2_tests = b.option(bool, "skip-stage2-tests", "Main test suite skips self-hosted compiler tests") orelse false;
6060 const skip_install_lib_files = b.option(bool, "skip-install-lib-files", "Do not copy lib/ files to installation prefix") orelse false;
......@@ -386,7 +386,7 @@ pub fn build(b: *Builder) !void {
386386 test_stage2_options.addOption(bool, "enable_logging", enable_logging);
387387 test_stage2_options.addOption(bool, "enable_link_snapshots", enable_link_snapshots);
388388 test_stage2_options.addOption(bool, "skip_non_native", skip_non_native);
389 test_stage2_options.addOption(bool, "skip_compile_errors", skip_compile_errors);
389 test_stage2_options.addOption(bool, "skip_stage1", skip_stage1);
390390 test_stage2_options.addOption(bool, "is_stage1", is_stage1);
391391 test_stage2_options.addOption(bool, "omit_stage2", omit_stage2);
392392 test_stage2_options.addOption(bool, "have_llvm", enable_llvm);
src/test.zig+114-45
......@@ -12,7 +12,7 @@ const enable_wasmtime: bool = build_options.enable_wasmtime;
1212const enable_darling: bool = build_options.enable_darling;
1313const enable_rosetta: bool = build_options.enable_rosetta;
1414const glibc_runtimes_dir: ?[]const u8 = build_options.glibc_runtimes_dir;
15const skip_compile_errors = build_options.skip_compile_errors;
15const skip_stage1 = build_options.skip_stage1;
1616const ThreadPool = @import("ThreadPool.zig");
1717const CrossTarget = std.zig.CrossTarget;
1818const print = std.debug.print;
......@@ -20,7 +20,6 @@ const assert = std.debug.assert;
2020
2121const zig_h = link.File.C.zig_h;
2222
23var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){};
2423const hr = "=" ** 80;
2524
2625test {
......@@ -28,9 +27,50 @@ test {
2827 @import("stage1.zig").os_init();
2928 }
3029
31 var ctx = TestContext.init();
30 var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
31 defer arena_allocator.deinit();
32 const arena = arena_allocator.allocator();
33
34 var ctx = TestContext.init(std.testing.allocator, arena);
3235 defer ctx.deinit();
3336
37 const compile_errors_dir_path = try std.fs.path.join(arena, &.{
38 std.fs.path.dirname(@src().file).?, "..", "test", "compile_errors",
39 });
40
41 var compile_errors_dir = try std.fs.cwd().openDir(compile_errors_dir_path, .{});
42 defer compile_errors_dir.close();
43
44 {
45 var stage2_dir = try compile_errors_dir.openDir("stage2", .{ .iterate = true });
46 defer stage2_dir.close();
47
48 // TODO make this incremental once the bug is solved that it triggers
49 ctx.addErrorCasesFromDir("stage2", stage2_dir, .stage2, .Obj, false, .independent);
50 }
51
52 if (!skip_stage1) {
53 var stage1_dir = try compile_errors_dir.openDir("stage1", .{});
54 defer stage1_dir.close();
55
56 const Config = struct {
57 name: []const u8,
58 is_test: bool,
59 output_mode: std.builtin.OutputMode,
60 };
61
62 for ([_]Config{
63 .{ .name = "obj", .is_test = false, .output_mode = .Obj },
64 .{ .name = "exe", .is_test = false, .output_mode = .Exe },
65 .{ .name = "test", .is_test = true, .output_mode = .Exe },
66 }) |config| {
67 var dir = try stage1_dir.openDir(config.name, .{ .iterate = true });
68 defer dir.close();
69
70 ctx.addErrorCasesFromDir("stage1", dir, .stage1, config.output_mode, config.is_test, .independent);
71 }
72 }
73
3474 try @import("test_cases").addCases(&ctx);
3575
3676 try ctx.run();
......@@ -114,6 +154,7 @@ const ErrorMsg = union(enum) {
114154};
115155
116156pub const TestContext = struct {
157 arena: Allocator,
117158 cases: std.ArrayList(Case),
118159
119160 pub const Update = struct {
......@@ -316,7 +357,7 @@ pub const TestContext = struct {
316357 .target = target,
317358 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
318359 .output_mode = .Exe,
319 .files = std.ArrayList(File).init(ctx.cases.allocator),
360 .files = std.ArrayList(File).init(ctx.arena),
320361 }) catch @panic("out of memory");
321362 return &ctx.cases.items[ctx.cases.items.len - 1];
322363 }
......@@ -327,7 +368,7 @@ pub const TestContext = struct {
327368 }
328369
329370 pub fn exeFromCompiledC(ctx: *TestContext, name: []const u8, target: CrossTarget) *Case {
330 const prefixed_name = std.fmt.allocPrint(ctx.cases.allocator, "CBE: {s}", .{name}) catch
371 const prefixed_name = std.fmt.allocPrint(ctx.arena, "CBE: {s}", .{name}) catch
331372 @panic("out of memory");
332373 ctx.cases.append(Case{
333374 .name = prefixed_name,
......@@ -335,7 +376,7 @@ pub const TestContext = struct {
335376 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
336377 .output_mode = .Exe,
337378 .object_format = .c,
338 .files = std.ArrayList(File).init(ctx.cases.allocator),
379 .files = std.ArrayList(File).init(ctx.arena),
339380 }) catch @panic("out of memory");
340381 return &ctx.cases.items[ctx.cases.items.len - 1];
341382 }
......@@ -348,7 +389,7 @@ pub const TestContext = struct {
348389 .target = target,
349390 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
350391 .output_mode = .Exe,
351 .files = std.ArrayList(File).init(ctx.cases.allocator),
392 .files = std.ArrayList(File).init(ctx.arena),
352393 .backend = .llvm,
353394 .link_libc = true,
354395 }) catch @panic("out of memory");
......@@ -365,7 +406,7 @@ pub const TestContext = struct {
365406 .target = target,
366407 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
367408 .output_mode = .Obj,
368 .files = std.ArrayList(File).init(ctx.cases.allocator),
409 .files = std.ArrayList(File).init(ctx.arena),
369410 }) catch @panic("out of memory");
370411 return &ctx.cases.items[ctx.cases.items.len - 1];
371412 }
......@@ -381,7 +422,7 @@ pub const TestContext = struct {
381422 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
382423 .output_mode = .Exe,
383424 .is_test = true,
384 .files = std.ArrayList(File).init(ctx.cases.allocator),
425 .files = std.ArrayList(File).init(ctx.arena),
385426 }) catch @panic("out of memory");
386427 return &ctx.cases.items[ctx.cases.items.len - 1];
387428 }
......@@ -404,7 +445,7 @@ pub const TestContext = struct {
404445 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
405446 .output_mode = .Obj,
406447 .object_format = .c,
407 .files = std.ArrayList(File).init(ctx.cases.allocator),
448 .files = std.ArrayList(File).init(ctx.arena),
408449 }) catch @panic("out of memory");
409450 return &ctx.cases.items[ctx.cases.items.len - 1];
410451 }
......@@ -423,7 +464,7 @@ pub const TestContext = struct {
423464 src: [:0]const u8,
424465 expected_errors: []const []const u8,
425466 ) void {
426 if (skip_compile_errors) return;
467 if (skip_stage1) return;
427468
428469 const case = ctx.addObj(name, .{});
429470 case.backend = .stage1;
......@@ -436,7 +477,7 @@ pub const TestContext = struct {
436477 src: [:0]const u8,
437478 expected_errors: []const []const u8,
438479 ) void {
439 if (skip_compile_errors) return;
480 if (skip_stage1) return;
440481
441482 const case = ctx.addTest(name, .{});
442483 case.backend = .stage1;
......@@ -449,7 +490,7 @@ pub const TestContext = struct {
449490 src: [:0]const u8,
450491 expected_errors: []const []const u8,
451492 ) void {
452 if (skip_compile_errors) return;
493 if (skip_stage1) return;
453494
454495 const case = ctx.addExe(name, .{});
455496 case.backend = .stage1;
......@@ -612,6 +653,8 @@ pub const TestContext = struct {
612653 case.compiles(fixed_src);
613654 }
614655
656 const Strategy = enum { incremental, independent };
657
615658 /// Adds a compile-error test for each file in the provided directory, using the
616659 /// selected backend and output mode. If `one_test_case_per_file` is true, a new
617660 /// test case is created for each file. Otherwise, a single test case is used for
......@@ -628,33 +671,58 @@ pub const TestContext = struct {
628671 backend: Backend,
629672 output_mode: std.builtin.OutputMode,
630673 is_test: bool,
631 one_test_case_per_file: bool,
632 ) !void {
633 if (skip_compile_errors) return;
674 strategy: Strategy,
675 ) void {
676 var current_file: []const u8 = "none";
677 addErrorCasesFromDirInner(ctx, name, dir, backend, output_mode, is_test, strategy, &current_file) catch |err| {
678 std.debug.panic("test harness failed to process file '{s}': {s}\n", .{
679 current_file, @errorName(err),
680 });
681 };
682 }
634683
635 const gpa = general_purpose_allocator.allocator();
684 fn addErrorCasesFromDirInner(
685 ctx: *TestContext,
686 name: []const u8,
687 dir: std.fs.Dir,
688 backend: Backend,
689 output_mode: std.builtin.OutputMode,
690 is_test: bool,
691 strategy: Strategy,
692 /// This is kept up to date with the currently being processed file so
693 /// that if any errors occur the caller knows it happened during this file.
694 current_file: *[]const u8,
695 ) !void {
636696 var opt_case: ?*Case = null;
637697
638698 var it = dir.iterate();
639699 while (try it.next()) |entry| {
640700 if (entry.kind != .File) continue;
641701
642 var contents = try dir.readFileAlloc(gpa, entry.name, std.math.maxInt(u32));
643 defer gpa.free(contents);
702 // Ignore stuff such as .swp files
703 switch (Compilation.classifyFileExt(entry.name)) {
704 .unknown => continue,
705 else => {},
706 }
707
708 current_file.* = try ctx.arena.dupe(u8, entry.name);
709
710 const max_file_size = 10 * 1024 * 1024;
711 const src = try dir.readFileAllocOptions(ctx.arena, entry.name, max_file_size, null, 1, 0);
644712
645713 // The manifest is the last contiguous block of comments in the file
646714 // We scan for the beginning by searching backward for the first non-empty line that does not start with "//"
647715 var manifest_start: ?usize = null;
648 var manifest_end: usize = contents.len;
649 if (contents.len > 0) {
650 var cursor: usize = contents.len - 1;
716 var manifest_end: usize = src.len;
717 if (src.len > 0) {
718 var cursor: usize = src.len - 1;
651719 while (true) {
652720 // Move to beginning of line
653 while (cursor > 0 and contents[cursor - 1] != '\n') cursor -= 1;
721 while (cursor > 0 and src[cursor - 1] != '\n') cursor -= 1;
654722
655723 // Check if line is non-empty and does not start with "//"
656 if (cursor + 1 < contents.len and contents[cursor + 1] != '\n' and contents[cursor + 1] != '\r') {
657 if (std.mem.startsWith(u8, contents[cursor..], "//")) {
724 if (cursor + 1 < src.len and src[cursor + 1] != '\n' and src[cursor + 1] != '\r') {
725 if (std.mem.startsWith(u8, src[cursor..], "//")) {
658726 manifest_start = cursor;
659727 } else {
660728 break;
......@@ -666,27 +734,23 @@ pub const TestContext = struct {
666734 }
667735 }
668736
669 var errors = std.ArrayList([]const u8).init(gpa);
670 defer errors.deinit();
737 var errors = std.ArrayList([]const u8).init(ctx.arena);
671738
672739 if (manifest_start) |start| {
673740 // Due to the above processing, we know that this is a contiguous block of comments
674 var manifest_it = std.mem.tokenize(u8, contents[start..manifest_end], "\r\n");
741 var manifest_it = std.mem.tokenize(u8, src[start..manifest_end], "\r\n");
675742
676743 // First line is the test case name
677 const first_line = manifest_it.next() orelse return error.InvalidFile;
678 const case_name = try std.mem.concat(gpa, u8, &.{ name, ": ", std.mem.trim(u8, first_line[2..], " \t") });
744 const first_line = manifest_it.next() orelse return error.MissingTestCaseName;
745 const case_name = try std.mem.concat(ctx.arena, u8, &.{ name, ": ", std.mem.trim(u8, first_line[2..], " \t") });
679746
680747 // If the second line is present, it should be blank
681748 if (manifest_it.next()) |second_line| {
682 if (std.mem.trim(u8, second_line[2..], " \t").len != 0) return error.InvalidFile;
749 if (std.mem.trim(u8, second_line[2..], " \t").len != 0) return error.SecondLineNotBlank;
683750 }
684751
685752 // All following lines are expected error messages
686 while (manifest_it.next()) |line| try errors.append(try gpa.dupe(u8, std.mem.trim(u8, line[2..], " \t")));
687
688 // The entire file contents is the source, including the manifest
689 const src = try gpa.dupeZ(u8, contents);
753 while (manifest_it.next()) |line| try errors.append(try ctx.arena.dupe(u8, std.mem.trim(u8, line[2..], " \t")));
690754
691755 const case = opt_case orelse case: {
692756 ctx.cases.append(TestContext.Case{
......@@ -702,22 +766,27 @@ pub const TestContext = struct {
702766 opt_case = case;
703767 break :case case;
704768 };
705 if (one_test_case_per_file) {
706 case.name = case_name;
707 case.addError(src, errors.items);
708 opt_case = null;
709 } else {
710 case.addErrorNamed(case_name, src, errors.items);
769 switch (strategy) {
770 .independent => {
771 case.name = case_name;
772 case.addError(src, errors.items);
773 opt_case = null;
774 },
775 .incremental => {
776 case.addErrorNamed(case_name, src, errors.items);
777 },
711778 }
712779 } else {
713 return error.InvalidFile; // Manifests are currently mandatory
780 return error.MissingManifest;
714781 }
715782 }
716783 }
717784
718 fn init() TestContext {
719 const allocator = std.heap.page_allocator;
720 return .{ .cases = std.ArrayList(Case).init(allocator) };
785 fn init(gpa: Allocator, arena: Allocator) TestContext {
786 return .{
787 .cases = std.ArrayList(Case).init(gpa),
788 .arena = arena,
789 };
721790 }
722791
723792 fn deinit(self: *TestContext) void {
test/compile_errors.zig-38
......@@ -3,44 +3,6 @@ const builtin = @import("builtin");
33const TestContext = @import("../src/test.zig").TestContext;
44
55pub fn addCases(ctx: *TestContext) !void {
6 var parent_dir = try std.fs.cwd().openDir(std.fs.path.dirname(@src().file).?, .{ .no_follow = true });
7 defer parent_dir.close();
8
9 var compile_errors_dir = try parent_dir.openDir("compile_errors", .{ .no_follow = true });
10 defer compile_errors_dir.close();
11
12 {
13 var stage2_dir = try compile_errors_dir.openDir("stage2", .{ .iterate = true, .no_follow = true });
14 defer stage2_dir.close();
15
16 // TODO make this false once the bug is solved that it triggers
17 const one_test_case_per_file = true;
18 try ctx.addErrorCasesFromDir("stage2", stage2_dir, .stage2, .Obj, false, one_test_case_per_file);
19 }
20
21 {
22 var stage1_dir = try compile_errors_dir.openDir("stage1", .{ .no_follow = true });
23 defer stage1_dir.close();
24 {
25 const one_test_case_per_file = true;
26
27 var obj_dir = try stage1_dir.openDir("obj", .{ .iterate = true, .no_follow = true });
28 defer obj_dir.close();
29
30 try ctx.addErrorCasesFromDir("stage1", obj_dir, .stage1, .Obj, false, one_test_case_per_file);
31
32 var exe_dir = try stage1_dir.openDir("exe", .{ .iterate = true, .no_follow = true });
33 defer exe_dir.close();
34
35 try ctx.addErrorCasesFromDir("stage1", exe_dir, .stage1, .Exe, false, one_test_case_per_file);
36
37 var test_dir = try stage1_dir.openDir("test", .{ .iterate = true, .no_follow = true });
38 defer test_dir.close();
39
40 try ctx.addErrorCasesFromDir("stage1", test_dir, .stage1, .Exe, true, one_test_case_per_file);
41 }
42 }
43
446 {
457 const case = ctx.obj("callconv(.Interrupt) on unsupported platform", .{
468 .cpu_arch = .aarch64,
test/compile_errors/stage1/exe/std.fmt_error_for_unused_arguments.zig deleted-7
......@@ -1,7 +0,0 @@
1pub fn main() !void {
2 @import("std").debug.print("{d} {d} {d} {d} {d}", .{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15});
3}
4
5// std.fmt error for unused arguments
6//
7// ?:?:?: error: 10 unused arguments in '{d} {d} {d} {d} {d}'
test/compile_errors/stage1/obj/constant_inside_comptime_function_has_compile_error.zig deleted-19
......@@ -1,19 +0,0 @@
1const ContextAllocator = MemoryPool(usize);
2
3pub fn MemoryPool(comptime T: type) type {
4 const free_list_t = @compileError("aoeu",);
5
6 return struct {
7 free_list: free_list_t,
8 };
9}
10
11export fn entry() void {
12 var allocator: ContextAllocator = undefined;
13}
14
15// constant inside comptime function has compile error
16//
17// tmp.zig:4:5: error: unreachable code
18// tmp.zig:4:25: note: control flow is diverted here
19// tmp.zig:12:9: error: unused local variable
test/compile_errors/stage1/obj/std.fmt_error_for_unused_arguments.zig created+7
......@@ -0,0 +1,7 @@
1export fn entry() void {
2 @import("std").debug.print("{d} {d} {d} {d} {d}", .{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15});
3}
4
5// std.fmt error for unused arguments
6//
7// ?:?:?: error: 10 unused arguments in '{d} {d} {d} {d} {d}'
test/compile_errors/stage1/obj/unused_variable_error_on_errdefer.zig created+11
......@@ -0,0 +1,11 @@
1fn foo() !void {
2 errdefer |a| unreachable;
3 return error.A;
4}
5export fn entry() void {
6 foo() catch unreachable;
7}
8
9// unused variable error on errdefer
10//
11// tmp.zig:2:15: error: unused variable: 'a'
test/compile_errors/stage1/test/duplicate-unused_labels.zig deleted-30
......@@ -1,30 +0,0 @@
1comptime {
2 blk: { blk: while (false) {} }
3}
4comptime {
5 blk: while (false) { blk: for (@as([0]void, undefined)) |_| {} }
6}
7comptime {
8 blk: for (@as([0]void, undefined)) |_| { blk: {} }
9}
10comptime {
11 blk: {}
12}
13comptime {
14 blk: while(false) {}
15}
16comptime {
17 blk: for(@as([0]void, undefined)) |_| {}
18}
19
20// duplicate/unused labels
21//
22// tmp.zig:2:12: error: redefinition of label 'blk'
23// tmp.zig:2:5: note: previous definition here
24// tmp.zig:5:26: error: redefinition of label 'blk'
25// tmp.zig:5:5: note: previous definition here
26// tmp.zig:8:46: error: redefinition of label 'blk'
27// tmp.zig:8:5: note: previous definition here
28// tmp.zig:11:5: error: unused block label
29// tmp.zig:14:5: error: unused while loop label
30// tmp.zig:17:5: error: unused for loop label
test/compile_errors/stage1/test/unused_variable_error_on_errdefer.zig deleted-11
......@@ -1,11 +0,0 @@
1fn foo() !void {
2 errdefer |a| unreachable;
3 return error.A;
4}
5export fn entry() void {
6 foo() catch unreachable;
7}
8
9// unused variable error on errdefer
10//
11// tmp.zig:2:15: error: unused variable: 'a'
test/compile_errors/stage2/constant_inside_comptime_function_has_compile_error.zig created+20
......@@ -0,0 +1,20 @@
1const ContextAllocator = MemoryPool(usize);
2
3pub fn MemoryPool(comptime T: type) type {
4 const free_list_t = @compileError("aoeu",);
5 _ = T;
6
7 return struct {
8 free_list: free_list_t,
9 };
10}
11
12export fn entry() void {
13 var allocator: ContextAllocator = undefined;
14 _ = allocator;
15}
16
17// constant inside comptime function has compile error
18//
19// :4:5: error: unreachable code
20// :4:25: note: control flow is diverted here
test/compile_errors/stage2/duplicate-unused_labels.zig created+30
......@@ -0,0 +1,30 @@
1comptime {
2 blk: { blk: while (false) {} }
3}
4comptime {
5 blk: while (false) { blk: for (@as([0]void, undefined)) |_| {} }
6}
7comptime {
8 blk: for (@as([0]void, undefined)) |_| { blk: {} }
9}
10comptime {
11 blk: {}
12}
13comptime {
14 blk: while(false) {}
15}
16comptime {
17 blk: for(@as([0]void, undefined)) |_| {}
18}
19
20// duplicate/unused labels
21//
22// :2:12: error: redefinition of label 'blk'
23// :2:5: note: previous definition here
24// :5:26: error: redefinition of label 'blk'
25// :5:5: note: previous definition here
26// :8:46: error: redefinition of label 'blk'
27// :8:5: note: previous definition here
28// :11:5: error: unused block label
29// :14:5: error: unused while loop label
30// :17:5: error: unused for loop label