authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-03-06 19:19:48-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-03-15 10:48:13-07:00
loge897637d8d25f5b6b118356d2355da7c9148d8cb
tree1fb34437dff2125933ed76b045cb76e409b2cbc9
parenta24af8e4005d00cf76755635eb6c661d9c260758

re-enable compare-output test cases

5 files changed, 180 insertions(+), 179 deletions(-)

build.zig+1-1
...@@ -444,7 +444,7 @@ pub fn build(b: *std.Build) !void {...@@ -444,7 +444,7 @@ pub fn build(b: *std.Build) !void {
444444
445 _ = enable_symlinks_windows;445 _ = enable_symlinks_windows;
446 _ = enable_macos_sdk;446 _ = enable_macos_sdk;
447 //test_step.dependOn(tests.addCompareOutputTests(b, test_filter, optimization_modes));447 test_step.dependOn(tests.addCompareOutputTests(b, test_filter, optimization_modes));
448 //test_step.dependOn(tests.addStandaloneTests(448 //test_step.dependOn(tests.addStandaloneTests(
449 // b,449 // b,
450 // test_filter,450 // test_filter,
lib/build_runner.zig+2-2
...@@ -887,6 +887,8 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi...@@ -887,6 +887,8 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi
887 \\ --verbose Print commands before executing them887 \\ --verbose Print commands before executing them
888 \\ --color [auto|off|on] Enable or disable colored error messages888 \\ --color [auto|off|on] Enable or disable colored error messages
889 \\ --prominent-compile-errors Output compile errors formatted for a human to read889 \\ --prominent-compile-errors Output compile errors formatted for a human to read
890 \\ -fsummary Print the build summary, even on success
891 \\ -fno-summary Omit the build summary, even on failure
890 \\ -j<N> Limit concurrent jobs (default is to use all CPU cores)892 \\ -j<N> Limit concurrent jobs (default is to use all CPU cores)
891 \\ --maxrss <bytes> Limit memory usage (default is to use available memory)893 \\ --maxrss <bytes> Limit memory usage (default is to use available memory)
892 \\894 \\
...@@ -920,8 +922,6 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi...@@ -920,8 +922,6 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi
920 \\Advanced Options:922 \\Advanced Options:
921 \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error923 \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error
922 \\ -fno-reference-trace Disable reference trace924 \\ -fno-reference-trace Disable reference trace
923 \\ -fsummary Print the build summary, even on success
924 \\ -fno-summary Omit the build summary, even on failure
925 \\ --build-file [file] Override path to build.zig925 \\ --build-file [file] Override path to build.zig
926 \\ --cache-dir [path] Override path to local Zig cache directory926 \\ --cache-dir [path] Override path to local Zig cache directory
927 \\ --global-cache-dir [path] Override path to global Zig cache directory927 \\ --global-cache-dir [path] Override path to global Zig cache directory
test/src/CompareOutput.zig created+176
...@@ -0,0 +1,176 @@
1//! This is the implementation of the test harness.
2//! For the actual test cases, see test/compare_output.zig.
3
4b: *std.Build,
5step: *std.Build.Step,
6test_index: usize,
7test_filter: ?[]const u8,
8optimize_modes: []const OptimizeMode,
9
10const Special = enum {
11 None,
12 Asm,
13 RuntimeSafety,
14};
15
16const TestCase = struct {
17 name: []const u8,
18 sources: ArrayList(SourceFile),
19 expected_output: []const u8,
20 link_libc: bool,
21 special: Special,
22 cli_args: []const []const u8,
23
24 const SourceFile = struct {
25 filename: []const u8,
26 source: []const u8,
27 };
28
29 pub fn addSourceFile(self: *TestCase, filename: []const u8, source: []const u8) void {
30 self.sources.append(SourceFile{
31 .filename = filename,
32 .source = source,
33 }) catch @panic("OOM");
34 }
35
36 pub fn setCommandLineArgs(self: *TestCase, args: []const []const u8) void {
37 self.cli_args = args;
38 }
39};
40
41pub fn createExtra(self: *CompareOutput, name: []const u8, source: []const u8, expected_output: []const u8, special: Special) TestCase {
42 var tc = TestCase{
43 .name = name,
44 .sources = ArrayList(TestCase.SourceFile).init(self.b.allocator),
45 .expected_output = expected_output,
46 .link_libc = false,
47 .special = special,
48 .cli_args = &[_][]const u8{},
49 };
50 const root_src_name = if (special == Special.Asm) "source.s" else "source.zig";
51 tc.addSourceFile(root_src_name, source);
52 return tc;
53}
54
55pub fn create(self: *CompareOutput, name: []const u8, source: []const u8, expected_output: []const u8) TestCase {
56 return createExtra(self, name, source, expected_output, Special.None);
57}
58
59pub fn addC(self: *CompareOutput, name: []const u8, source: []const u8, expected_output: []const u8) void {
60 var tc = self.create(name, source, expected_output);
61 tc.link_libc = true;
62 self.addCase(tc);
63}
64
65pub fn add(self: *CompareOutput, name: []const u8, source: []const u8, expected_output: []const u8) void {
66 const tc = self.create(name, source, expected_output);
67 self.addCase(tc);
68}
69
70pub fn addAsm(self: *CompareOutput, name: []const u8, source: []const u8, expected_output: []const u8) void {
71 const tc = self.createExtra(name, source, expected_output, Special.Asm);
72 self.addCase(tc);
73}
74
75pub fn addRuntimeSafety(self: *CompareOutput, name: []const u8, source: []const u8) void {
76 const tc = self.createExtra(name, source, undefined, Special.RuntimeSafety);
77 self.addCase(tc);
78}
79
80pub fn addCase(self: *CompareOutput, case: TestCase) void {
81 const b = self.b;
82
83 const write_src = b.addWriteFiles();
84 for (case.sources.items) |src_file| {
85 write_src.add(src_file.filename, src_file.source);
86 }
87
88 switch (case.special) {
89 Special.Asm => {
90 const annotated_case_name = fmt.allocPrint(self.b.allocator, "run assemble-and-link {s}", .{
91 case.name,
92 }) catch @panic("OOM");
93 if (self.test_filter) |filter| {
94 if (mem.indexOf(u8, annotated_case_name, filter) == null) return;
95 }
96
97 const exe = b.addExecutable(.{
98 .name = "test",
99 .target = .{},
100 .optimize = .Debug,
101 });
102 exe.addAssemblyFileSource(write_src.getFileSource(case.sources.items[0].filename).?);
103
104 const run = exe.run();
105 run.setName(annotated_case_name);
106 run.addArgs(case.cli_args);
107 run.expectStdErrEqual("");
108 run.expectStdOutEqual(case.expected_output);
109
110 self.step.dependOn(&run.step);
111 },
112 Special.None => {
113 for (self.optimize_modes) |optimize| {
114 const annotated_case_name = fmt.allocPrint(self.b.allocator, "run compare-output {s} ({s})", .{
115 case.name, @tagName(optimize),
116 }) catch @panic("OOM");
117 if (self.test_filter) |filter| {
118 if (mem.indexOf(u8, annotated_case_name, filter) == null) continue;
119 }
120
121 const basename = case.sources.items[0].filename;
122 const exe = b.addExecutable(.{
123 .name = "test",
124 .root_source_file = write_src.getFileSource(basename).?,
125 .optimize = optimize,
126 .target = .{},
127 });
128 if (case.link_libc) {
129 exe.linkSystemLibrary("c");
130 }
131
132 const run = exe.run();
133 run.setName(annotated_case_name);
134 run.addArgs(case.cli_args);
135 run.expectStdErrEqual("");
136 run.expectStdOutEqual(case.expected_output);
137
138 self.step.dependOn(&run.step);
139 }
140 },
141 Special.RuntimeSafety => {
142 // TODO iterate over self.optimize_modes and test this in both
143 // debug and release safe mode
144 const annotated_case_name = fmt.allocPrint(self.b.allocator, "run safety {s}", .{case.name}) catch @panic("OOM");
145 if (self.test_filter) |filter| {
146 if (mem.indexOf(u8, annotated_case_name, filter) == null) return;
147 }
148
149 const basename = case.sources.items[0].filename;
150 const exe = b.addExecutable(.{
151 .name = "test",
152 .root_source_file = write_src.getFileSource(basename).?,
153 .target = .{},
154 .optimize = .Debug,
155 });
156 if (case.link_libc) {
157 exe.linkSystemLibrary("c");
158 }
159
160 const run = exe.run();
161 run.setName(annotated_case_name);
162 run.addArgs(case.cli_args);
163 run.expectExitCode(126);
164
165 self.step.dependOn(&run.step);
166 },
167 }
168}
169
170const CompareOutput = @This();
171const std = @import("std");
172const ArrayList = std.ArrayList;
173const fmt = std.fmt;
174const mem = std.mem;
175const fs = std.fs;
176const OptimizeMode = std.builtin.OptimizeMode;
test/src/compare_output.zig deleted-175
...@@ -1,175 +0,0 @@
1// This is the implementation of the test harness.
2// For the actual test cases, see test/compare_output.zig.
3const std = @import("std");
4const ArrayList = std.ArrayList;
5const fmt = std.fmt;
6const mem = std.mem;
7const fs = std.fs;
8const OptimizeMode = std.builtin.OptimizeMode;
9
10pub const CompareOutputContext = struct {
11 b: *std.Build,
12 step: *std.Build.Step,
13 test_index: usize,
14 test_filter: ?[]const u8,
15 optimize_modes: []const OptimizeMode,
16
17 const Special = enum {
18 None,
19 Asm,
20 RuntimeSafety,
21 };
22
23 const TestCase = struct {
24 name: []const u8,
25 sources: ArrayList(SourceFile),
26 expected_output: []const u8,
27 link_libc: bool,
28 special: Special,
29 cli_args: []const []const u8,
30
31 const SourceFile = struct {
32 filename: []const u8,
33 source: []const u8,
34 };
35
36 pub fn addSourceFile(self: *TestCase, filename: []const u8, source: []const u8) void {
37 self.sources.append(SourceFile{
38 .filename = filename,
39 .source = source,
40 }) catch unreachable;
41 }
42
43 pub fn setCommandLineArgs(self: *TestCase, args: []const []const u8) void {
44 self.cli_args = args;
45 }
46 };
47
48 pub fn createExtra(self: *CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8, special: Special) TestCase {
49 var tc = TestCase{
50 .name = name,
51 .sources = ArrayList(TestCase.SourceFile).init(self.b.allocator),
52 .expected_output = expected_output,
53 .link_libc = false,
54 .special = special,
55 .cli_args = &[_][]const u8{},
56 };
57 const root_src_name = if (special == Special.Asm) "source.s" else "source.zig";
58 tc.addSourceFile(root_src_name, source);
59 return tc;
60 }
61
62 pub fn create(self: *CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8) TestCase {
63 return createExtra(self, name, source, expected_output, Special.None);
64 }
65
66 pub fn addC(self: *CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8) void {
67 var tc = self.create(name, source, expected_output);
68 tc.link_libc = true;
69 self.addCase(tc);
70 }
71
72 pub fn add(self: *CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8) void {
73 const tc = self.create(name, source, expected_output);
74 self.addCase(tc);
75 }
76
77 pub fn addAsm(self: *CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8) void {
78 const tc = self.createExtra(name, source, expected_output, Special.Asm);
79 self.addCase(tc);
80 }
81
82 pub fn addRuntimeSafety(self: *CompareOutputContext, name: []const u8, source: []const u8) void {
83 const tc = self.createExtra(name, source, undefined, Special.RuntimeSafety);
84 self.addCase(tc);
85 }
86
87 pub fn addCase(self: *CompareOutputContext, case: TestCase) void {
88 const b = self.b;
89
90 const write_src = b.addWriteFiles();
91 for (case.sources.items) |src_file| {
92 write_src.add(src_file.filename, src_file.source);
93 }
94
95 switch (case.special) {
96 Special.Asm => {
97 const annotated_case_name = fmt.allocPrint(self.b.allocator, "assemble-and-link {s}", .{
98 case.name,
99 }) catch unreachable;
100 if (self.test_filter) |filter| {
101 if (mem.indexOf(u8, annotated_case_name, filter) == null) return;
102 }
103
104 const exe = b.addExecutable(.{
105 .name = "test",
106 .target = .{},
107 .optimize = .Debug,
108 });
109 exe.addAssemblyFileSource(write_src.getFileSource(case.sources.items[0].filename).?);
110
111 const run = exe.run();
112 run.addArgs(case.cli_args);
113 run.expectStdErrEqual("");
114 run.expectStdOutEqual(case.expected_output);
115
116 self.step.dependOn(&run.step);
117 },
118 Special.None => {
119 for (self.optimize_modes) |optimize| {
120 const annotated_case_name = fmt.allocPrint(self.b.allocator, "{s} {s} ({s})", .{
121 "compare-output",
122 case.name,
123 @tagName(optimize),
124 }) catch unreachable;
125 if (self.test_filter) |filter| {
126 if (mem.indexOf(u8, annotated_case_name, filter) == null) continue;
127 }
128
129 const basename = case.sources.items[0].filename;
130 const exe = b.addExecutable(.{
131 .name = "test",
132 .root_source_file = write_src.getFileSource(basename).?,
133 .optimize = optimize,
134 .target = .{},
135 });
136 if (case.link_libc) {
137 exe.linkSystemLibrary("c");
138 }
139
140 const run = exe.run();
141 run.addArgs(case.cli_args);
142 run.expectStdErrEqual("");
143 run.expectStdOutEqual(case.expected_output);
144
145 self.step.dependOn(&run.step);
146 }
147 },
148 Special.RuntimeSafety => {
149 // TODO iterate over self.optimize_modes and test this in both
150 // debug and release safe mode
151 const annotated_case_name = fmt.allocPrint(self.b.allocator, "safety {s}", .{case.name}) catch unreachable;
152 if (self.test_filter) |filter| {
153 if (mem.indexOf(u8, annotated_case_name, filter) == null) return;
154 }
155
156 const basename = case.sources.items[0].filename;
157 const exe = b.addExecutable(.{
158 .name = "test",
159 .root_source_file = write_src.getFileSource(basename).?,
160 .target = .{},
161 .optimize = .Debug,
162 });
163 if (case.link_libc) {
164 exe.linkSystemLibrary("c");
165 }
166
167 const run = exe.run();
168 run.addArgs(case.cli_args);
169 run.expectExitCode(126);
170
171 self.step.dependOn(&run.step);
172 },
173 }
174 }
175};
test/tests.zig+1-1
...@@ -25,7 +25,7 @@ const link = @import("link.zig");...@@ -25,7 +25,7 @@ const link = @import("link.zig");
25// Implementations25// Implementations
26pub const TranslateCContext = @import("src/translate_c.zig").TranslateCContext;26pub const TranslateCContext = @import("src/translate_c.zig").TranslateCContext;
27pub const RunTranslatedCContext = @import("src/run_translated_c.zig").RunTranslatedCContext;27pub const RunTranslatedCContext = @import("src/run_translated_c.zig").RunTranslatedCContext;
28pub const CompareOutputContext = @import("src/compare_output.zig").CompareOutputContext;28pub const CompareOutputContext = @import("src/CompareOutput.zig");
29pub const StackTracesContext = @import("src/StackTrace.zig");29pub const StackTracesContext = @import("src/StackTrace.zig");
30pub const StandaloneContext = @import("src/Standalone.zig");30pub const StandaloneContext = @import("src/Standalone.zig");
3131