authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-12-28 20:01:17-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-12-28 20:01:17-07:00
log7561fca4352e18f03fd4950591341150b9839fda
treebd2a9d9d5147f129359000b620141c92b2870242
parent7ca9f3bc7ba1a413ea7f8b88dfafba3d0f842f6e

stage2: improve test harness to support executing generated C code

2 files changed, 138 insertions(+), 92 deletions(-)

src/test.zig+115-92
......@@ -11,8 +11,9 @@ const enable_wine: bool = build_options.enable_wine;
1111const enable_wasmtime: bool = build_options.enable_wasmtime;
1212const glibc_multi_install_dir: ?[]const u8 = build_options.glibc_multi_install_dir;
1313const ThreadPool = @import("ThreadPool.zig");
14const CrossTarget = std.zig.CrossTarget;
1415
15const cheader = @embedFile("link/cbe.h");
16const c_header = @embedFile("link/cbe.h");
1617
1718test "self-hosted" {
1819 var ctx = TestContext.init();
......@@ -88,6 +89,9 @@ pub const TestContext = struct {
8889 /// A transformation update transforms the input and tests against
8990 /// the expected output ZIR.
9091 Transformation: [:0]const u8,
92 /// Check the main binary output file against an expected set of bytes.
93 /// This is most useful with, for example, `-ofmt=c`.
94 CompareObjectFile: []const u8,
9195 /// An error update attempts to compile bad code, and ensures that it
9296 /// fails to compile, and for the expected reasons.
9397 /// A slice containing the expected errors *in sequential order*.
......@@ -109,12 +113,12 @@ pub const TestContext = struct {
109113 path: []const u8,
110114 };
111115
112 pub const TestType = enum {
116 pub const Extension = enum {
113117 Zig,
114118 ZIR,
115119 };
116120
117 /// A Case consists of a set of *updates*. The same Compilation is used for each
121 /// A `Case` consists of a list of `Update`. The same `Compilation` is used for each
118122 /// update, so each update's source is treated as a single file being
119123 /// updated by the test harness and incrementally compiled.
120124 pub const Case = struct {
......@@ -123,13 +127,14 @@ pub const TestContext = struct {
123127 name: []const u8,
124128 /// The platform the test targets. For non-native platforms, an emulator
125129 /// such as QEMU is required for tests to complete.
126 target: std.zig.CrossTarget,
130 target: CrossTarget,
127131 /// In order to be able to run e.g. Execution updates, this must be set
128132 /// to Executable.
129133 output_mode: std.builtin.OutputMode,
130134 updates: std.ArrayList(Update),
131 extension: TestType,
132 cbe: bool = false,
135 extension: Extension,
136 object_format: ?std.builtin.ObjectFormat = null,
137 emit_h: bool = false,
133138
134139 files: std.ArrayList(File),
135140
......@@ -145,6 +150,7 @@ pub const TestContext = struct {
145150 /// Adds a subcase in which the module is updated with `src`, and a C
146151 /// header is generated.
147152 pub fn addHeader(self: *Case, src: [:0]const u8, result: [:0]const u8) void {
153 self.emit_h = true;
148154 self.updates.append(.{
149155 .src = src,
150156 .case = .{ .Header = result },
......@@ -160,6 +166,15 @@ pub const TestContext = struct {
160166 }) catch unreachable;
161167 }
162168
169 /// Adds a subcase in which the module is updated with `src`, compiled,
170 /// and the object file data is compared against `result`.
171 pub fn addCompareObjectFile(self: *Case, src: [:0]const u8, result: []const u8) void {
172 self.updates.append(.{
173 .src = src,
174 .case = .{ .CompareObjectFile = result },
175 }) catch unreachable;
176 }
177
163178 /// Adds a subcase in which the module is updated with `src`, which
164179 /// should contain invalid input, and ensures that compilation fails
165180 /// for the expected reasons, given in sequential order in `errors` in
......@@ -214,86 +229,100 @@ pub const TestContext = struct {
214229 pub fn addExe(
215230 ctx: *TestContext,
216231 name: []const u8,
217 target: std.zig.CrossTarget,
218 T: TestType,
232 target: CrossTarget,
233 extension: Extension,
219234 ) *Case {
220235 ctx.cases.append(Case{
221236 .name = name,
222237 .target = target,
223238 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
224239 .output_mode = .Exe,
225 .extension = T,
240 .extension = extension,
226241 .files = std.ArrayList(File).init(ctx.cases.allocator),
227242 }) catch unreachable;
228243 return &ctx.cases.items[ctx.cases.items.len - 1];
229244 }
230245
231246 /// Adds a test case for Zig input, producing an executable
232 pub fn exe(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget) *Case {
247 pub fn exe(ctx: *TestContext, name: []const u8, target: CrossTarget) *Case {
233248 return ctx.addExe(name, target, .Zig);
234249 }
235250
236251 /// Adds a test case for ZIR input, producing an executable
237 pub fn exeZIR(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget) *Case {
252 pub fn exeZIR(ctx: *TestContext, name: []const u8, target: CrossTarget) *Case {
238253 return ctx.addExe(name, target, .ZIR);
239254 }
240255
256 pub fn exeFromCompiledC(ctx: *TestContext, name: []const u8, target: CrossTarget) *Case {
257 ctx.cases.append(Case{
258 .name = name,
259 .target = target,
260 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
261 .output_mode = .Exe,
262 .extension = .Zig,
263 .object_format = .c,
264 .files = std.ArrayList(File).init(ctx.cases.allocator),
265 }) catch unreachable;
266 return &ctx.cases.items[ctx.cases.items.len - 1];
267 }
268
241269 pub fn addObj(
242270 ctx: *TestContext,
243271 name: []const u8,
244 target: std.zig.CrossTarget,
245 T: TestType,
272 target: CrossTarget,
273 extension: Extension,
246274 ) *Case {
247275 ctx.cases.append(Case{
248276 .name = name,
249277 .target = target,
250278 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
251279 .output_mode = .Obj,
252 .extension = T,
280 .extension = extension,
253281 .files = std.ArrayList(File).init(ctx.cases.allocator),
254282 }) catch unreachable;
255283 return &ctx.cases.items[ctx.cases.items.len - 1];
256284 }
257285
258 /// Adds a test case for Zig input, producing an object file
259 pub fn obj(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget) *Case {
286 /// Adds a test case for Zig input, producing an object file.
287 pub fn obj(ctx: *TestContext, name: []const u8, target: CrossTarget) *Case {
260288 return ctx.addObj(name, target, .Zig);
261289 }
262290
263 /// Adds a test case for ZIR input, producing an object file
264 pub fn objZIR(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget) *Case {
291 /// Adds a test case for ZIR input, producing an object file.
292 pub fn objZIR(ctx: *TestContext, name: []const u8, target: CrossTarget) *Case {
265293 return ctx.addObj(name, target, .ZIR);
266294 }
267295
268 pub fn addC(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget, T: TestType) *Case {
296 /// Adds a test case for Zig or ZIR input, producing C code.
297 pub fn addC(ctx: *TestContext, name: []const u8, target: CrossTarget, ext: Extension) *Case {
269298 ctx.cases.append(Case{
270299 .name = name,
271300 .target = target,
272301 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
273302 .output_mode = .Obj,
274 .extension = T,
275 .cbe = true,
303 .extension = ext,
304 .object_format = .c,
276305 .files = std.ArrayList(File).init(ctx.cases.allocator),
277306 }) catch unreachable;
278307 return &ctx.cases.items[ctx.cases.items.len - 1];
279308 }
280309
281 pub fn c(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget, src: [:0]const u8, comptime out: [:0]const u8) void {
282 ctx.addC(name, target, .Zig).addTransform(src, cheader ++ out);
310 pub fn c(ctx: *TestContext, name: []const u8, target: CrossTarget, src: [:0]const u8, comptime out: [:0]const u8) void {
311 ctx.addC(name, target, .Zig).addCompareObjectFile(src, c_header ++ out);
283312 }
284313
285 pub fn h(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget, src: [:0]const u8, comptime out: [:0]const u8) void {
286 ctx.addC(name, target, .Zig).addHeader(src, cheader ++ out);
314 pub fn h(ctx: *TestContext, name: []const u8, target: CrossTarget, src: [:0]const u8, comptime out: [:0]const u8) void {
315 ctx.addC(name, target, .Zig).addHeader(src, c_header ++ out);
287316 }
288317
289318 pub fn addCompareOutput(
290319 ctx: *TestContext,
291320 name: []const u8,
292 T: TestType,
321 extension: Extension,
293322 src: [:0]const u8,
294323 expected_stdout: []const u8,
295324 ) void {
296 ctx.addExe(name, .{}, T).addCompareOutput(src, expected_stdout);
325 ctx.addExe(name, .{}, extension).addCompareOutput(src, expected_stdout);
297326 }
298327
299328 /// Adds a test case that compiles the Zig source given in `src`, executes
......@@ -321,12 +350,12 @@ pub const TestContext = struct {
321350 pub fn addTransform(
322351 ctx: *TestContext,
323352 name: []const u8,
324 target: std.zig.CrossTarget,
325 T: TestType,
353 target: CrossTarget,
354 extension: Extension,
326355 src: [:0]const u8,
327356 result: [:0]const u8,
328357 ) void {
329 ctx.addObj(name, target, T).addTransform(src, result);
358 ctx.addObj(name, target, extension).addTransform(src, result);
330359 }
331360
332361 /// Adds a test case that compiles the Zig given in `src` to ZIR and tests
......@@ -334,7 +363,7 @@ pub const TestContext = struct {
334363 pub fn transform(
335364 ctx: *TestContext,
336365 name: []const u8,
337 target: std.zig.CrossTarget,
366 target: CrossTarget,
338367 src: [:0]const u8,
339368 result: [:0]const u8,
340369 ) void {
......@@ -346,7 +375,7 @@ pub const TestContext = struct {
346375 pub fn transformZIR(
347376 ctx: *TestContext,
348377 name: []const u8,
349 target: std.zig.CrossTarget,
378 target: CrossTarget,
350379 src: [:0]const u8,
351380 result: [:0]const u8,
352381 ) void {
......@@ -356,12 +385,12 @@ pub const TestContext = struct {
356385 pub fn addError(
357386 ctx: *TestContext,
358387 name: []const u8,
359 target: std.zig.CrossTarget,
360 T: TestType,
388 target: CrossTarget,
389 extension: Extension,
361390 src: [:0]const u8,
362391 expected_errors: []const []const u8,
363392 ) void {
364 ctx.addObj(name, target, T).addError(src, expected_errors);
393 ctx.addObj(name, target, extension).addError(src, expected_errors);
365394 }
366395
367396 /// Adds a test case that ensures that the Zig given in `src` fails to
......@@ -370,7 +399,7 @@ pub const TestContext = struct {
370399 pub fn compileError(
371400 ctx: *TestContext,
372401 name: []const u8,
373 target: std.zig.CrossTarget,
402 target: CrossTarget,
374403 src: [:0]const u8,
375404 expected_errors: []const []const u8,
376405 ) void {
......@@ -383,7 +412,7 @@ pub const TestContext = struct {
383412 pub fn compileErrorZIR(
384413 ctx: *TestContext,
385414 name: []const u8,
386 target: std.zig.CrossTarget,
415 target: CrossTarget,
387416 src: [:0]const u8,
388417 expected_errors: []const []const u8,
389418 ) void {
......@@ -393,11 +422,11 @@ pub const TestContext = struct {
393422 pub fn addCompiles(
394423 ctx: *TestContext,
395424 name: []const u8,
396 target: std.zig.CrossTarget,
397 T: TestType,
425 target: CrossTarget,
426 extension: Extension,
398427 src: [:0]const u8,
399428 ) void {
400 ctx.addObj(name, target, T).compiles(src);
429 ctx.addObj(name, target, extension).compiles(src);
401430 }
402431
403432 /// Adds a test case that asserts that the Zig given in `src` compiles
......@@ -405,7 +434,7 @@ pub const TestContext = struct {
405434 pub fn compiles(
406435 ctx: *TestContext,
407436 name: []const u8,
408 target: std.zig.CrossTarget,
437 target: CrossTarget,
409438 src: [:0]const u8,
410439 ) void {
411440 ctx.addCompiles(name, target, .Zig, src);
......@@ -416,7 +445,7 @@ pub const TestContext = struct {
416445 pub fn compilesZIR(
417446 ctx: *TestContext,
418447 name: []const u8,
419 target: std.zig.CrossTarget,
448 target: CrossTarget,
420449 src: [:0]const u8,
421450 ) void {
422451 ctx.addCompiles(name, target, .ZIR, src);
......@@ -430,7 +459,7 @@ pub const TestContext = struct {
430459 pub fn incrementalFailure(
431460 ctx: *TestContext,
432461 name: []const u8,
433 target: std.zig.CrossTarget,
462 target: CrossTarget,
434463 src: [:0]const u8,
435464 expected_errors: []const []const u8,
436465 fixed_src: [:0]const u8,
......@@ -448,7 +477,7 @@ pub const TestContext = struct {
448477 pub fn incrementalFailureZIR(
449478 ctx: *TestContext,
450479 name: []const u8,
451 target: std.zig.CrossTarget,
480 target: CrossTarget,
452481 src: [:0]const u8,
453482 expected_errors: []const []const u8,
454483 fixed_src: [:0]const u8,
......@@ -548,12 +577,11 @@ pub const TestContext = struct {
548577 .root_src_path = tmp_src_path,
549578 };
550579
551 const ofmt: ?std.builtin.ObjectFormat = if (case.cbe) .c else null;
552580 const bin_name = try std.zig.binNameAlloc(arena, .{
553581 .root_name = "test_case",
554582 .target = target,
555583 .output_mode = case.output_mode,
556 .object_format = ofmt,
584 .object_format = case.object_format,
557585 });
558586
559587 const emit_directory: Compilation.Directory = .{
......@@ -564,7 +592,7 @@ pub const TestContext = struct {
564592 .directory = emit_directory,
565593 .basename = bin_name,
566594 };
567 const emit_h: ?Compilation.EmitLoc = if (case.cbe)
595 const emit_h: ?Compilation.EmitLoc = if (case.emit_h)
568596 .{
569597 .directory = emit_directory,
570598 .basename = "test_case.h",
......@@ -588,7 +616,7 @@ pub const TestContext = struct {
588616 .emit_h = emit_h,
589617 .root_pkg = &root_pkg,
590618 .keep_source_files_loaded = true,
591 .object_format = ofmt,
619 .object_format = case.object_format,
592620 .is_native_os = case.target.isNativeOs(),
593621 .is_native_abi = case.target.isNativeAbi(),
594622 });
......@@ -631,9 +659,10 @@ pub const TestContext = struct {
631659 },
632660 }
633661 }
634 if (case.cbe) {
635 const C = comp.bin_file.cast(link.File.C).?;
636 std.debug.print("Generated C: \n===============\n{}\n\n===========\n\n", .{C.main.items});
662 if (comp.bin_file.cast(link.File.C)) |c_file| {
663 std.debug.print("Generated C: \n===============\n{}\n\n===========\n\n", .{
664 c_file.main.items,
665 });
637666 }
638667 std.debug.print("Test failed.\n", .{});
639668 std.process.exit(1);
......@@ -644,39 +673,37 @@ pub const TestContext = struct {
644673 .Header => |expected_output| {
645674 var file = try tmp.dir.openFile("test_case.h", .{ .read = true });
646675 defer file.close();
647 var out = file.reader().readAllAlloc(arena, 1024 * 1024) catch @panic("Unable to read headeroutput!");
676 const out = try file.reader().readAllAlloc(arena, 5 * 1024 * 1024);
677
678 std.testing.expectEqualStrings(expected_output, out);
679 },
680 .CompareObjectFile => |expected_output| {
681 var file = try tmp.dir.openFile(bin_name, .{ .read = true });
682 defer file.close();
683 const out = try file.reader().readAllAlloc(arena, 5 * 1024 * 1024);
648684
649685 std.testing.expectEqualStrings(expected_output, out);
650686 },
651687 .Transformation => |expected_output| {
652 if (case.cbe) {
653 // The C file is always closed after an update, because we don't support
654 // incremental updates.
655 var file = try tmp.dir.openFile(bin_name, .{ .read = true });
656 defer file.close();
657 var out = file.reader().readAllAlloc(arena, 1024 * 1024) catch @panic("Unable to read C output!");
658 std.testing.expectEqualStrings(expected_output, out);
659 } else {
660 update_node.setEstimatedTotalItems(5);
661 var emit_node = update_node.start("emit", 0);
662 emit_node.activate();
663 var new_zir_module = try zir.emit(allocator, comp.bin_file.options.module.?);
664 defer new_zir_module.deinit(allocator);
665 emit_node.end();
666
667 var write_node = update_node.start("write", 0);
668 write_node.activate();
669 var out_zir = std.ArrayList(u8).init(allocator);
670 defer out_zir.deinit();
671 try new_zir_module.writeToStream(allocator, out_zir.outStream());
672 write_node.end();
673
674 var test_node = update_node.start("assert", 0);
675 test_node.activate();
676 defer test_node.end();
677
678 std.testing.expectEqualStrings(expected_output, out_zir.items);
679 }
688 update_node.setEstimatedTotalItems(5);
689 var emit_node = update_node.start("emit", 0);
690 emit_node.activate();
691 var new_zir_module = try zir.emit(allocator, comp.bin_file.options.module.?);
692 defer new_zir_module.deinit(allocator);
693 emit_node.end();
694
695 var write_node = update_node.start("write", 0);
696 write_node.activate();
697 var out_zir = std.ArrayList(u8).init(allocator);
698 defer out_zir.deinit();
699 try new_zir_module.writeToStream(allocator, out_zir.outStream());
700 write_node.end();
701
702 var test_node = update_node.start("assert", 0);
703 test_node.activate();
704 defer test_node.end();
705
706 std.testing.expectEqualStrings(expected_output, out_zir.items);
680707 },
681708 .Error => |e| {
682709 var test_node = update_node.start("assert", 0);
......@@ -734,8 +761,6 @@ pub const TestContext = struct {
734761 }
735762 },
736763 .Execution => |expected_stdout| {
737 std.debug.assert(!case.cbe);
738
739764 update_node.setEstimatedTotalItems(4);
740765 var exec_result = x: {
741766 var exec_node = update_node.start("execute", 0);
......@@ -745,9 +770,12 @@ pub const TestContext = struct {
745770 var argv = std.ArrayList([]const u8).init(allocator);
746771 defer argv.deinit();
747772
748 const exe_path = try std.fmt.allocPrint(arena, "." ++ std.fs.path.sep_str ++ "{}", .{bin_name});
749
750 switch (case.target.getExternalExecutor()) {
773 const exe_path = try std.fmt.allocPrint(arena, "." ++ std.fs.path.sep_str ++ "{s}", .{bin_name});
774 if (case.object_format != null and case.object_format.? == .c) {
775 try argv.appendSlice(&[_][]const u8{
776 std.testing.zig_exe_path, "run", exe_path, "-lc",
777 });
778 } else switch (case.target.getExternalExecutor()) {
751779 .native => try argv.append(exe_path),
752780 .unavailable => {
753781 try self.runInterpreterIfAvailable(allocator, &exec_node, case, tmp.dir, bin_name);
......@@ -809,18 +837,13 @@ pub const TestContext = struct {
809837 switch (exec_result.term) {
810838 .Exited => |code| {
811839 if (code != 0) {
812 std.debug.print("elf file exited with code {}\n", .{code});
840 std.debug.print("execution exited with code {}\n", .{code});
813841 return error.BinaryBadExitCode;
814842 }
815843 },
816844 else => return error.BinaryCrashed,
817845 }
818 if (!std.mem.eql(u8, expected_stdout, exec_result.stdout)) {
819 std.debug.panic(
820 "update index {}, mismatched stdout\n====Expected (len={}):====\n{}\n====Actual (len={}):====\n{}\n========\n",
821 .{ update_index, expected_stdout.len, expected_stdout, exec_result.stdout.len, exec_result.stdout },
822 );
823 }
846 std.testing.expectEqualStrings(expected_stdout, exec_result.stdout);
824847 },
825848 }
826849 }
test/stage2/cbe.zig+23
......@@ -9,6 +9,29 @@ const linux_x64 = std.zig.CrossTarget{
99};
1010
1111pub fn addCases(ctx: *TestContext) !void {
12 {
13 var case = ctx.exeFromCompiledC("hello world with updates", .{});
14
15 // Regular old hello world
16 case.addCompareOutput(
17 \\extern fn puts(s: [*:0]const u8) c_int;
18 \\export fn main() c_int {
19 \\ _ = puts("hello world!");
20 \\ return 0;
21 \\}
22 , "hello world!" ++ std.cstr.line_sep);
23
24 // Now change the message only
25 // TODO fix C backend not supporting updates
26 //case.addCompareOutput(
27 // \\extern fn puts(s: [*:0]const u8) c_int;
28 // \\export fn main() c_int {
29 // \\ _ = puts("yo");
30 // \\ return 0;
31 // \\}
32 //, "yo" ++ std.cstr.line_sep);
33 }
34
1235 ctx.c("empty start function", linux_x64,
1336 \\export fn _start() noreturn {
1437 \\ unreachable;