| ... | ... | @@ -1,6 +1,7 @@ |
| 1 | 1 | gpa: Allocator, |
| 2 | 2 | arena: Allocator, |
| 3 | 3 | cases: std.ArrayList(Case), |
| 4 | translate: std.ArrayList(Translate), |
| 4 | 5 | incremental_cases: std.ArrayList(IncrementalCase), |
| 5 | 6 | |
| 6 | 7 | pub const IncrementalCase = struct { |
| ... | ... | @@ -36,7 +37,7 @@ pub const Update = struct { |
| 36 | 37 | Execution: []const u8, |
| 37 | 38 | /// A header update compiles the input with the equivalent of |
| 38 | 39 | /// `-femit-h` and tests the produced header against the |
| 39 | | /// expected result |
| 40 | /// expected result. |
| 40 | 41 | Header: []const u8, |
| 41 | 42 | }, |
| 42 | 43 | |
| ... | ... | @@ -61,6 +62,11 @@ pub const Backend = enum { |
| 61 | 62 | llvm, |
| 62 | 63 | }; |
| 63 | 64 | |
| 65 | pub const CFrontend = enum { |
| 66 | clang, |
| 67 | aro, |
| 68 | }; |
| 69 | |
| 64 | 70 | /// A `Case` consists of a list of `Update`. The same `Compilation` is used for each |
| 65 | 71 | /// update, so each update's source is treated as a single file being |
| 66 | 72 | /// updated by the test harness and incrementally compiled. |
| ... | ... | @@ -143,6 +149,25 @@ pub const Case = struct { |
| 143 | 149 | } |
| 144 | 150 | }; |
| 145 | 151 | |
| 152 | pub const Translate = struct { |
| 153 | /// The name of the test case. This is shown if a test fails, and |
| 154 | /// otherwise ignored. |
| 155 | name: []const u8, |
| 156 | |
| 157 | input: [:0]const u8, |
| 158 | target: CrossTarget, |
| 159 | link_libc: bool, |
| 160 | c_frontend: CFrontend, |
| 161 | kind: union(enum) { |
| 162 | /// Translate the input, run it and check that it |
| 163 | /// outputs the expected text. |
| 164 | run: []const u8, |
| 165 | /// Translate the input and check that it contains |
| 166 | /// the expected lines of code. |
| 167 | translate: []const []const u8, |
| 168 | }, |
| 169 | }; |
| 170 | |
| 146 | 171 | pub fn addExe( |
| 147 | 172 | ctx: *Cases, |
| 148 | 173 | name: []const u8, |
| ... | ... | @@ -346,9 +371,12 @@ pub fn addCompile( |
| 346 | 371 | pub fn addFromDir(ctx: *Cases, dir: std.fs.IterableDir) void { |
| 347 | 372 | var current_file: []const u8 = "none"; |
| 348 | 373 | ctx.addFromDirInner(dir, &current_file) catch |err| { |
| 349 | | std.debug.panic("test harness failed to process file '{s}': {s}\n", .{ |
| 350 | | current_file, @errorName(err), |
| 351 | | }); |
| 374 | std.debug.panicExtra( |
| 375 | @errorReturnTrace(), |
| 376 | @returnAddress(), |
| 377 | "test harness failed to process file '{s}': {s}\n", |
| 378 | .{ current_file, @errorName(err) }, |
| 379 | ); |
| 352 | 380 | }; |
| 353 | 381 | } |
| 354 | 382 | |
| ... | ... | @@ -395,10 +423,44 @@ fn addFromDirInner( |
| 395 | 423 | |
| 396 | 424 | const backends = try manifest.getConfigForKeyAlloc(ctx.arena, "backend", Backend); |
| 397 | 425 | const targets = try manifest.getConfigForKeyAlloc(ctx.arena, "target", CrossTarget); |
| 426 | const c_frontends = try manifest.getConfigForKeyAlloc(ctx.arena, "c_frontend", CFrontend); |
| 398 | 427 | const is_test = try manifest.getConfigForKeyAssertSingle("is_test", bool); |
| 399 | 428 | const link_libc = try manifest.getConfigForKeyAssertSingle("link_libc", bool); |
| 400 | 429 | const output_mode = try manifest.getConfigForKeyAssertSingle("output_mode", std.builtin.OutputMode); |
| 401 | 430 | |
| 431 | if (manifest.type == .translate_c) { |
| 432 | for (c_frontends) |c_frontend| { |
| 433 | for (targets) |target| { |
| 434 | const output = try manifest.trailingLinesSplit(ctx.arena); |
| 435 | try ctx.translate.append(.{ |
| 436 | .name = std.fs.path.stem(filename), |
| 437 | .c_frontend = c_frontend, |
| 438 | .target = target, |
| 439 | .link_libc = link_libc, |
| 440 | .input = src, |
| 441 | .kind = .{ .translate = output }, |
| 442 | }); |
| 443 | } |
| 444 | } |
| 445 | continue; |
| 446 | } |
| 447 | if (manifest.type == .run_translated_c) { |
| 448 | for (c_frontends) |c_frontend| { |
| 449 | for (targets) |target| { |
| 450 | const output = try manifest.trailingSplit(ctx.arena); |
| 451 | try ctx.translate.append(.{ |
| 452 | .name = std.fs.path.stem(filename), |
| 453 | .c_frontend = c_frontend, |
| 454 | .target = target, |
| 455 | .link_libc = link_libc, |
| 456 | .input = src, |
| 457 | .kind = .{ .run = output }, |
| 458 | }); |
| 459 | } |
| 460 | } |
| 461 | continue; |
| 462 | } |
| 463 | |
| 402 | 464 | var cases = std.ArrayList(usize).init(ctx.arena); |
| 403 | 465 | |
| 404 | 466 | // Cross-product to get all possible test combinations |
| ... | ... | @@ -439,21 +501,15 @@ fn addFromDirInner( |
| 439 | 501 | case.addCompile(src); |
| 440 | 502 | }, |
| 441 | 503 | .@"error" => { |
| 442 | | const errors = try manifest.trailingAlloc(ctx.arena); |
| 504 | const errors = try manifest.trailingLines(ctx.arena); |
| 443 | 505 | case.addError(src, errors); |
| 444 | 506 | }, |
| 445 | 507 | .run => { |
| 446 | | var output = std.ArrayList(u8).init(ctx.arena); |
| 447 | | var trailing_it = manifest.trailing(); |
| 448 | | while (trailing_it.next()) |line| { |
| 449 | | try output.appendSlice(line); |
| 450 | | try output.append('\n'); |
| 451 | | } |
| 452 | | if (output.items.len > 0) { |
| 453 | | try output.resize(output.items.len - 1); |
| 454 | | } |
| 455 | | case.addCompareOutput(src, try output.toOwnedSlice()); |
| 508 | const output = try manifest.trailingSplit(ctx.arena); |
| 509 | case.addCompareOutput(src, output); |
| 456 | 510 | }, |
| 511 | .translate_c => @panic("c_frontend specified for compile case"), |
| 512 | .run_translated_c => @panic("c_frontend specified for compile case"), |
| 457 | 513 | .cli => @panic("TODO cli tests"), |
| 458 | 514 | } |
| 459 | 515 | } |
| ... | ... | @@ -468,6 +524,7 @@ pub fn init(gpa: Allocator, arena: Allocator) Cases { |
| 468 | 524 | return .{ |
| 469 | 525 | .gpa = gpa, |
| 470 | 526 | .cases = std.ArrayList(Case).init(gpa), |
| 527 | .translate = std.ArrayList(Translate).init(gpa), |
| 471 | 528 | .incremental_cases = std.ArrayList(IncrementalCase).init(gpa), |
| 472 | 529 | .arena = arena, |
| 473 | 530 | }; |
| ... | ... | @@ -482,7 +539,7 @@ pub fn lowerToBuildSteps( |
| 482 | 539 | incremental_exe: *std.Build.Step.Compile, |
| 483 | 540 | ) void { |
| 484 | 541 | const host = std.zig.system.NativeTargetInfo.detect(.{}) catch |err| |
| 485 | | std.debug.panic("unable to detect notive host: {s}\n", .{@errorName(err)}); |
| 542 | std.debug.panic("unable to detect native host: {s}\n", .{@errorName(err)}); |
| 486 | 543 | |
| 487 | 544 | for (self.incremental_cases.items) |incr_case| { |
| 488 | 545 | if (true) { |
| ... | ... | @@ -589,7 +646,7 @@ pub fn lowerToBuildSteps( |
| 589 | 646 | .Execution => |expected_stdout| no_exec: { |
| 590 | 647 | const run = if (case.target.ofmt == .c) run_step: { |
| 591 | 648 | const target_info = std.zig.system.NativeTargetInfo.detect(case.target) catch |err| |
| 592 | | std.debug.panic("unable to detect notive host: {s}\n", .{@errorName(err)}); |
| 649 | std.debug.panic("unable to detect target host: {s}\n", .{@errorName(err)}); |
| 593 | 650 | if (host.getExternalExecutor(&target_info, .{ .link_libc = true }) != .native) { |
| 594 | 651 | // We wouldn't be able to run the compiled C code. |
| 595 | 652 | break :no_exec; |
| ... | ... | @@ -623,6 +680,68 @@ pub fn lowerToBuildSteps( |
| 623 | 680 | .Header => @panic("TODO"), |
| 624 | 681 | } |
| 625 | 682 | } |
| 683 | |
| 684 | for (self.translate.items) |*case| switch (case.kind) { |
| 685 | .run => |output| { |
| 686 | const annotated_case_name = b.fmt("run-translated-c {s}", .{case.name}); |
| 687 | if (opt_test_filter) |filter| { |
| 688 | if (std.mem.indexOf(u8, annotated_case_name, filter) == null) return; |
| 689 | } |
| 690 | if (!std.process.can_spawn) { |
| 691 | std.debug.print("Unable to spawn child processes on {s}, skipping test.\n", .{@tagName(builtin.os.tag)}); |
| 692 | continue; // Pass test. |
| 693 | } |
| 694 | |
| 695 | const target_info = std.zig.system.NativeTargetInfo.detect(case.target) catch |err| |
| 696 | std.debug.panic("unable to detect target host: {s}\n", .{@errorName(err)}); |
| 697 | if (host.getExternalExecutor(&target_info, .{ .link_libc = true }) != .native) { |
| 698 | // We wouldn't be able to run the compiled C code. |
| 699 | continue; // Pass test. |
| 700 | } |
| 701 | |
| 702 | const write_src = b.addWriteFiles(); |
| 703 | const file_source = write_src.add("tmp.c", case.input); |
| 704 | |
| 705 | const translate_c = b.addTranslateC(.{ |
| 706 | .source_file = file_source, |
| 707 | .optimize = .Debug, |
| 708 | .target = case.target, |
| 709 | .link_libc = case.link_libc, |
| 710 | .use_clang = case.c_frontend == .clang, |
| 711 | }); |
| 712 | translate_c.step.name = b.fmt("{s} translate-c", .{annotated_case_name}); |
| 713 | |
| 714 | const run_exe = translate_c.addExecutable(.{}); |
| 715 | run_exe.step.name = b.fmt("{s} build-exe", .{annotated_case_name}); |
| 716 | run_exe.linkLibC(); |
| 717 | const run = b.addRunArtifact(run_exe); |
| 718 | run.step.name = b.fmt("{s} run", .{annotated_case_name}); |
| 719 | run.expectStdOutEqual(output); |
| 720 | |
| 721 | parent_step.dependOn(&run.step); |
| 722 | }, |
| 723 | .translate => |output| { |
| 724 | const annotated_case_name = b.fmt("zig translate-c {s}", .{case.name}); |
| 725 | if (opt_test_filter) |filter| { |
| 726 | if (std.mem.indexOf(u8, annotated_case_name, filter) == null) return; |
| 727 | } |
| 728 | |
| 729 | const write_src = b.addWriteFiles(); |
| 730 | const file_source = write_src.add("tmp.c", case.input); |
| 731 | |
| 732 | const translate_c = b.addTranslateC(.{ |
| 733 | .source_file = file_source, |
| 734 | .optimize = .Debug, |
| 735 | .target = case.target, |
| 736 | .link_libc = case.link_libc, |
| 737 | .use_clang = case.c_frontend == .clang, |
| 738 | }); |
| 739 | translate_c.step.name = annotated_case_name; |
| 740 | |
| 741 | const check_file = translate_c.addCheckFile(output); |
| 742 | parent_step.dependOn(&check_file.step); |
| 743 | }, |
| 744 | }; |
| 626 | 745 | } |
| 627 | 746 | |
| 628 | 747 | /// Sort test filenames in-place, so that incremental test cases ("foo.0.zig", |
| ... | ... | @@ -780,7 +899,7 @@ const TestManifestConfigDefaults = struct { |
| 780 | 899 | if (std.mem.eql(u8, key, "backend")) { |
| 781 | 900 | return "stage2"; |
| 782 | 901 | } else if (std.mem.eql(u8, key, "target")) { |
| 783 | | if (@"type" == .@"error") { |
| 902 | if (@"type" == .@"error" or @"type" == .translate_c or @"type" == .run_translated_c) { |
| 784 | 903 | return "native"; |
| 785 | 904 | } |
| 786 | 905 | return comptime blk: { |
| ... | ... | @@ -807,12 +926,16 @@ const TestManifestConfigDefaults = struct { |
| 807 | 926 | .@"error" => "Obj", |
| 808 | 927 | .run => "Exe", |
| 809 | 928 | .compile => "Obj", |
| 929 | .translate_c => "Obj", |
| 930 | .run_translated_c => "Obj", |
| 810 | 931 | .cli => @panic("TODO test harness for CLI tests"), |
| 811 | 932 | }; |
| 812 | 933 | } else if (std.mem.eql(u8, key, "is_test")) { |
| 813 | | return "0"; |
| 934 | return "false"; |
| 814 | 935 | } else if (std.mem.eql(u8, key, "link_libc")) { |
| 815 | | return "0"; |
| 936 | return "false"; |
| 937 | } else if (std.mem.eql(u8, key, "c_frontend")) { |
| 938 | return "clang"; |
| 816 | 939 | } else unreachable; |
| 817 | 940 | } |
| 818 | 941 | }; |
| ... | ... | @@ -844,6 +967,8 @@ const TestManifest = struct { |
| 844 | 967 | run, |
| 845 | 968 | cli, |
| 846 | 969 | compile, |
| 970 | translate_c, |
| 971 | run_translated_c, |
| 847 | 972 | }; |
| 848 | 973 | |
| 849 | 974 | const TrailingIterator = struct { |
| ... | ... | @@ -912,6 +1037,10 @@ const TestManifest = struct { |
| 912 | 1037 | break :blk .cli; |
| 913 | 1038 | } else if (std.mem.eql(u8, raw, "compile")) { |
| 914 | 1039 | break :blk .compile; |
| 1040 | } else if (std.mem.eql(u8, raw, "translate-c")) { |
| 1041 | break :blk .translate_c; |
| 1042 | } else if (std.mem.eql(u8, raw, "run-translated-c")) { |
| 1043 | break :blk .run_translated_c; |
| 915 | 1044 | } else { |
| 916 | 1045 | std.log.warn("unknown test case type requested: {s}", .{raw}); |
| 917 | 1046 | return error.UnknownTestCaseType; |
| ... | ... | @@ -979,7 +1108,21 @@ const TestManifest = struct { |
| 979 | 1108 | }; |
| 980 | 1109 | } |
| 981 | 1110 | |
| 982 | | fn trailingAlloc(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const []const u8 { |
| 1111 | fn trailingSplit(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const u8 { |
| 1112 | var out = std.ArrayList(u8).init(allocator); |
| 1113 | defer out.deinit(); |
| 1114 | var trailing_it = self.trailing(); |
| 1115 | while (trailing_it.next()) |line| { |
| 1116 | try out.appendSlice(line); |
| 1117 | try out.append('\n'); |
| 1118 | } |
| 1119 | if (out.items.len > 0) { |
| 1120 | try out.resize(out.items.len - 1); |
| 1121 | } |
| 1122 | return try out.toOwnedSlice(); |
| 1123 | } |
| 1124 | |
| 1125 | fn trailingLines(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const []const u8 { |
| 983 | 1126 | var out = std.ArrayList([]const u8).init(allocator); |
| 984 | 1127 | defer out.deinit(); |
| 985 | 1128 | var it = self.trailing(); |
| ... | ... | @@ -989,6 +1132,28 @@ const TestManifest = struct { |
| 989 | 1132 | return try out.toOwnedSlice(); |
| 990 | 1133 | } |
| 991 | 1134 | |
| 1135 | fn trailingLinesSplit(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const []const u8 { |
| 1136 | // Collect output lines split by empty lines |
| 1137 | var out = std.ArrayList([]const u8).init(allocator); |
| 1138 | defer out.deinit(); |
| 1139 | var buf = std.ArrayList(u8).init(allocator); |
| 1140 | defer buf.deinit(); |
| 1141 | var it = self.trailing(); |
| 1142 | while (it.next()) |line| { |
| 1143 | if (line.len == 0) { |
| 1144 | if (buf.items.len != 0) { |
| 1145 | try out.append(try buf.toOwnedSlice()); |
| 1146 | buf.items.len = 0; |
| 1147 | } |
| 1148 | continue; |
| 1149 | } |
| 1150 | try buf.appendSlice(line); |
| 1151 | try buf.append('\n'); |
| 1152 | } |
| 1153 | try out.append(try buf.toOwnedSlice()); |
| 1154 | return try out.toOwnedSlice(); |
| 1155 | } |
| 1156 | |
| 992 | 1157 | fn ParseFn(comptime T: type) type { |
| 993 | 1158 | return fn ([]const u8) anyerror!T; |
| 994 | 1159 | } |
| ... | ... | @@ -1011,8 +1176,10 @@ const TestManifest = struct { |
| 1011 | 1176 | }.parse, |
| 1012 | 1177 | .Bool => return struct { |
| 1013 | 1178 | fn parse(str: []const u8) anyerror!T { |
| 1014 | | const as_int = try std.fmt.parseInt(u1, str, 0); |
| 1015 | | return as_int > 0; |
| 1179 | if (std.mem.eql(u8, str, "true")) return true; |
| 1180 | if (std.mem.eql(u8, str, "false")) return false; |
| 1181 | std.debug.print("{s}\n", .{str}); |
| 1182 | return error.InvalidBool; |
| 1016 | 1183 | } |
| 1017 | 1184 | }.parse, |
| 1018 | 1185 | .Enum => return struct { |
| ... | ... | @@ -1124,9 +1291,47 @@ pub fn main() !void { |
| 1124 | 1291 | if (cases.items.len == 0) { |
| 1125 | 1292 | const backends = try manifest.getConfigForKeyAlloc(arena, "backend", Backend); |
| 1126 | 1293 | const targets = try manifest.getConfigForKeyAlloc(arena, "target", CrossTarget); |
| 1294 | const c_frontends = try manifest.getConfigForKeyAlloc(ctx.arena, "c_frontend", CFrontend); |
| 1127 | 1295 | const is_test = try manifest.getConfigForKeyAssertSingle("is_test", bool); |
| 1296 | const link_libc = try manifest.getConfigForKeyAssertSingle("link_libc", bool); |
| 1128 | 1297 | const output_mode = try manifest.getConfigForKeyAssertSingle("output_mode", std.builtin.OutputMode); |
| 1129 | 1298 | |
| 1299 | if (manifest.type == .translate_c) { |
| 1300 | for (c_frontends) |c_frontend| { |
| 1301 | for (targets) |target| { |
| 1302 | const output = try manifest.trailingLinesSplit(ctx.arena); |
| 1303 | try ctx.translate.append(.{ |
| 1304 | .name = std.fs.path.stem(filename), |
| 1305 | .c_frontend = c_frontend, |
| 1306 | .target = target, |
| 1307 | .is_test = is_test, |
| 1308 | .link_libc = link_libc, |
| 1309 | .input = src, |
| 1310 | .kind = .{ .translate = output }, |
| 1311 | }); |
| 1312 | } |
| 1313 | } |
| 1314 | continue; |
| 1315 | } |
| 1316 | if (manifest.type == .run_translated_c) { |
| 1317 | for (c_frontends) |c_frontend| { |
| 1318 | for (targets) |target| { |
| 1319 | const output = try manifest.trailingSplit(ctx.arena); |
| 1320 | try ctx.translate.append(.{ |
| 1321 | .name = std.fs.path.stem(filename), |
| 1322 | .c_frontend = c_frontend, |
| 1323 | .target = target, |
| 1324 | .is_test = is_test, |
| 1325 | .link_libc = link_libc, |
| 1326 | .output = output, |
| 1327 | .input = src, |
| 1328 | .kind = .{ .run = output }, |
| 1329 | }); |
| 1330 | } |
| 1331 | } |
| 1332 | continue; |
| 1333 | } |
| 1334 | |
| 1130 | 1335 | // Cross-product to get all possible test combinations |
| 1131 | 1336 | for (backends) |backend| { |
| 1132 | 1337 | for (targets) |target| { |
| ... | ... | @@ -1158,7 +1363,7 @@ pub fn main() !void { |
| 1158 | 1363 | case.addCompile(src); |
| 1159 | 1364 | }, |
| 1160 | 1365 | .@"error" => { |
| 1161 | | const errors = try manifest.trailingAlloc(arena); |
| 1366 | const errors = try manifest.trailingLines(arena); |
| 1162 | 1367 | switch (strategy) { |
| 1163 | 1368 | .independent => { |
| 1164 | 1369 | case.addError(src, errors); |
| ... | ... | @@ -1169,17 +1374,11 @@ pub fn main() !void { |
| 1169 | 1374 | } |
| 1170 | 1375 | }, |
| 1171 | 1376 | .run => { |
| 1172 | | var output = std.ArrayList(u8).init(arena); |
| 1173 | | var trailing_it = manifest.trailing(); |
| 1174 | | while (trailing_it.next()) |line| { |
| 1175 | | try output.appendSlice(line); |
| 1176 | | try output.append('\n'); |
| 1177 | | } |
| 1178 | | if (output.items.len > 0) { |
| 1179 | | try output.resize(output.items.len - 1); |
| 1180 | | } |
| 1181 | | case.addCompareOutput(src, try output.toOwnedSlice()); |
| 1377 | const output = try manifest.trailingSplit(ctx.arena); |
| 1378 | case.addCompareOutput(src, output); |
| 1182 | 1379 | }, |
| 1380 | .translate_c => @panic("c_frontend specified for compile case"), |
| 1381 | .run_translated_c => @panic("c_frontend specified for compile case"), |
| 1183 | 1382 | .cli => @panic("TODO cli tests"), |
| 1184 | 1383 | } |
| 1185 | 1384 | } |
| ... | ... | @@ -1255,6 +1454,11 @@ fn runCases(self: *Cases, zig_exe_path: []const u8) !void { |
| 1255 | 1454 | host, |
| 1256 | 1455 | ); |
| 1257 | 1456 | } |
| 1457 | |
| 1458 | for (self.translate.items) |*case| { |
| 1459 | _ = case; |
| 1460 | @panic("TODO is this even used?"); |
| 1461 | } |
| 1258 | 1462 | } |
| 1259 | 1463 | } |
| 1260 | 1464 | |