| ... | ... | @@ -46,6 +46,7 @@ test { |
| 46 | 46 | defer stage2_dir.close(); |
| 47 | 47 | |
| 48 | 48 | // TODO make this incremental once the bug is solved that it triggers |
| 49 | // See: https://github.com/ziglang/zig/issues/11344 |
| 49 | 50 | ctx.addErrorCasesFromDir("stage2", stage2_dir, .stage2, .Obj, false, .independent); |
| 50 | 51 | } |
| 51 | 52 | |
| ... | ... | @@ -653,7 +654,14 @@ pub const TestContext = struct { |
| 653 | 654 | case.compiles(fixed_src); |
| 654 | 655 | } |
| 655 | 656 | |
| 656 | | const Strategy = enum { incremental, independent }; |
| 657 | const Strategy = enum { |
| 658 | /// Execute tests as independent compilations, unless they are explicitly |
| 659 | /// incremental ("foo.1.zig", "foo.2.zig", etc.) |
| 660 | independent, |
| 661 | /// Execute all tests as incremental updates to a single compilation. Explicitly |
| 662 | /// incremental tests ("foo.1.zig", "foo.2.zig", etc.) still execute in order |
| 663 | incremental, |
| 664 | }; |
| 657 | 665 | |
| 658 | 666 | /// Adds a compile-error test for each file in the provided directory, using the |
| 659 | 667 | /// selected backend and output mode. If `one_test_case_per_file` is true, a new |
| ... | ... | @@ -681,6 +689,66 @@ pub const TestContext = struct { |
| 681 | 689 | }; |
| 682 | 690 | } |
| 683 | 691 | |
| 692 | /// For a filename in the format "<filename>.X.<ext>" or "<filename>.<ext>", returns |
| 693 | /// "<filename>", "<ext>" and X parsed as a decimal number. If X is not present, or |
| 694 | /// cannot be parsed as a decimal number, it is treated as part of <filename> |
| 695 | fn getTestFileNameParts(name: []const u8) struct { |
| 696 | base_name: []const u8, |
| 697 | file_ext: []const u8, |
| 698 | test_index: ?usize, |
| 699 | } { |
| 700 | const file_ext = std.fs.path.extension(name); |
| 701 | const trimmed = name[0 .. name.len - file_ext.len]; // Trim off ".<ext>" |
| 702 | const maybe_index = std.fs.path.extension(trimmed); // Extract ".X" |
| 703 | |
| 704 | // Attempt to parse index |
| 705 | const index: ?usize = if (maybe_index.len > 0) |
| 706 | std.fmt.parseInt(usize, maybe_index[1..], 10) catch null |
| 707 | else |
| 708 | null; |
| 709 | |
| 710 | // Adjust "<filename>" extent based on parsing success |
| 711 | const base_name_end = trimmed.len - if (index != null) maybe_index.len else 0; |
| 712 | return .{ |
| 713 | .base_name = name[0..base_name_end], |
| 714 | .file_ext = if (file_ext.len > 0) file_ext[1..] else file_ext, |
| 715 | .test_index = index, |
| 716 | }; |
| 717 | } |
| 718 | |
| 719 | /// Sort test filenames in-place, so that incremental test cases ("foo.1.zig", |
| 720 | /// "foo.2.zig", etc.) are contiguous and appear in numerical order. |
| 721 | fn sortTestFilenames( |
| 722 | filenames: [][]const u8, |
| 723 | ) void { |
| 724 | const Context = struct { |
| 725 | pub fn lessThan(_: @This(), a: []const u8, b: []const u8) bool { |
| 726 | const a_parts = getTestFileNameParts(a); |
| 727 | const b_parts = getTestFileNameParts(b); |
| 728 | |
| 729 | // Sort "<base_name>.X.<file_ext>" based on "<base_name>" and "<file_ext>" first |
| 730 | return switch (std.mem.order(u8, a_parts.base_name, b_parts.base_name)) { |
| 731 | .lt => true, |
| 732 | .gt => false, |
| 733 | .eq => switch (std.mem.order(u8, a_parts.file_ext, b_parts.file_ext)) { |
| 734 | .lt => true, |
| 735 | .gt => false, |
| 736 | .eq => b: { // a and b differ only in their ".X" part |
| 737 | |
| 738 | // Sort "<base_name>.<file_ext>" before any "<base_name>.X.<file_ext>" |
| 739 | if (a_parts.test_index == null) break :b true; |
| 740 | if (b_parts.test_index == null) break :b false; |
| 741 | |
| 742 | // Make sure that incremental tests appear in linear order |
| 743 | return a_parts.test_index.? < b_parts.test_index.?; |
| 744 | }, |
| 745 | }, |
| 746 | }; |
| 747 | } |
| 748 | }; |
| 749 | std.sort.sort([]const u8, filenames, Context{}, Context.lessThan); |
| 750 | } |
| 751 | |
| 684 | 752 | fn addErrorCasesFromDirInner( |
| 685 | 753 | ctx: *TestContext, |
| 686 | 754 | name: []const u8, |
| ... | ... | @@ -696,6 +764,9 @@ pub const TestContext = struct { |
| 696 | 764 | var opt_case: ?*Case = null; |
| 697 | 765 | |
| 698 | 766 | var it = dir.iterate(); |
| 767 | var filenames = std.ArrayList([]const u8).init(ctx.arena); |
| 768 | defer filenames.deinit(); |
| 769 | |
| 699 | 770 | while (try it.next()) |entry| { |
| 700 | 771 | if (entry.kind != .File) continue; |
| 701 | 772 | |
| ... | ... | @@ -704,11 +775,46 @@ pub const TestContext = struct { |
| 704 | 775 | .unknown => continue, |
| 705 | 776 | else => {}, |
| 706 | 777 | } |
| 778 | try filenames.append(try ctx.arena.dupe(u8, entry.name)); |
| 779 | } |
| 707 | 780 | |
| 708 | | current_file.* = try ctx.arena.dupe(u8, entry.name); |
| 781 | // Sort filenames, so that incremental tests are contiguous and in-order |
| 782 | sortTestFilenames(filenames.items); |
| 783 | |
| 784 | var prev_filename: []const u8 = ""; |
| 785 | for (filenames.items) |filename| { |
| 786 | current_file.* = filename; |
| 787 | |
| 788 | { // First, check if this file is part of an incremental update sequence |
| 789 | |
| 790 | // Split filename into "<base_name>.<index>.<file_ext>" |
| 791 | const prev_parts = getTestFileNameParts(prev_filename); |
| 792 | const new_parts = getTestFileNameParts(filename); |
| 793 | |
| 794 | // If base_name and file_ext match, these files are in the same test sequence |
| 795 | // and the new one should be the incremented version of the previous test |
| 796 | if (std.mem.eql(u8, prev_parts.base_name, new_parts.base_name) and |
| 797 | std.mem.eql(u8, prev_parts.file_ext, new_parts.file_ext)) |
| 798 | { |
| 799 | |
| 800 | // This is "foo.X.zig" followed by "foo.Y.zig". Make sure that X = Y + 1 |
| 801 | if (prev_parts.test_index == null) return error.InvalidIncrementalTestIndex; |
| 802 | if (new_parts.test_index == null) return error.InvalidIncrementalTestIndex; |
| 803 | if (new_parts.test_index.? != prev_parts.test_index.? + 1) return error.InvalidIncrementalTestIndex; |
| 804 | } else { |
| 805 | |
| 806 | // This is not the same test sequence, so the new file must be the first file |
| 807 | // in a new sequence ("*.1.zig") or an independent test file ("*.zig") |
| 808 | if (new_parts.test_index != null and new_parts.test_index.? != 1) return error.InvalidIncrementalTestIndex; |
| 809 | |
| 810 | if (strategy == .independent) |
| 811 | opt_case = null; // Generate a new independent test case for this update |
| 812 | } |
| 813 | } |
| 814 | prev_filename = filename; |
| 709 | 815 | |
| 710 | 816 | const max_file_size = 10 * 1024 * 1024; |
| 711 | | const src = try dir.readFileAllocOptions(ctx.arena, entry.name, max_file_size, null, 1, 0); |
| 817 | const src = try dir.readFileAllocOptions(ctx.arena, filename, max_file_size, null, 1, 0); |
| 712 | 818 | |
| 713 | 819 | // The manifest is the last contiguous block of comments in the file |
| 714 | 820 | // We scan for the beginning by searching backward for the first non-empty line that does not start with "//" |
| ... | ... | @@ -720,14 +826,17 @@ pub const TestContext = struct { |
| 720 | 826 | // Move to beginning of line |
| 721 | 827 | while (cursor > 0 and src[cursor - 1] != '\n') cursor -= 1; |
| 722 | 828 | |
| 723 | | // Check if line is non-empty and does not start with "//" |
| 724 | | if (cursor + 1 < src.len and src[cursor + 1] != '\n' and src[cursor + 1] != '\r') { |
| 725 | | if (std.mem.startsWith(u8, src[cursor..], "//")) { |
| 726 | | manifest_start = cursor; |
| 727 | | } else { |
| 728 | | break; |
| 729 | | } |
| 730 | | } else manifest_end = cursor; |
| 829 | if (std.mem.startsWith(u8, src[cursor..], "//")) { |
| 830 | manifest_start = cursor; // Contiguous comment line, include in manifest |
| 831 | } else { |
| 832 | if (manifest_start != null) break; // Encountered non-comment line, end of manifest |
| 833 | |
| 834 | // We ignore all-whitespace lines following the comment block, but anything else |
| 835 | // means that there is no manifest present. |
| 836 | if (std.mem.trim(u8, src[cursor..manifest_end], " \r\n\t").len == 0) { |
| 837 | manifest_end = cursor; |
| 838 | } else break; // If it's not whitespace, there is no manifest |
| 839 | } |
| 731 | 840 | |
| 732 | 841 | // Move to previous line |
| 733 | 842 | if (cursor != 0) cursor -= 1 else break; |
| ... | ... | @@ -738,6 +847,7 @@ pub const TestContext = struct { |
| 738 | 847 | |
| 739 | 848 | if (manifest_start) |start| { |
| 740 | 849 | // Due to the above processing, we know that this is a contiguous block of comments |
| 850 | // and do not need to re-validate the leading "//" on each line |
| 741 | 851 | var manifest_it = std.mem.tokenize(u8, src[start..manifest_end], "\r\n"); |
| 742 | 852 | |
| 743 | 853 | // First line is the test case name |
| ... | ... | @@ -770,7 +880,6 @@ pub const TestContext = struct { |
| 770 | 880 | .independent => { |
| 771 | 881 | case.name = case_name; |
| 772 | 882 | case.addError(src, errors.items); |
| 773 | | opt_case = null; |
| 774 | 883 | }, |
| 775 | 884 | .incremental => { |
| 776 | 885 | case.addErrorNamed(case_name, src, errors.items); |
| ... | ... | @@ -1130,7 +1239,7 @@ pub const TestContext = struct { |
| 1130 | 1239 | if (all_errors.list.len != 0) { |
| 1131 | 1240 | print( |
| 1132 | 1241 | "\nCase '{s}': unexpected errors at update_index={d}:\n{s}\n", |
| 1133 | | .{ case.name, update_index, hr }, |
| 1242 | .{ case.name, update_index + 1, hr }, |
| 1134 | 1243 | ); |
| 1135 | 1244 | for (all_errors.list) |err_msg| { |
| 1136 | 1245 | switch (err_msg) { |
| ... | ... | @@ -1292,7 +1401,7 @@ pub const TestContext = struct { |
| 1292 | 1401 | } |
| 1293 | 1402 | |
| 1294 | 1403 | if (any_failed) { |
| 1295 | | print("\nupdate_index={d} ", .{update_index}); |
| 1404 | print("\nupdate_index={d}\n", .{update_index + 1}); |
| 1296 | 1405 | return error.WrongCompileErrors; |
| 1297 | 1406 | } |
| 1298 | 1407 | }, |
| ... | ... | @@ -1399,7 +1508,7 @@ pub const TestContext = struct { |
| 1399 | 1508 | .cwd = tmp_dir_path, |
| 1400 | 1509 | }) catch |err| { |
| 1401 | 1510 | print("\nupdate_index={d} The following command failed with {s}:\n", .{ |
| 1402 | | update_index, @errorName(err), |
| 1511 | update_index + 1, @errorName(err), |
| 1403 | 1512 | }); |
| 1404 | 1513 | dumpArgs(argv.items); |
| 1405 | 1514 | return error.ChildProcessExecution; |
Andrew Kelley <andrew@ziglang.org> 2022-04-12 06:27:09-04:00
GitHub <noreply@github.com> 2022-04-12 06:27:09-04:00
Signed by PGP key 4AEE18F83AFDEB23
src