| ... | @@ -21,9 +21,10 @@ const ErrorMsg = struct { | ... | @@ -21,9 +21,10 @@ const ErrorMsg = struct { |
| 21 | }; | 21 | }; |
| 22 | | 22 | |
| 23 | pub const TestContext = struct { | 23 | pub const TestContext = struct { |
| 24 | zir_cases: std.ArrayList(Case), | 24 | /// TODO: find a way to treat cases as individual tests (shouldn't show "1 test passed" if there are 200 cases) |
| | 25 | cases: std.ArrayList(Case), |
| 25 | | 26 | |
| 26 | pub const ZIRUpdate = struct { | 27 | pub const Update = struct { |
| 27 | /// The input to the current update. We simulate an incremental update | 28 | /// The input to the current update. We simulate an incremental update |
| 28 | /// with the file's contents changed to this value each update. | 29 | /// with the file's contents changed to this value each update. |
| 29 | /// | 30 | /// |
| ... | @@ -40,67 +41,70 @@ pub const TestContext = struct { | ... | @@ -40,67 +41,70 @@ pub const TestContext = struct { |
| 40 | /// fails to compile, and for the expected reasons. | 41 | /// fails to compile, and for the expected reasons. |
| 41 | /// A slice containing the expected errors *in sequential order*. | 42 | /// A slice containing the expected errors *in sequential order*. |
| 42 | Error: []const ErrorMsg, | 43 | Error: []const ErrorMsg, |
| 43 | /// An execution update compiles and runs the input ZIR, feeding in | 44 | /// An execution update compiles and runs the input, testing the |
| 44 | /// provided input and ensuring that the stdout match what is expected. | 45 | /// stdout against the expected results |
| 45 | Execution: []const u8, | 46 | Execution: []const u8, |
| 46 | }, | 47 | }, |
| 47 | }; | 48 | }; |
| 48 | | 49 | |
| 49 | /// A Case consists of a set of *updates*. A update can transform ZIR, | 50 | pub const TestType = enum { |
| 50 | /// compile it, ensure that compilation fails, and more. The same Module is | 51 | Zig, |
| 51 | /// used for each update, so each update's source is treated as a single file | 52 | ZIR, |
| 52 | /// being updated by the test harness and incrementally compiled. | 53 | }; |
| | 54 | |
| | 55 | /// A Case consists of a set of *updates*. The same Module is used for each |
| | 56 | /// update, so each update's source is treated as a single file being |
| | 57 | /// updated by the test harness and incrementally compiled. |
| 53 | pub const Case = struct { | 58 | pub const Case = struct { |
| 54 | name: []const u8, | 59 | name: []const u8, |
| 55 | /// The platform the ZIR targets. For non-native platforms, an emulator | 60 | /// The platform the test targets. For non-native platforms, an emulator |
| 56 | /// such as QEMU is required for tests to complete. | 61 | /// such as QEMU is required for tests to complete. |
| 57 | target: std.zig.CrossTarget, | 62 | target: std.zig.CrossTarget, |
| 58 | updates: std.ArrayList(ZIRUpdate), | | |
| 59 | output_mode: std.builtin.OutputMode, | 63 | output_mode: std.builtin.OutputMode, |
| 60 | /// Either ".zir" or ".zig" | 64 | updates: std.ArrayList(Update), |
| 61 | extension: [4]u8, | 65 | @"type": TestType, |
| 62 | | 66 | |
| 63 | /// Adds a subcase in which the module is updated with new ZIR, and the | 67 | /// Adds a subcase in which the module is updated with new ZIR, and the |
| 64 | /// resulting ZIR is validated. | 68 | /// resulting ZIR is validated. |
| 65 | pub fn addTransform(self: *Case, src: [:0]const u8, result: [:0]const u8) void { | 69 | pub fn addTransform(self: *Case, src: [:0]const u8, result: [:0]const u8) !void { |
| 66 | self.updates.append(.{ | 70 | try self.updates.append(.{ |
| 67 | .src = src, | 71 | .src = src, |
| 68 | .case = .{ .Transformation = result }, | 72 | .case = .{ .Transformation = result }, |
| 69 | }) catch unreachable; | 73 | }); |
| 70 | } | 74 | } |
| 71 | | 75 | |
| 72 | pub fn addCompareOutput(self: *Case, src: [:0]const u8, result: []const u8) void { | 76 | pub fn addCompareOutput(self: *Case, src: [:0]const u8, result: []const u8) !void { |
| 73 | self.updates.append(.{ | 77 | try self.updates.append(.{ |
| 74 | .src = src, | 78 | .src = src, |
| 75 | .case = .{ .Execution = result }, | 79 | .case = .{ .Execution = result }, |
| 76 | }) catch unreachable; | 80 | }); |
| 77 | } | 81 | } |
| 78 | | 82 | |
| 79 | /// Adds a subcase in which the module is updated with invalid ZIR, and | 83 | /// Adds a subcase in which the module is updated with invalid ZIR, and |
| 80 | /// ensures that compilation fails for the expected reasons. | 84 | /// ensures that compilation fails for the expected reasons. |
| 81 | /// | 85 | /// |
| 82 | /// Errors must be specified in sequential order. | 86 | /// Errors must be specified in sequential order. |
| 83 | pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) void { | 87 | pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) !void { |
| 84 | var array = self.updates.allocator.alloc(ErrorMsg, errors.len) catch unreachable; | 88 | var array = try self.updates.allocator.alloc(ErrorMsg, errors.len); |
| 85 | for (errors) |e, i| { | 89 | for (errors) |e, i| { |
| 86 | if (e[0] != ':') { | 90 | if (e[0] != ':') { |
| 87 | std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{}); | 91 | @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n"); |
| 88 | } | 92 | } |
| 89 | var cur = e[1..]; | 93 | var cur = e[1..]; |
| 90 | var line_index = std.mem.indexOf(u8, cur, ":"); | 94 | var line_index = std.mem.indexOf(u8, cur, ":"); |
| 91 | if (line_index == null) { | 95 | if (line_index == null) { |
| 92 | std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{}); | 96 | @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n"); |
| 93 | } | 97 | } |
| 94 | const line = std.fmt.parseInt(u32, cur[0..line_index.?], 10) catch @panic("Unable to parse line number"); | 98 | const line = std.fmt.parseInt(u32, cur[0..line_index.?], 10) catch @panic("Unable to parse line number"); |
| 95 | cur = cur[line_index.? + 1 ..]; | 99 | cur = cur[line_index.? + 1 ..]; |
| 96 | const column_index = std.mem.indexOf(u8, cur, ":"); | 100 | const column_index = std.mem.indexOf(u8, cur, ":"); |
| 97 | if (column_index == null) { | 101 | if (column_index == null) { |
| 98 | std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{}); | 102 | @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n"); |
| 99 | } | 103 | } |
| 100 | const column = std.fmt.parseInt(u32, cur[0..column_index.?], 10) catch @panic("Unable to parse column number"); | 104 | const column = std.fmt.parseInt(u32, cur[0..column_index.?], 10) catch @panic("Unable to parse column number"); |
| 101 | cur = cur[column_index.? + 2 ..]; | 105 | cur = cur[column_index.? + 2 ..]; |
| 102 | if (!std.mem.eql(u8, cur[0..7], "error: ")) { | 106 | if (!std.mem.eql(u8, cur[0..7], "error: ")) { |
| 103 | std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{}); | 107 | @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n"); |
| 104 | } | 108 | } |
| 105 | const msg = cur[7..]; | 109 | const msg = cur[7..]; |
| 106 | | 110 | |
| ... | @@ -114,125 +118,87 @@ pub const TestContext = struct { | ... | @@ -114,125 +118,87 @@ pub const TestContext = struct { |
| 114 | .column = column - 1, | 118 | .column = column - 1, |
| 115 | }; | 119 | }; |
| 116 | } | 120 | } |
| 117 | self.updates.append(.{ .src = src, .case = .{ .Error = array } }) catch unreachable; | 121 | try self.updates.append(.{ .src = src, .case = .{ .Error = array } }); |
| 118 | } | 122 | } |
| 119 | }; | 123 | }; |
| 120 | | 124 | |
| 121 | pub fn addExeZIR( | | |
| 122 | ctx: *TestContext, | | |
| 123 | name: []const u8, | | |
| 124 | target: std.zig.CrossTarget, | | |
| 125 | ) *Case { | | |
| 126 | const case = Case{ | | |
| 127 | .name = name, | | |
| 128 | .target = target, | | |
| 129 | .updates = std.ArrayList(ZIRUpdate).init(ctx.zir_cases.allocator), | | |
| 130 | .output_mode = .Exe, | | |
| 131 | .extension = ".zir".*, | | |
| 132 | }; | | |
| 133 | ctx.zir_cases.append(case) catch unreachable; | | |
| 134 | return &ctx.zir_cases.items[ctx.zir_cases.items.len - 1]; | | |
| 135 | } | | |
| 136 | | | |
| 137 | pub fn addObjZIR( | | |
| 138 | ctx: *TestContext, | | |
| 139 | name: []const u8, | | |
| 140 | target: std.zig.CrossTarget, | | |
| 141 | ) *Case { | | |
| 142 | const case = Case{ | | |
| 143 | .name = name, | | |
| 144 | .target = target, | | |
| 145 | .updates = std.ArrayList(ZIRUpdate).init(ctx.zir_cases.allocator), | | |
| 146 | .output_mode = .Obj, | | |
| 147 | .extension = ".zir".*, | | |
| 148 | }; | | |
| 149 | ctx.zir_cases.append(case) catch unreachable; | | |
| 150 | return &ctx.zir_cases.items[ctx.zir_cases.items.len - 1]; | | |
| 151 | } | | |
| 152 | | | |
| 153 | pub fn addExe( | 125 | pub fn addExe( |
| 154 | ctx: *TestContext, | 126 | ctx: *TestContext, |
| 155 | name: []const u8, | 127 | name: []const u8, |
| 156 | target: std.zig.CrossTarget, | 128 | target: std.zig.CrossTarget, |
| 157 | ) *Case { | 129 | T: TestType, |
| | 130 | ) !*Case { |
| 158 | const case = Case{ | 131 | const case = Case{ |
| 159 | .name = name, | 132 | .name = name, |
| 160 | .target = target, | 133 | .target = target, |
| 161 | .updates = std.ArrayList(ZIRUpdate).init(ctx.zir_cases.allocator), | 134 | .updates = std.ArrayList(Update).init(ctx.cases.allocator), |
| 162 | .output_mode = .Exe, | 135 | .output_mode = .Exe, |
| 163 | .extension = ".zig".*, | 136 | .@"type" = T, |
| 164 | }; | 137 | }; |
| 165 | ctx.zir_cases.append(case) catch unreachable; | 138 | try ctx.cases.append(case); |
| 166 | return &ctx.zir_cases.items[ctx.zir_cases.items.len - 1]; | 139 | return &ctx.cases.items[ctx.cases.items.len - 1]; |
| 167 | } | 140 | } |
| 168 | | 141 | |
| 169 | pub fn addObj( | 142 | pub fn addObj( |
| 170 | ctx: *TestContext, | 143 | ctx: *TestContext, |
| 171 | name: []const u8, | 144 | name: []const u8, |
| 172 | target: std.zig.CrossTarget, | 145 | target: std.zig.CrossTarget, |
| 173 | ) *Case { | 146 | T: TestType, |
| 174 | const case = Case{ | 147 | ) !*Case { |
| | 148 | try ctx.cases.append(Case{ |
| 175 | .name = name, | 149 | .name = name, |
| 176 | .target = target, | 150 | .target = target, |
| 177 | .updates = std.ArrayList(ZIRUpdate).init(ctx.zir_cases.allocator), | 151 | .updates = std.ArrayList(Update).init(ctx.cases.allocator), |
| 178 | .output_mode = .Obj, | 152 | .output_mode = .Obj, |
| 179 | .extension = ".zig".*, | 153 | .@"type" = T, |
| 180 | }; | 154 | }); |
| 181 | ctx.zir_cases.append(case) catch unreachable; | 155 | return &ctx.cases.items[ctx.cases.items.len - 1]; |
| 182 | return &ctx.zir_cases.items[ctx.zir_cases.items.len - 1]; | | |
| 183 | } | | |
| 184 | | | |
| 185 | pub fn addZIRCompareOutput( | | |
| 186 | ctx: *TestContext, | | |
| 187 | name: []const u8, | | |
| 188 | src: [:0]const u8, | | |
| 189 | expected_stdout: []const u8, | | |
| 190 | ) void { | | |
| 191 | var c = ctx.addExeZIR(name, .{}); | | |
| 192 | c.addCompareOutput(src, expected_stdout); | | |
| 193 | } | 156 | } |
| 194 | | 157 | |
| 195 | pub fn addCompareOutput( | 158 | pub fn addCompareOutput( |
| 196 | ctx: *TestContext, | 159 | ctx: *TestContext, |
| 197 | name: []const u8, | 160 | name: []const u8, |
| | 161 | T: TestType, |
| 198 | src: [:0]const u8, | 162 | src: [:0]const u8, |
| 199 | expected_stdout: []const u8, | 163 | expected_stdout: []const u8, |
| 200 | ) void { | 164 | ) !void { |
| 201 | var c = ctx.addExe(name, .{}); | 165 | var c = try ctx.addExe(name, .{}, T); |
| 202 | c.addCompareOutput(src, expected_stdout); | 166 | try c.addCompareOutput(src, expected_stdout); |
| 203 | } | 167 | } |
| 204 | | 168 | |
| 205 | pub fn addZIRTransform( | 169 | pub fn addTransform( |
| 206 | ctx: *TestContext, | 170 | ctx: *TestContext, |
| 207 | name: []const u8, | 171 | name: []const u8, |
| 208 | target: std.zig.CrossTarget, | 172 | target: std.zig.CrossTarget, |
| | 173 | T: TestType, |
| 209 | src: [:0]const u8, | 174 | src: [:0]const u8, |
| 210 | result: [:0]const u8, | 175 | result: [:0]const u8, |
| 211 | ) void { | 176 | ) !void { |
| 212 | var c = ctx.addObjZIR(name, target); | 177 | var c = try ctx.addObj(name, target, T); |
| 213 | c.addTransform(src, result); | 178 | try c.addTransform(src, result); |
| 214 | } | 179 | } |
| 215 | | 180 | |
| 216 | pub fn addZIRError( | 181 | pub fn addError( |
| 217 | ctx: *TestContext, | 182 | ctx: *TestContext, |
| 218 | name: []const u8, | 183 | name: []const u8, |
| 219 | target: std.zig.CrossTarget, | 184 | target: std.zig.CrossTarget, |
| | 185 | T: TestType, |
| 220 | src: [:0]const u8, | 186 | src: [:0]const u8, |
| 221 | expected_errors: []const []const u8, | 187 | expected_errors: []const []const u8, |
| 222 | ) void { | 188 | ) !void { |
| 223 | var c = ctx.addObjZIR(name, target); | 189 | var c = try ctx.addObj(name, target, T); |
| 224 | c.addError(src, expected_errors); | 190 | try c.addError(src, expected_errors); |
| 225 | } | 191 | } |
| 226 | | 192 | |
| 227 | fn init() TestContext { | 193 | fn init() TestContext { |
| 228 | const allocator = std.heap.page_allocator; | 194 | const allocator = std.heap.page_allocator; |
| 229 | return .{ | 195 | return .{ |
| 230 | .zir_cases = std.ArrayList(Case).init(allocator), | 196 | .cases = std.ArrayList(Case).init(allocator), |
| 231 | }; | 197 | }; |
| 232 | } | 198 | } |
| 233 | | 199 | |
| 234 | fn deinit(self: *TestContext) void { | 200 | fn deinit(self: *TestContext) void { |
| 235 | for (self.zir_cases.items) |c| { | 201 | for (self.cases.items) |c| { |
| 236 | for (c.updates.items) |u| { | 202 | for (c.updates.items) |u| { |
| 237 | if (u.case == .Error) { | 203 | if (u.case == .Error) { |
| 238 | c.updates.allocator.free(u.case.Error); | 204 | c.updates.allocator.free(u.case.Error); |
| ... | @@ -240,18 +206,18 @@ pub const TestContext = struct { | ... | @@ -240,18 +206,18 @@ pub const TestContext = struct { |
| 240 | } | 206 | } |
| 241 | c.updates.deinit(); | 207 | c.updates.deinit(); |
| 242 | } | 208 | } |
| 243 | self.zir_cases.deinit(); | 209 | self.cases.deinit(); |
| 244 | self.* = undefined; | 210 | self.* = undefined; |
| 245 | } | 211 | } |
| 246 | | 212 | |
| 247 | fn run(self: *TestContext) !void { | 213 | fn run(self: *TestContext) !void { |
| 248 | var progress = std.Progress{}; | 214 | var progress = std.Progress{}; |
| 249 | const root_node = try progress.start("zir", self.zir_cases.items.len); | 215 | const root_node = try progress.start("tests", self.cases.items.len); |
| 250 | defer root_node.end(); | 216 | defer root_node.end(); |
| 251 | | 217 | |
| 252 | const native_info = try std.zig.system.NativeTargetInfo.detect(std.heap.page_allocator, .{}); | 218 | const native_info = try std.zig.system.NativeTargetInfo.detect(std.heap.page_allocator, .{}); |
| 253 | | 219 | |
| 254 | for (self.zir_cases.items) |case| { | 220 | for (self.cases.items) |case| { |
| 255 | std.testing.base_allocator_instance.reset(); | 221 | std.testing.base_allocator_instance.reset(); |
| 256 | | 222 | |
| 257 | var prg_node = root_node.start(case.name, case.updates.items.len); | 223 | var prg_node = root_node.start(case.name, case.updates.items.len); |
| ... | @@ -267,17 +233,19 @@ pub const TestContext = struct { | ... | @@ -267,17 +233,19 @@ pub const TestContext = struct { |
| 267 | } | 233 | } |
| 268 | } | 234 | } |
| 269 | | 235 | |
| 270 | fn runOneCase(self: *TestContext, allocator: *Allocator, prg_node: *std.Progress.Node, case: Case, target: std.Target) !void { | 236 | fn runOneCase(self: *TestContext, allocator: *Allocator, root_node: *std.Progress.Node, case: Case, target: std.Target) !void { |
| 271 | var tmp = std.testing.tmpDir(.{}); | 237 | var tmp = std.testing.tmpDir(.{}); |
| 272 | defer tmp.cleanup(); | 238 | defer tmp.cleanup(); |
| 273 | | 239 | |
| 274 | const root_name = "test_case"; | 240 | const tmp_src_path = if (case.type == .Zig) "test_case.zig" else if (case.type == .ZIR) "test_case.zir" else unreachable; |
| 275 | const tmp_src_path = try std.fmt.allocPrint(allocator, "{}{}", .{ root_name, case.extension }); | | |
| 276 | defer allocator.free(tmp_src_path); | | |
| 277 | const root_pkg = try Package.create(allocator, tmp.dir, ".", tmp_src_path); | 241 | const root_pkg = try Package.create(allocator, tmp.dir, ".", tmp_src_path); |
| 278 | defer root_pkg.destroy(); | 242 | defer root_pkg.destroy(); |
| 279 | | 243 | |
| 280 | const bin_name = try std.zig.binNameAlloc(allocator, root_name, target, case.output_mode, null); | 244 | var prg_node = root_node.start(case.name, case.updates.items.len); |
| | 245 | prg_node.activate(); |
| | 246 | defer prg_node.end(); |
| | 247 | |
| | 248 | const bin_name = try std.zig.binNameAlloc(allocator, "test_case", target, case.output_mode, null); |
| 281 | defer allocator.free(bin_name); | 249 | defer allocator.free(bin_name); |
| 282 | | 250 | |
| 283 | var module = try Module.init(allocator, .{ | 251 | var module = try Module.init(allocator, .{ |