authorgravatar for joachim.schmidt557@outlook.comJoachim Schmidt <joachim.schmidt557@outlook.com> 2022-11-02 09:59:15+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-02 14:10:52-07:00
log99b1eaaac47567727f2ac76c9243dd7311177a5e
treee3f4c8cbea4e9f07ad04c579cee7d9a14c451ad6
parent54bdc0108a38cccbcfd42030ff4f899686a19fc3

Merge pull request #13404 from joachimschmidt557/stage2-aarch64

stage2 aarch64: enable printing test results in the test runner

5 files changed, 521 insertions(+), 112 deletions(-)

lib/test_runner.zig+1
...@@ -130,6 +130,7 @@ pub fn main2() anyerror!void {...@@ -130,6 +130,7 @@ pub fn main2() anyerror!void {
130 }130 }
131 if (builtin.zig_backend == .stage2_wasm or131 if (builtin.zig_backend == .stage2_wasm or
132 builtin.zig_backend == .stage2_x86_64 or132 builtin.zig_backend == .stage2_x86_64 or
133 builtin.zig_backend == .stage2_aarch64 or
133 builtin.zig_backend == .stage2_llvm or134 builtin.zig_backend == .stage2_llvm or
134 builtin.zig_backend == .stage2_c)135 builtin.zig_backend == .stage2_c)
135 {136 {
src/arch/aarch64/CodeGen.zig+462-56
...@@ -91,7 +91,7 @@ register_manager: RegisterManager = .{},...@@ -91,7 +91,7 @@ register_manager: RegisterManager = .{},
91/// Maps offset to what is stored there.91/// Maps offset to what is stored there.
92stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},92stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
93/// Tracks the current instruction allocated to the compare flags93/// Tracks the current instruction allocated to the compare flags
94condition_flags_inst: ?Air.Inst.Index = null,94compare_flags_inst: ?Air.Inst.Index = null,
9595
96/// Offset from the stack base, representing the end of the stack frame.96/// Offset from the stack base, representing the end of the stack frame.
97max_end_stack: u32 = 0,97max_end_stack: u32 = 0,
...@@ -154,7 +154,7 @@ const MCValue = union(enum) {...@@ -154,7 +154,7 @@ const MCValue = union(enum) {
154 /// The value resides in the N, Z, C, V flags. The value is 1 (if154 /// The value resides in the N, Z, C, V flags. The value is 1 (if
155 /// the type is u1) or true (if the type in bool) iff the155 /// the type is u1) or true (if the type in bool) iff the
156 /// specified condition is true.156 /// specified condition is true.
157 condition_flags: Condition,157 compare_flags: Condition,
158 /// The value is a function argument passed via the stack.158 /// The value is a function argument passed via the stack.
159 stack_argument_offset: u32,159 stack_argument_offset: u32,
160};160};
...@@ -201,6 +201,29 @@ const BigTomb = struct {...@@ -201,6 +201,29 @@ const BigTomb = struct {
201 log.debug("%{d} => {}", .{ bt.inst, result });201 log.debug("%{d} => {}", .{ bt.inst, result });
202 const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];202 const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];
203 branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);203 branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);
204
205 switch (result) {
206 .register => |reg| {
207 // In some cases (such as bitcast), an operand
208 // may be the same MCValue as the result. If
209 // that operand died and was a register, it
210 // was freed by processDeath. We have to
211 // "re-allocate" the register.
212 if (bt.function.register_manager.isRegFree(reg)) {
213 bt.function.register_manager.getRegAssumeFree(reg, bt.inst);
214 }
215 },
216 .register_with_overflow => |rwo| {
217 if (bt.function.register_manager.isRegFree(rwo.reg)) {
218 bt.function.register_manager.getRegAssumeFree(rwo.reg, bt.inst);
219 }
220 bt.function.compare_flags_inst = bt.inst;
221 },
222 .compare_flags => |_| {
223 bt.function.compare_flags_inst = bt.inst;
224 },
225 else => {},
226 }
204 }227 }
205 bt.function.finishAirBookkeeping();228 bt.function.finishAirBookkeeping();
206 }229 }
...@@ -539,8 +562,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -539,8 +562,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
539 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),562 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
540 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),563 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
541564
542 .min => try self.airMin(inst),565 .min => try self.airMinMax(inst),
543 .max => try self.airMax(inst),566 .max => try self.airMinMax(inst),
544567
545 .add_sat => try self.airAddSat(inst),568 .add_sat => try self.airAddSat(inst),
546 .sub_sat => try self.airSubSat(inst),569 .sub_sat => try self.airSubSat(inst),
...@@ -764,10 +787,10 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void {...@@ -764,10 +787,10 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void {
764 },787 },
765 .register_with_overflow => |rwo| {788 .register_with_overflow => |rwo| {
766 self.register_manager.freeReg(rwo.reg);789 self.register_manager.freeReg(rwo.reg);
767 self.condition_flags_inst = null;790 self.compare_flags_inst = null;
768 },791 },
769 .condition_flags => {792 .compare_flags => {
770 self.condition_flags_inst = null;793 self.compare_flags_inst = null;
771 },794 },
772 else => {}, // TODO process stack allocation death795 else => {}, // TODO process stack allocation death
773 }796 }
...@@ -808,6 +831,15 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live...@@ -808,6 +831,15 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live
808 self.register_manager.getRegAssumeFree(reg, inst);831 self.register_manager.getRegAssumeFree(reg, inst);
809 }832 }
810 },833 },
834 .register_with_overflow => |rwo| {
835 if (self.register_manager.isRegFree(rwo.reg)) {
836 self.register_manager.getRegAssumeFree(rwo.reg, inst);
837 }
838 self.compare_flags_inst = inst;
839 },
840 .compare_flags => |_| {
841 self.compare_flags_inst = inst;
842 },
811 else => {},843 else => {},
812 }844 }
813 }845 }
...@@ -931,11 +963,11 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void...@@ -931,11 +963,11 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
931/// Save the current instruction stored in the compare flags if963/// Save the current instruction stored in the compare flags if
932/// occupied964/// occupied
933fn spillCompareFlagsIfOccupied(self: *Self) !void {965fn spillCompareFlagsIfOccupied(self: *Self) !void {
934 if (self.condition_flags_inst) |inst_to_save| {966 if (self.compare_flags_inst) |inst_to_save| {
935 const ty = self.air.typeOfIndex(inst_to_save);967 const ty = self.air.typeOfIndex(inst_to_save);
936 const mcv = self.getResolvedInstValue(inst_to_save);968 const mcv = self.getResolvedInstValue(inst_to_save);
937 const new_mcv = switch (mcv) {969 const new_mcv = switch (mcv) {
938 .condition_flags => try self.allocRegOrMem(ty, true, inst_to_save),970 .compare_flags => try self.allocRegOrMem(ty, true, inst_to_save),
939 .register_with_overflow => try self.allocRegOrMem(ty, false, inst_to_save),971 .register_with_overflow => try self.allocRegOrMem(ty, false, inst_to_save),
940 else => unreachable, // mcv doesn't occupy the compare flags972 else => unreachable, // mcv doesn't occupy the compare flags
941 };973 };
...@@ -946,7 +978,7 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void {...@@ -946,7 +978,7 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void {
946 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];978 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
947 try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);979 try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
948980
949 self.condition_flags_inst = null;981 self.compare_flags_inst = null;
950982
951 // TODO consolidate with register manager and spillInstruction983 // TODO consolidate with register manager and spillInstruction
952 // this call should really belong in the register manager!984 // this call should really belong in the register manager!
...@@ -984,8 +1016,27 @@ fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {...@@ -984,8 +1016,27 @@ fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
984}1016}
9851017
986fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {1018fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
987 const stack_offset = try self.allocMemPtr(inst);1019 const result: MCValue = switch (self.ret_mcv) {
988 return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });1020 .none, .register => .{ .ptr_stack_offset = try self.allocMemPtr(inst) },
1021 .stack_offset => blk: {
1022 // self.ret_mcv is an address to where this function
1023 // should store its result into
1024 const ret_ty = self.fn_type.fnReturnType();
1025 var ptr_ty_payload: Type.Payload.ElemType = .{
1026 .base = .{ .tag = .single_mut_pointer },
1027 .data = ret_ty,
1028 };
1029 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
1030
1031 // addr_reg will contain the address of where to store the
1032 // result into
1033 const addr_reg = try self.copyToTmpRegister(ptr_ty, self.ret_mcv);
1034 break :blk .{ .register = addr_reg };
1035 },
1036 else => unreachable, // invalid return result
1037 };
1038
1039 return self.finishAir(inst, result, .{ .none, .none, .none });
989}1040}
9901041
991fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {1042fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
...@@ -1155,7 +1206,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {...@@ -1155,7 +1206,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1155 switch (operand) {1206 switch (operand) {
1156 .dead => unreachable,1207 .dead => unreachable,
1157 .unreach => unreachable,1208 .unreach => unreachable,
1158 .condition_flags => |cond| break :result MCValue{ .condition_flags = cond.negate() },1209 .compare_flags => |cond| break :result MCValue{ .compare_flags = cond.negate() },
1159 else => {1210 else => {
1160 switch (operand_ty.zigTypeTag()) {1211 switch (operand_ty.zigTypeTag()) {
1161 .Bool => {1212 .Bool => {
...@@ -1234,15 +1285,102 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {...@@ -1234,15 +1285,102 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1234 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });1285 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1235}1286}
12361287
1237fn airMin(self: *Self, inst: Air.Inst.Index) !void {1288fn minMax(
1238 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1289 self: *Self,
1239 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement min for {}", .{self.target.cpu.arch});1290 tag: Air.Inst.Tag,
1240 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1291 lhs_bind: ReadArg.Bind,
1292 rhs_bind: ReadArg.Bind,
1293 lhs_ty: Type,
1294 rhs_ty: Type,
1295 maybe_inst: ?Air.Inst.Index,
1296) !MCValue {
1297 switch (lhs_ty.zigTypeTag()) {
1298 .Float => return self.fail("TODO ARM min/max on floats", .{}),
1299 .Vector => return self.fail("TODO ARM min/max on vectors", .{}),
1300 .Int => {
1301 const mod = self.bin_file.options.module.?;
1302 assert(lhs_ty.eql(rhs_ty, mod));
1303 const int_info = lhs_ty.intInfo(self.target.*);
1304 if (int_info.bits <= 64) {
1305 var lhs_reg: Register = undefined;
1306 var rhs_reg: Register = undefined;
1307 var dest_reg: Register = undefined;
1308
1309 const read_args = [_]ReadArg{
1310 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
1311 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
1312 };
1313 const write_args = [_]WriteArg{
1314 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1315 };
1316 try self.allocRegs(
1317 &read_args,
1318 &write_args,
1319 if (maybe_inst) |inst| .{
1320 .corresponding_inst = inst,
1321 .operand_mapping = &.{ 0, 1 },
1322 } else null,
1323 );
1324
1325 // lhs == reg should have been checked by airMinMax
1326 assert(lhs_reg != rhs_reg); // see note above
1327
1328 _ = try self.addInst(.{
1329 .tag = .cmp_shifted_register,
1330 .data = .{ .rr_imm6_shift = .{
1331 .rn = lhs_reg,
1332 .rm = rhs_reg,
1333 .imm6 = 0,
1334 .shift = .lsl,
1335 } },
1336 });
1337
1338 const cond_choose_lhs: Condition = switch (tag) {
1339 .max => switch (int_info.signedness) {
1340 .signed => Condition.gt,
1341 .unsigned => Condition.hi,
1342 },
1343 .min => switch (int_info.signedness) {
1344 .signed => Condition.lt,
1345 .unsigned => Condition.cc,
1346 },
1347 else => unreachable,
1348 };
1349
1350 _ = try self.addInst(.{
1351 .tag = .csel,
1352 .data = .{ .rrr_cond = .{
1353 .rd = dest_reg,
1354 .rn = lhs_reg,
1355 .rm = rhs_reg,
1356 .cond = cond_choose_lhs,
1357 } },
1358 });
1359
1360 return MCValue{ .register = dest_reg };
1361 } else {
1362 return self.fail("TODO ARM min/max on integers > u32/i32", .{});
1363 }
1364 },
1365 else => unreachable,
1366 }
1241}1367}
12421368
1243fn airMax(self: *Self, inst: Air.Inst.Index) !void {1369fn airMinMax(self: *Self, inst: Air.Inst.Index) !void {
1370 const tag = self.air.instructions.items(.tag)[inst];
1244 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1371 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1245 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement max for {}", .{self.target.cpu.arch});1372 const lhs_ty = self.air.typeOf(bin_op.lhs);
1373 const rhs_ty = self.air.typeOf(bin_op.rhs);
1374
1375 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1376 const lhs_bind: ReadArg.Bind = .{ .inst = bin_op.lhs };
1377 const rhs_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
1378
1379 const lhs = try self.resolveInst(bin_op.lhs);
1380 if (bin_op.lhs == bin_op.rhs) break :result lhs;
1381
1382 break :result try self.minMax(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst);
1383 };
1246 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1384 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1247}1385}
12481386
...@@ -1477,9 +1615,9 @@ fn allocRegs(...@@ -1477,9 +1615,9 @@ fn allocRegs(
1477 // If the previous MCValue occupied some space we track, we1615 // If the previous MCValue occupied some space we track, we
1478 // need to make sure it is marked as free now.1616 // need to make sure it is marked as free now.
1479 switch (mcv) {1617 switch (mcv) {
1480 .condition_flags => {1618 .compare_flags => {
1481 assert(self.condition_flags_inst.? == inst);1619 assert(self.compare_flags_inst.? == inst);
1482 self.condition_flags_inst = null;1620 self.compare_flags_inst = null;
1483 },1621 },
1484 .register => |prev_reg| {1622 .register => |prev_reg| {
1485 assert(!self.register_manager.isRegFree(prev_reg));1623 assert(!self.register_manager.isRegFree(prev_reg));
...@@ -2276,7 +2414,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2276,7 +2414,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
2276 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);2414 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);
22772415
2278 try self.spillCompareFlagsIfOccupied();2416 try self.spillCompareFlagsIfOccupied();
2279 self.condition_flags_inst = null;2417 self.compare_flags_inst = null;
22802418
2281 const base_tag: Air.Inst.Tag = switch (tag) {2419 const base_tag: Air.Inst.Tag = switch (tag) {
2282 .add_with_overflow => .add,2420 .add_with_overflow => .add,
...@@ -2308,7 +2446,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2308,7 +2446,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
2308 });2446 });
23092447
2310 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });2448 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
2311 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .condition_flags = .ne });2449 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .compare_flags = .ne });
23122450
2313 break :result MCValue{ .stack_offset = stack_offset };2451 break :result MCValue{ .stack_offset = stack_offset };
2314 },2452 },
...@@ -2343,7 +2481,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2343,7 +2481,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
2343 };2481 };
23442482
2345 try self.spillCompareFlagsIfOccupied();2483 try self.spillCompareFlagsIfOccupied();
2346 self.condition_flags_inst = inst;2484 self.compare_flags_inst = inst;
23472485
2348 const dest = blk: {2486 const dest = blk: {
2349 if (rhs_immediate_ok) {2487 if (rhs_immediate_ok) {
...@@ -2452,7 +2590,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2452,7 +2590,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2452 }2590 }
24532591
2454 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });2592 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
2455 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .condition_flags = .ne });2593 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .compare_flags = .ne });
24562594
2457 break :result MCValue{ .stack_offset = stack_offset };2595 break :result MCValue{ .stack_offset = stack_offset };
2458 } else if (int_info.bits <= 64) {2596 } else if (int_info.bits <= 64) {
...@@ -2592,7 +2730,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2592,7 +2730,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2592 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);2730 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
25932731
2594 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });2732 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
2595 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .condition_flags = .ne });2733 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .compare_flags = .ne });
25962734
2597 break :result MCValue{ .stack_offset = stack_offset };2735 break :result MCValue{ .stack_offset = stack_offset };
2598 } else return self.fail("TODO implement mul_with_overflow for integers > u64/i64", .{});2736 } else return self.fail("TODO implement mul_with_overflow for integers > u64/i64", .{});
...@@ -2724,7 +2862,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2724,7 +2862,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2724 });2862 });
27252863
2726 try self.genSetStack(lhs_ty, stack_offset, .{ .register = dest_reg });2864 try self.genSetStack(lhs_ty, stack_offset, .{ .register = dest_reg });
2727 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .condition_flags = .ne });2865 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .compare_flags = .ne });
27282866
2729 break :result MCValue{ .stack_offset = stack_offset };2867 break :result MCValue{ .stack_offset = stack_offset };
2730 } else {2868 } else {
...@@ -2890,7 +3028,23 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {...@@ -2890,7 +3028,23 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2890/// T to E!T3028/// T to E!T
2891fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {3029fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2892 const ty_op = self.air.instructions.items(.data)[inst].ty_op;3030 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2893 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch});3031 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3032 const error_union_ty = self.air.getRefType(ty_op.ty);
3033 const error_ty = error_union_ty.errorUnionSet();
3034 const payload_ty = error_union_ty.errorUnionPayload();
3035 const operand = try self.resolveInst(ty_op.operand);
3036 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) break :result operand;
3037
3038 const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
3039 const abi_align = error_union_ty.abiAlignment(self.target.*);
3040 const stack_offset = try self.allocMem(abi_size, abi_align, inst);
3041 const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
3042 const err_off = errUnionErrorOffset(payload_ty, self.target.*);
3043 try self.genSetStack(payload_ty, stack_offset - @intCast(u32, payload_off), operand);
3044 try self.genSetStack(error_ty, stack_offset - @intCast(u32, err_off), .{ .immediate = 0 });
3045
3046 break :result MCValue{ .stack_offset = stack_offset };
3047 };
2894 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3048 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2895}3049}
28963050
...@@ -2899,11 +3053,20 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2899,11 +3053,20 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2899 const ty_op = self.air.instructions.items(.data)[inst].ty_op;3053 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2900 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {3054 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2901 const error_union_ty = self.air.getRefType(ty_op.ty);3055 const error_union_ty = self.air.getRefType(ty_op.ty);
3056 const error_ty = error_union_ty.errorUnionSet();
2902 const payload_ty = error_union_ty.errorUnionPayload();3057 const payload_ty = error_union_ty.errorUnionPayload();
2903 const mcv = try self.resolveInst(ty_op.operand);3058 const operand = try self.resolveInst(ty_op.operand);
2904 if (!payload_ty.hasRuntimeBits()) break :result mcv;3059 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) break :result operand;
29053060
2906 return self.fail("TODO implement wrap errunion error for non-empty payloads", .{});3061 const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
3062 const abi_align = error_union_ty.abiAlignment(self.target.*);
3063 const stack_offset = try self.allocMem(abi_size, abi_align, inst);
3064 const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
3065 const err_off = errUnionErrorOffset(payload_ty, self.target.*);
3066 try self.genSetStack(error_ty, stack_offset - @intCast(u32, err_off), operand);
3067 try self.genSetStack(payload_ty, stack_offset - @intCast(u32, payload_off), .undef);
3068
3069 break :result MCValue{ .stack_offset = stack_offset };
2907 };3070 };
2908 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3071 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2909}3072}
...@@ -3175,7 +3338,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo...@@ -3175,7 +3338,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
3175 .undef => unreachable,3338 .undef => unreachable,
3176 .unreach => unreachable,3339 .unreach => unreachable,
3177 .dead => unreachable,3340 .dead => unreachable,
3178 .condition_flags,3341 .compare_flags,
3179 .register_with_overflow,3342 .register_with_overflow,
3180 => unreachable, // cannot hold an address3343 => unreachable, // cannot hold an address
3181 .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),3344 .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
...@@ -3187,7 +3350,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo...@@ -3187,7 +3350,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
3187 switch (dst_mcv) {3350 switch (dst_mcv) {
3188 .dead => unreachable,3351 .dead => unreachable,
3189 .undef => unreachable,3352 .undef => unreachable,
3190 .condition_flags => unreachable,3353 .compare_flags => unreachable,
3191 .register => |dst_reg| {3354 .register => |dst_reg| {
3192 try self.genLdrRegister(dst_reg, addr_reg, elem_ty);3355 try self.genLdrRegister(dst_reg, addr_reg, elem_ty);
3193 },3356 },
...@@ -3315,6 +3478,104 @@ fn genInlineMemcpy(...@@ -3315,6 +3478,104 @@ fn genInlineMemcpy(
3315 // end:3478 // end:
3316}3479}
33173480
3481fn genInlineMemset(
3482 self: *Self,
3483 dst: MCValue,
3484 val: MCValue,
3485 len: MCValue,
3486) !void {
3487 const dst_reg = switch (dst) {
3488 .register => |r| r,
3489 else => try self.copyToTmpRegister(Type.initTag(.manyptr_u8), dst),
3490 };
3491 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
3492 defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
3493
3494 const val_reg = switch (val) {
3495 .register => |r| r,
3496 else => try self.copyToTmpRegister(Type.initTag(.u8), val),
3497 };
3498 const val_reg_lock = self.register_manager.lockReg(val_reg);
3499 defer if (val_reg_lock) |lock| self.register_manager.unlockReg(lock);
3500
3501 const len_reg = switch (len) {
3502 .register => |r| r,
3503 else => try self.copyToTmpRegister(Type.usize, len),
3504 };
3505 const len_reg_lock = self.register_manager.lockReg(len_reg);
3506 defer if (len_reg_lock) |lock| self.register_manager.unlockReg(lock);
3507
3508 const count_reg = try self.register_manager.allocReg(null, gp);
3509
3510 try self.genInlineMemsetCode(dst_reg, val_reg, len_reg, count_reg);
3511}
3512
3513fn genInlineMemsetCode(
3514 self: *Self,
3515 dst: Register,
3516 val: Register,
3517 len: Register,
3518 count: Register,
3519) !void {
3520 // mov count, #0
3521 _ = try self.addInst(.{
3522 .tag = .movz,
3523 .data = .{ .r_imm16_sh = .{
3524 .rd = count,
3525 .imm16 = 0,
3526 } },
3527 });
3528
3529 // loop:
3530 // cmp count, len
3531 _ = try self.addInst(.{
3532 .tag = .cmp_shifted_register,
3533 .data = .{ .rr_imm6_shift = .{
3534 .rn = count,
3535 .rm = len,
3536 .imm6 = 0,
3537 .shift = .lsl,
3538 } },
3539 });
3540
3541 // bge end
3542 _ = try self.addInst(.{
3543 .tag = .b_cond,
3544 .data = .{ .inst_cond = .{
3545 .inst = @intCast(u32, self.mir_instructions.len + 4),
3546 .cond = .ge,
3547 } },
3548 });
3549
3550 // strb val, [src, count]
3551 _ = try self.addInst(.{
3552 .tag = .strb_register,
3553 .data = .{ .load_store_register_register = .{
3554 .rt = val,
3555 .rn = dst,
3556 .offset = Instruction.LoadStoreOffset.reg(count).register,
3557 } },
3558 });
3559
3560 // add count, count, #1
3561 _ = try self.addInst(.{
3562 .tag = .add_immediate,
3563 .data = .{ .rr_imm12_sh = .{
3564 .rd = count,
3565 .rn = count,
3566 .imm12 = 1,
3567 } },
3568 });
3569
3570 // b loop
3571 _ = try self.addInst(.{
3572 .tag = .b,
3573 .data = .{ .inst = @intCast(u32, self.mir_instructions.len - 4) },
3574 });
3575
3576 // end:
3577}
3578
3318fn airLoad(self: *Self, inst: Air.Inst.Index) !void {3579fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
3319 const ty_op = self.air.instructions.items(.data)[inst].ty_op;3580 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3320 const elem_ty = self.air.typeOfIndex(inst);3581 const elem_ty = self.air.typeOfIndex(inst);
...@@ -3389,6 +3650,7 @@ fn genStrRegister(self: *Self, value_reg: Register, addr_reg: Register, ty: Type...@@ -3389,6 +3650,7 @@ fn genStrRegister(self: *Self, value_reg: Register, addr_reg: Register, ty: Type
3389}3650}
33903651
3391fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {3652fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {
3653 log.debug("store: storing {} to {}", .{ value, ptr });
3392 const abi_size = value_ty.abiSize(self.target.*);3654 const abi_size = value_ty.abiSize(self.target.*);
33933655
3394 switch (ptr) {3656 switch (ptr) {
...@@ -3396,7 +3658,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -3396,7 +3658,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
3396 .undef => unreachable,3658 .undef => unreachable,
3397 .unreach => unreachable,3659 .unreach => unreachable,
3398 .dead => unreachable,3660 .dead => unreachable,
3399 .condition_flags,3661 .compare_flags,
3400 .register_with_overflow,3662 .register_with_overflow,
3401 => unreachable, // cannot hold an address3663 => unreachable, // cannot hold an address
3402 .immediate => |imm| {3664 .immediate => |imm| {
...@@ -3413,6 +3675,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -3413,6 +3675,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
3413 .dead => unreachable,3675 .dead => unreachable,
3414 .undef => unreachable,3676 .undef => unreachable,
3415 .register => |value_reg| {3677 .register => |value_reg| {
3678 log.debug("store: register {} to {}", .{ value_reg, addr_reg });
3416 try self.genStrRegister(value_reg, addr_reg, value_ty);3679 try self.genStrRegister(value_reg, addr_reg, value_ty);
3417 },3680 },
3418 else => {3681 else => {
...@@ -3431,8 +3694,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -3431,8 +3694,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
3431 self.register_manager.unlockReg(reg);3694 self.register_manager.unlockReg(reg);
3432 };3695 };
34333696
3434 const src_reg = addr_reg;3697 const src_reg = regs[0];
3435 const dst_reg = regs[0];3698 const dst_reg = addr_reg;
3436 const len_reg = regs[1];3699 const len_reg = regs[1];
3437 const count_reg = regs[2];3700 const count_reg = regs[2];
3438 const tmp_reg = regs[3];3701 const tmp_reg = regs[3];
...@@ -3442,7 +3705,6 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -3442,7 +3705,6 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
3442 // sub src_reg, fp, #off3705 // sub src_reg, fp, #off
3443 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });3706 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
3444 },3707 },
3445 .memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = @intCast(u32, addr) }),
3446 .stack_argument_offset => |off| {3708 .stack_argument_offset => |off| {
3447 _ = try self.addInst(.{3709 _ = try self.addInst(.{
3448 .tag = .ldr_ptr_stack_argument,3710 .tag = .ldr_ptr_stack_argument,
...@@ -3452,6 +3714,24 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -3452,6 +3714,24 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
3452 } },3714 } },
3453 });3715 });
3454 },3716 },
3717 .memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = @intCast(u32, addr) }),
3718 .linker_load => |load_struct| {
3719 const tag: Mir.Inst.Tag = switch (load_struct.@"type") {
3720 .got => .load_memory_ptr_got,
3721 .direct => .load_memory_ptr_direct,
3722 };
3723 const mod = self.bin_file.options.module.?;
3724 _ = try self.addInst(.{
3725 .tag = tag,
3726 .data = .{
3727 .payload = try self.addExtra(Mir.LoadMemoryPie{
3728 .register = @enumToInt(src_reg),
3729 .atom_index = mod.declPtr(self.mod_fn.owner_decl).link.macho.sym_index,
3730 .sym_index = load_struct.sym_index,
3731 }),
3732 },
3733 });
3734 },
3455 else => return self.fail("TODO store {} to register", .{value}),3735 else => return self.fail("TODO store {} to register", .{value}),
3456 }3736 }
34573737
...@@ -3551,7 +3831,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -3551,7 +3831,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3551 0 => MCValue{ .register = rwo.reg },3831 0 => MCValue{ .register = rwo.reg },
35523832
3553 // get overflow bit: return C or V flag3833 // get overflow bit: return C or V flag
3554 1 => MCValue{ .condition_flags = rwo.flag },3834 1 => MCValue{ .compare_flags = rwo.flag },
35553835
3556 else => unreachable,3836 else => unreachable,
3557 };3837 };
...@@ -4005,8 +4285,8 @@ fn cmp(...@@ -4005,8 +4285,8 @@ fn cmp(
4005 }4285 }
40064286
4007 return switch (int_info.signedness) {4287 return switch (int_info.signedness) {
4008 .signed => MCValue{ .condition_flags = Condition.fromCompareOperatorSigned(op) },4288 .signed => MCValue{ .compare_flags = Condition.fromCompareOperatorSigned(op) },
4009 .unsigned => MCValue{ .condition_flags = Condition.fromCompareOperatorUnsigned(op) },4289 .unsigned => MCValue{ .compare_flags = Condition.fromCompareOperatorUnsigned(op) },
4010 };4290 };
4011 } else {4291 } else {
4012 return self.fail("TODO AArch64 cmp for ints > 64 bits", .{});4292 return self.fail("TODO AArch64 cmp for ints > 64 bits", .{});
...@@ -4064,7 +4344,7 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {...@@ -4064,7 +4344,7 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
40644344
4065fn condBr(self: *Self, condition: MCValue) !Mir.Inst.Index {4345fn condBr(self: *Self, condition: MCValue) !Mir.Inst.Index {
4066 switch (condition) {4346 switch (condition) {
4067 .condition_flags => |cond| return try self.addInst(.{4347 .compare_flags => |cond| return try self.addInst(.{
4068 .tag = .b_cond,4348 .tag = .b_cond,
4069 .data = .{4349 .data = .{
4070 .inst_cond = .{4350 .inst_cond = .{
...@@ -4120,7 +4400,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -4120,7 +4400,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
4120 var parent_stack = try self.stack.clone(self.gpa);4400 var parent_stack = try self.stack.clone(self.gpa);
4121 defer parent_stack.deinit(self.gpa);4401 defer parent_stack.deinit(self.gpa);
4122 const parent_registers = self.register_manager.registers;4402 const parent_registers = self.register_manager.registers;
4123 const parent_condition_flags_inst = self.condition_flags_inst;4403 const parent_compare_flags_inst = self.compare_flags_inst;
41244404
4125 try self.branch_stack.append(.{});4405 try self.branch_stack.append(.{});
4126 errdefer {4406 errdefer {
...@@ -4139,7 +4419,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -4139,7 +4419,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
4139 defer saved_then_branch.deinit(self.gpa);4419 defer saved_then_branch.deinit(self.gpa);
41404420
4141 self.register_manager.registers = parent_registers;4421 self.register_manager.registers = parent_registers;
4142 self.condition_flags_inst = parent_condition_flags_inst;4422 self.compare_flags_inst = parent_compare_flags_inst;
41434423
4144 self.stack.deinit(self.gpa);4424 self.stack.deinit(self.gpa);
4145 self.stack = parent_stack;4425 self.stack = parent_stack;
...@@ -4186,7 +4466,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -4186,7 +4466,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
4186 if (else_value == .dead)4466 if (else_value == .dead)
4187 continue;4467 continue;
4188 // The instruction is only overridden in the else branch.4468 // The instruction is only overridden in the else branch.
4189 var i: usize = self.branch_stack.items.len - 2;4469 var i: usize = self.branch_stack.items.len - 1;
4190 while (true) {4470 while (true) {
4191 i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?4471 i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
4192 if (self.branch_stack.items[i].inst_table.get(else_key)) |mcv| {4472 if (self.branch_stack.items[i].inst_table.get(else_key)) |mcv| {
...@@ -4213,7 +4493,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -4213,7 +4493,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
4213 if (then_value == .dead)4493 if (then_value == .dead)
4214 continue;4494 continue;
4215 const parent_mcv = blk: {4495 const parent_mcv = blk: {
4216 var i: usize = self.branch_stack.items.len - 2;4496 var i: usize = self.branch_stack.items.len - 1;
4217 while (true) {4497 while (true) {
4218 i -= 1;4498 i -= 1;
4219 if (self.branch_stack.items[i].inst_table.get(then_key)) |mcv| {4499 if (self.branch_stack.items[i].inst_table.get(then_key)) |mcv| {
...@@ -4267,9 +4547,9 @@ fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {...@@ -4267,9 +4547,9 @@ fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
4267fn isNonErr(self: *Self, ty: Type, operand: MCValue) !MCValue {4547fn isNonErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
4268 const is_err_result = try self.isErr(ty, operand);4548 const is_err_result = try self.isErr(ty, operand);
4269 switch (is_err_result) {4549 switch (is_err_result) {
4270 .condition_flags => |cond| {4550 .compare_flags => |cond| {
4271 assert(cond == .hi);4551 assert(cond == .hi);
4272 return MCValue{ .condition_flags = cond.negate() };4552 return MCValue{ .compare_flags = cond.negate() };
4273 },4553 },
4274 .immediate => |imm| {4554 .immediate => |imm| {
4275 assert(imm == 0);4555 assert(imm == 0);
...@@ -4432,10 +4712,132 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {...@@ -4432,10 +4712,132 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
44324712
4433fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {4713fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
4434 const pl_op = self.air.instructions.items(.data)[inst].pl_op;4714 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
4435 const condition = pl_op.operand;4715 const condition_ty = self.air.typeOf(pl_op.operand);
4436 _ = condition;4716 const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
4717 const liveness = try self.liveness.getSwitchBr(
4718 self.gpa,
4719 inst,
4720 switch_br.data.cases_len + 1,
4721 );
4722 defer self.gpa.free(liveness.deaths);
4723
4724 var extra_index: usize = switch_br.end;
4725 var case_i: u32 = 0;
4726 while (case_i < switch_br.data.cases_len) : (case_i += 1) {
4727 const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
4728 const items = @ptrCast([]const Air.Inst.Ref, self.air.extra[case.end..][0..case.data.items_len]);
4729 assert(items.len > 0);
4730 const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];
4731 extra_index = case.end + items.len + case_body.len;
4732
4733 // For every item, we compare it to condition and branch into
4734 // the prong if they are equal. After we compared to all
4735 // items, we branch into the next prong (or if no other prongs
4736 // exist out of the switch statement).
4737 //
4738 // cmp condition, item1
4739 // beq prong
4740 // cmp condition, item2
4741 // beq prong
4742 // cmp condition, item3
4743 // beq prong
4744 // b out
4745 // prong: ...
4746 // ...
4747 // out: ...
4748 const branch_into_prong_relocs = try self.gpa.alloc(u32, items.len);
4749 defer self.gpa.free(branch_into_prong_relocs);
4750
4751 for (items) |item, idx| {
4752 const cmp_result = try self.cmp(.{ .inst = pl_op.operand }, .{ .inst = item }, condition_ty, .neq);
4753 branch_into_prong_relocs[idx] = try self.condBr(cmp_result);
4754 }
44374755
4438 return self.fail("TODO airSwitch for {}", .{self.target.cpu.arch});4756 const branch_away_from_prong_reloc = try self.addInst(.{
4757 .tag = .b,
4758 .data = .{ .inst = undefined }, // populated later through performReloc
4759 });
4760
4761 for (branch_into_prong_relocs) |reloc| {
4762 try self.performReloc(reloc);
4763 }
4764
4765 // Capture the state of register and stack allocation state so that we can revert to it.
4766 const parent_next_stack_offset = self.next_stack_offset;
4767 const parent_free_registers = self.register_manager.free_registers;
4768 const parent_compare_flags_inst = self.compare_flags_inst;
4769 var parent_stack = try self.stack.clone(self.gpa);
4770 defer parent_stack.deinit(self.gpa);
4771 const parent_registers = self.register_manager.registers;
4772
4773 try self.branch_stack.append(.{});
4774 errdefer {
4775 _ = self.branch_stack.pop();
4776 }
4777
4778 try self.ensureProcessDeathCapacity(liveness.deaths[case_i].len);
4779 for (liveness.deaths[case_i]) |operand| {
4780 self.processDeath(operand);
4781 }
4782 try self.genBody(case_body);
4783
4784 // Revert to the previous register and stack allocation state.
4785 var saved_case_branch = self.branch_stack.pop();
4786 defer saved_case_branch.deinit(self.gpa);
4787
4788 self.register_manager.registers = parent_registers;
4789 self.compare_flags_inst = parent_compare_flags_inst;
4790 self.stack.deinit(self.gpa);
4791 self.stack = parent_stack;
4792 parent_stack = .{};
4793
4794 self.next_stack_offset = parent_next_stack_offset;
4795 self.register_manager.free_registers = parent_free_registers;
4796
4797 try self.performReloc(branch_away_from_prong_reloc);
4798 }
4799
4800 if (switch_br.data.else_body_len > 0) {
4801 const else_body = self.air.extra[extra_index..][0..switch_br.data.else_body_len];
4802
4803 // Capture the state of register and stack allocation state so that we can revert to it.
4804 const parent_next_stack_offset = self.next_stack_offset;
4805 const parent_free_registers = self.register_manager.free_registers;
4806 const parent_compare_flags_inst = self.compare_flags_inst;
4807 var parent_stack = try self.stack.clone(self.gpa);
4808 defer parent_stack.deinit(self.gpa);
4809 const parent_registers = self.register_manager.registers;
4810
4811 try self.branch_stack.append(.{});
4812 errdefer {
4813 _ = self.branch_stack.pop();
4814 }
4815
4816 const else_deaths = liveness.deaths.len - 1;
4817 try self.ensureProcessDeathCapacity(liveness.deaths[else_deaths].len);
4818 for (liveness.deaths[else_deaths]) |operand| {
4819 self.processDeath(operand);
4820 }
4821 try self.genBody(else_body);
4822
4823 // Revert to the previous register and stack allocation state.
4824 var saved_case_branch = self.branch_stack.pop();
4825 defer saved_case_branch.deinit(self.gpa);
4826
4827 self.register_manager.registers = parent_registers;
4828 self.compare_flags_inst = parent_compare_flags_inst;
4829 self.stack.deinit(self.gpa);
4830 self.stack = parent_stack;
4831 parent_stack = .{};
4832
4833 self.next_stack_offset = parent_next_stack_offset;
4834 self.register_manager.free_registers = parent_free_registers;
4835
4836 // TODO consolidate returned MCValues between prongs and else branch like we do
4837 // in airCondBr.
4838 }
4839
4840 return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
4439}4841}
44404842
4441fn performReloc(self: *Self, inst: Mir.Inst.Index) !void {4843fn performReloc(self: *Self, inst: Mir.Inst.Index) !void {
...@@ -4464,7 +4866,7 @@ fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {...@@ -4464,7 +4866,7 @@ fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
4464 block_data.mcv = switch (operand_mcv) {4866 block_data.mcv = switch (operand_mcv) {
4465 .none, .dead, .unreach => unreachable,4867 .none, .dead, .unreach => unreachable,
4466 .register, .stack_offset, .memory => operand_mcv,4868 .register, .stack_offset, .memory => operand_mcv,
4467 .immediate, .stack_argument_offset, .condition_flags => blk: {4869 .immediate, .stack_argument_offset, .compare_flags => blk: {
4468 const new_mcv = try self.allocRegOrMem(self.air.typeOfIndex(block), true, block);4870 const new_mcv = try self.allocRegOrMem(self.air.typeOfIndex(block), true, block);
4469 try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv);4871 try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv);
4470 break :blk new_mcv;4872 break :blk new_mcv;
...@@ -4644,10 +5046,14 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro...@@ -4644,10 +5046,14 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
4644 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),5046 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
4645 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),5047 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
4646 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),5048 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
4647 else => return self.fail("TODO implement memset", .{}),5049 else => try self.genInlineMemset(
5050 .{ .ptr_stack_offset = stack_offset },
5051 .{ .immediate = 0xaa },
5052 .{ .immediate = abi_size },
5053 ),
4648 }5054 }
4649 },5055 },
4650 .condition_flags,5056 .compare_flags,
4651 .immediate,5057 .immediate,
4652 .ptr_stack_offset,5058 .ptr_stack_offset,
4653 => {5059 => {
...@@ -4807,7 +5213,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -4807,7 +5213,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
4807 } },5213 } },
4808 });5214 });
4809 },5215 },
4810 .condition_flags => |condition| {5216 .compare_flags => |condition| {
4811 _ = try self.addInst(.{5217 _ = try self.addInst(.{
4812 .tag = .cset,5218 .tag = .cset,
4813 .data = .{ .r_cond = .{5219 .data = .{ .r_cond = .{
...@@ -5084,7 +5490,7 @@ fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) I...@@ -5084,7 +5490,7 @@ fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) I
5084 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);5490 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
5085 }5491 }
5086 },5492 },
5087 .condition_flags,5493 .compare_flags,
5088 .immediate,5494 .immediate,
5089 .ptr_stack_offset,5495 .ptr_stack_offset,
5090 => {5496 => {
src/arch/aarch64/Emit.zig+47-31
...@@ -131,6 +131,7 @@ pub fn emitMir(...@@ -131,6 +131,7 @@ pub fn emitMir(
131 .subs_extended_register => try emit.mirAddSubtractExtendedRegister(inst),131 .subs_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
132 .cmp_extended_register => try emit.mirAddSubtractExtendedRegister(inst),132 .cmp_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
133133
134 .csel => try emit.mirConditionalSelect(inst),
134 .cset => try emit.mirConditionalSelect(inst),135 .cset => try emit.mirConditionalSelect(inst),
135136
136 .dbg_line => try emit.mirDbgLine(inst),137 .dbg_line => try emit.mirDbgLine(inst),
...@@ -804,6 +805,14 @@ fn mirAddSubtractExtendedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {...@@ -804,6 +805,14 @@ fn mirAddSubtractExtendedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
804fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void {805fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void {
805 const tag = emit.mir.instructions.items(.tag)[inst];806 const tag = emit.mir.instructions.items(.tag)[inst];
806 switch (tag) {807 switch (tag) {
808 .csel => {
809 const rrr_cond = emit.mir.instructions.items(.data)[inst].rrr_cond;
810 const rd = rrr_cond.rd;
811 const rn = rrr_cond.rn;
812 const rm = rrr_cond.rm;
813 const cond = rrr_cond.cond;
814 try emit.writeInstruction(Instruction.csel(rd, rn, rm, cond));
815 },
807 .cset => {816 .cset => {
808 const r_cond = emit.mir.instructions.items(.data)[inst].r_cond;817 const r_cond = emit.mir.instructions.items(.data)[inst].r_cond;
809 const zr: Register = switch (r_cond.rd.size()) {818 const zr: Register = switch (r_cond.rd.size()) {
...@@ -1182,42 +1191,50 @@ fn mirNop(emit: *Emit) !void {...@@ -1182,42 +1191,50 @@ fn mirNop(emit: *Emit) !void {
1182 try emit.writeInstruction(Instruction.nop());1191 try emit.writeInstruction(Instruction.nop());
1183}1192}
11841193
1194fn regListIsSet(reg_list: u32, reg: Register) bool {
1195 return reg_list & @as(u32, 1) << @intCast(u5, reg.id()) != 0;
1196}
1197
1185fn mirPushPopRegs(emit: *Emit, inst: Mir.Inst.Index) !void {1198fn mirPushPopRegs(emit: *Emit, inst: Mir.Inst.Index) !void {
1186 const tag = emit.mir.instructions.items(.tag)[inst];1199 const tag = emit.mir.instructions.items(.tag)[inst];
1187 const reg_list = emit.mir.instructions.items(.data)[inst].reg_list;1200 const reg_list = emit.mir.instructions.items(.data)[inst].reg_list;
11881201
1189 if (reg_list & @as(u32, 1) << 31 != 0) return emit.fail("xzr is not a valid register for {}", .{tag});1202 if (regListIsSet(reg_list, .xzr)) return emit.fail("xzr is not a valid register for {}", .{tag});
11901203
1191 // sp must be aligned at all times, so we only use stp and ldp1204 // sp must be aligned at all times, so we only use stp and ldp
1192 // instructions for minimal instruction count. However, if we do1205 // instructions for minimal instruction count.
1193 // not have an even number of registers, we use str and ldr1206 //
1207 // However, if we have an odd number of registers, for pop_regs we
1208 // use one ldr instruction followed by zero or more ldp
1209 // instructions; for push_regs we use zero or more stp
1210 // instructions followed by one str instruction.
1194 const number_of_regs = @popCount(reg_list);1211 const number_of_regs = @popCount(reg_list);
1212 const odd_number_of_regs = number_of_regs % 2 != 0;
11951213
1196 switch (tag) {1214 switch (tag) {
1197 .pop_regs => {1215 .pop_regs => {
1198 var i: u6 = 32;1216 var i: u6 = 32;
1199 var count: u6 = 0;1217 var count: u6 = 0;
1200 var other_reg: Register = undefined;1218 var other_reg: ?Register = null;
1201 while (i > 0) : (i -= 1) {1219 while (i > 0) : (i -= 1) {
1202 const reg = @intToEnum(Register, i - 1);1220 const reg = @intToEnum(Register, i - 1);
1203 if (reg_list & @as(u32, 1) << @intCast(u5, reg.id()) != 0) {1221 if (regListIsSet(reg_list, reg)) {
1204 if (count % 2 == 0) {1222 if (count == 0 and odd_number_of_regs) {
1205 if (count == number_of_regs - 1) {1223 try emit.writeInstruction(Instruction.ldr(
1206 try emit.writeInstruction(Instruction.ldr(1224 reg,
1207 reg,1225 .sp,
1208 .sp,1226 Instruction.LoadStoreOffset.imm_post_index(16),
1209 Instruction.LoadStoreOffset.imm_post_index(16),1227 ));
1210 ));1228 } else if (other_reg) |r| {
1211 } else {
1212 other_reg = reg;
1213 }
1214 } else {
1215 try emit.writeInstruction(Instruction.ldp(1229 try emit.writeInstruction(Instruction.ldp(
1216 reg,1230 reg,
1217 other_reg,1231 r,
1218 .sp,1232 .sp,
1219 Instruction.LoadStorePairOffset.post_index(16),1233 Instruction.LoadStorePairOffset.post_index(16),
1220 ));1234 ));
1235 other_reg = null;
1236 } else {
1237 other_reg = reg;
1221 }1238 }
1222 count += 1;1239 count += 1;
1223 }1240 }
...@@ -1227,27 +1244,26 @@ fn mirPushPopRegs(emit: *Emit, inst: Mir.Inst.Index) !void {...@@ -1227,27 +1244,26 @@ fn mirPushPopRegs(emit: *Emit, inst: Mir.Inst.Index) !void {
1227 .push_regs => {1244 .push_regs => {
1228 var i: u6 = 0;1245 var i: u6 = 0;
1229 var count: u6 = 0;1246 var count: u6 = 0;
1230 var other_reg: Register = undefined;1247 var other_reg: ?Register = null;
1231 while (i < 32) : (i += 1) {1248 while (i < 32) : (i += 1) {
1232 const reg = @intToEnum(Register, i);1249 const reg = @intToEnum(Register, i);
1233 if (reg_list & @as(u32, 1) << @intCast(u5, reg.id()) != 0) {1250 if (regListIsSet(reg_list, reg)) {
1234 if (count % 2 == 0) {1251 if (count == number_of_regs - 1 and odd_number_of_regs) {
1235 if (count == number_of_regs - 1) {1252 try emit.writeInstruction(Instruction.str(
1236 try emit.writeInstruction(Instruction.str(1253 reg,
1237 reg,1254 .sp,
1238 .sp,1255 Instruction.LoadStoreOffset.imm_pre_index(-16),
1239 Instruction.LoadStoreOffset.imm_pre_index(-16),1256 ));
1240 ));1257 } else if (other_reg) |r| {
1241 } else {
1242 other_reg = reg;
1243 }
1244 } else {
1245 try emit.writeInstruction(Instruction.stp(1258 try emit.writeInstruction(Instruction.stp(
1246 other_reg,1259 r,
1247 reg,1260 reg,
1248 .sp,1261 .sp,
1249 Instruction.LoadStorePairOffset.pre_index(-16),1262 Instruction.LoadStorePairOffset.pre_index(-16),
1250 ));1263 ));
1264 other_reg = null;
1265 } else {
1266 other_reg = reg;
1251 }1267 }
1252 count += 1;1268 count += 1;
1253 }1269 }
src/arch/aarch64/Mir.zig+11
...@@ -62,6 +62,8 @@ pub const Inst = struct {...@@ -62,6 +62,8 @@ pub const Inst = struct {
62 cmp_shifted_register,62 cmp_shifted_register,
63 /// Compare (extended register)63 /// Compare (extended register)
64 cmp_extended_register,64 cmp_extended_register,
65 /// Conditional Select
66 csel,
65 /// Conditional set67 /// Conditional set
66 cset,68 cset,
67 /// Pseudo-instruction: End of prologue69 /// Pseudo-instruction: End of prologue
...@@ -387,6 +389,15 @@ pub const Inst = struct {...@@ -387,6 +389,15 @@ pub const Inst = struct {
387 rn: Register,389 rn: Register,
388 rm: Register,390 rm: Register,
389 },391 },
392 /// Three registers and a condition
393 ///
394 /// Used by e.g. csel
395 rrr_cond: struct {
396 rd: Register,
397 rn: Register,
398 rm: Register,
399 cond: bits.Instruction.Condition,
400 },
390 /// Three registers and a shift (shift type and 6-bit amount)401 /// Three registers and a shift (shift type and 6-bit amount)
391 ///402 ///
392 /// Used by e.g. add_shifted_register403 /// Used by e.g. add_shifted_register
test/behavior/switch.zig-25
...@@ -5,8 +5,6 @@ const expectError = std.testing.expectError;...@@ -5,8 +5,6 @@ const expectError = std.testing.expectError;
5const expectEqual = std.testing.expectEqual;5const expectEqual = std.testing.expectEqual;
66
7test "switch with numbers" {7test "switch with numbers" {
8 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9
10 try testSwitchWithNumbers(13);8 try testSwitchWithNumbers(13);
11}9}
1210
...@@ -20,8 +18,6 @@ fn testSwitchWithNumbers(x: u32) !void {...@@ -20,8 +18,6 @@ fn testSwitchWithNumbers(x: u32) !void {
20}18}
2119
22test "switch with all ranges" {20test "switch with all ranges" {
23 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
24
25 try expect(testSwitchWithAllRanges(50, 3) == 1);21 try expect(testSwitchWithAllRanges(50, 3) == 1);
26 try expect(testSwitchWithAllRanges(101, 0) == 2);22 try expect(testSwitchWithAllRanges(101, 0) == 2);
27 try expect(testSwitchWithAllRanges(300, 5) == 3);23 try expect(testSwitchWithAllRanges(300, 5) == 3);
...@@ -53,8 +49,6 @@ test "implicit comptime switch" {...@@ -53,8 +49,6 @@ test "implicit comptime switch" {
53}49}
5450
55test "switch on enum" {51test "switch on enum" {
56 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
57
58 const fruit = Fruit.Orange;52 const fruit = Fruit.Orange;
59 nonConstSwitchOnEnum(fruit);53 nonConstSwitchOnEnum(fruit);
60}54}
...@@ -72,8 +66,6 @@ fn nonConstSwitchOnEnum(fruit: Fruit) void {...@@ -72,8 +66,6 @@ fn nonConstSwitchOnEnum(fruit: Fruit) void {
72}66}
7367
74test "switch statement" {68test "switch statement" {
75 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
76
77 try nonConstSwitch(SwitchStatementFoo.C);69 try nonConstSwitch(SwitchStatementFoo.C);
78}70}
79fn nonConstSwitch(foo: SwitchStatementFoo) !void {71fn nonConstSwitch(foo: SwitchStatementFoo) !void {
...@@ -89,7 +81,6 @@ const SwitchStatementFoo = enum { A, B, C, D };...@@ -89,7 +81,6 @@ const SwitchStatementFoo = enum { A, B, C, D };
8981
90test "switch with multiple expressions" {82test "switch with multiple expressions" {
91 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO83 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
92 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9384
94 const x = switch (returnsFive()) {85 const x = switch (returnsFive()) {
95 1, 2, 3 => 1,86 1, 2, 3 => 1,
...@@ -103,8 +94,6 @@ fn returnsFive() i32 {...@@ -103,8 +94,6 @@ fn returnsFive() i32 {
103}94}
10495
105test "switch on type" {96test "switch on type" {
106 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
107
108 try expect(trueIfBoolFalseOtherwise(bool));97 try expect(trueIfBoolFalseOtherwise(bool));
109 try expect(!trueIfBoolFalseOtherwise(i32));98 try expect(!trueIfBoolFalseOtherwise(i32));
110}99}
...@@ -117,8 +106,6 @@ fn trueIfBoolFalseOtherwise(comptime T: type) bool {...@@ -117,8 +106,6 @@ fn trueIfBoolFalseOtherwise(comptime T: type) bool {
117}106}
118107
119test "switching on booleans" {108test "switching on booleans" {
120 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
121
122 try testSwitchOnBools();109 try testSwitchOnBools();
123 comptime try testSwitchOnBools();110 comptime try testSwitchOnBools();
124}111}
...@@ -170,8 +157,6 @@ test "undefined.u0" {...@@ -170,8 +157,6 @@ test "undefined.u0" {
170}157}
171158
172test "switch with disjoint range" {159test "switch with disjoint range" {
173 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
174
175 var q: u8 = 0;160 var q: u8 = 0;
176 switch (q) {161 switch (q) {
177 0...125 => {},162 0...125 => {},
...@@ -214,8 +199,6 @@ fn poll() void {...@@ -214,8 +199,6 @@ fn poll() void {
214}199}
215200
216test "switch on global mutable var isn't constant-folded" {201test "switch on global mutable var isn't constant-folded" {
217 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
218
219 while (state < 2) {202 while (state < 2) {
220 poll();203 poll();
221 }204 }
...@@ -273,7 +256,6 @@ fn testSwitchEnumPtrCapture() !void {...@@ -273,7 +256,6 @@ fn testSwitchEnumPtrCapture() !void {
273256
274test "switch handles all cases of number" {257test "switch handles all cases of number" {
275 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO258 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
276 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
277259
278 try testSwitchHandleAllCases();260 try testSwitchHandleAllCases();
279 comptime try testSwitchHandleAllCases();261 comptime try testSwitchHandleAllCases();
...@@ -363,8 +345,6 @@ test "anon enum literal used in switch on union enum" {...@@ -363,8 +345,6 @@ test "anon enum literal used in switch on union enum" {
363}345}
364346
365test "switch all prongs unreachable" {347test "switch all prongs unreachable" {
366 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
367
368 try testAllProngsUnreachable();348 try testAllProngsUnreachable();
369 comptime try testAllProngsUnreachable();349 comptime try testAllProngsUnreachable();
370}350}
...@@ -400,7 +380,6 @@ fn return_a_number() anyerror!i32 {...@@ -400,7 +380,6 @@ fn return_a_number() anyerror!i32 {
400380
401test "switch on integer with else capturing expr" {381test "switch on integer with else capturing expr" {
402 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO382 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
403 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
404383
405 const S = struct {384 const S = struct {
406 fn doTheTest() !void {385 fn doTheTest() !void {
...@@ -641,8 +620,6 @@ test "switch capture copies its payload" {...@@ -641,8 +620,6 @@ test "switch capture copies its payload" {
641}620}
642621
643test "capture of integer forwards the switch condition directly" {622test "capture of integer forwards the switch condition directly" {
644 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
645
646 const S = struct {623 const S = struct {
647 fn foo(x: u8) !void {624 fn foo(x: u8) !void {
648 switch (x) {625 switch (x) {
...@@ -662,8 +639,6 @@ test "capture of integer forwards the switch condition directly" {...@@ -662,8 +639,6 @@ test "capture of integer forwards the switch condition directly" {
662}639}
663640
664test "enum value without tag name used as switch item" {641test "enum value without tag name used as switch item" {
665 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
666
667 const E = enum(u32) {642 const E = enum(u32) {
668 a = 1,643 a = 1,
669 b = 2,644 b = 2,