authorgravatar for joachim.schmidt557@outlook.comJoachim Schmidt <joachim.schmidt557@outlook.com> 2022-11-20 21:02:20+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-09 14:35:32-07:00
log10a660ebea6b616644b1010b3766bf9899eeaa75
tree05785dd3b39ce238085c368175af0619f61a07de
parent9498c95fafad5b1561f37461a7588e8b2a922044

stage2 ARM: misc fixes

- remove redundant `new` from `binOpRegisterNew` name - fix mul_with_overflow

1 files changed, 105 insertions(+), 66 deletions(-)

src/arch/arm/CodeGen.zig+105-66
...@@ -1416,8 +1416,9 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {...@@ -1416,8 +1416,9 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
14161416
1417 .div_float => try self.divFloat(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),1417 .div_float => try self.divFloat(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
14181418
1419 .div_trunc => try self.div(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),1419 .div_trunc => try self.divTrunc(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1420 .div_floor => try self.div(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),1420
1421 .div_floor => try self.divFloor(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
14211422
1422 .div_exact => try self.divExact(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),1423 .div_exact => try self.divExact(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
14231424
...@@ -1567,12 +1568,12 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1567,12 +1568,12 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
15671568
1568 const dest = blk: {1569 const dest = blk: {
1569 if (rhs_immediate_ok) {1570 if (rhs_immediate_ok) {
1570 break :blk try self.binOpImmediateNew(mir_tag, lhs_bind, rhs_immediate.?, lhs_ty, false, null);1571 break :blk try self.binOpImmediate(mir_tag, lhs_bind, rhs_immediate.?, lhs_ty, false, null);
1571 } else if (lhs_immediate_ok) {1572 } else if (lhs_immediate_ok) {
1572 // swap lhs and rhs1573 // swap lhs and rhs
1573 break :blk try self.binOpImmediateNew(mir_tag, rhs_bind, lhs_immediate.?, rhs_ty, true, null);1574 break :blk try self.binOpImmediate(mir_tag, rhs_bind, lhs_immediate.?, rhs_ty, true, null);
1574 } else {1575 } else {
1575 break :blk try self.binOpRegisterNew(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, null);1576 break :blk try self.binOpRegister(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, null);
1576 }1577 }
1577 };1578 };
15781579
...@@ -1625,7 +1626,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1625,7 +1626,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1625 .unsigned => .mul,1626 .unsigned => .mul,
1626 };1627 };
16271628
1628 const dest = try self.binOpRegisterNew(base_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, null);1629 const dest = try self.binOpRegister(base_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, null);
1629 const dest_reg = dest.register;1630 const dest_reg = dest.register;
1630 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);1631 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
1631 defer self.register_manager.unlockReg(dest_reg_lock);1632 defer self.register_manager.unlockReg(dest_reg_lock);
...@@ -3123,7 +3124,7 @@ fn allocRegs(...@@ -3123,7 +3124,7 @@ fn allocRegs(
3123/// instructions which are binary operations acting on two registers3124/// instructions which are binary operations acting on two registers
3124///3125///
3125/// Returns the destination register3126/// Returns the destination register
3126fn binOpRegisterNew(3127fn binOpRegister(
3127 self: *Self,3128 self: *Self,
3128 mir_tag: Mir.Inst.Tag,3129 mir_tag: Mir.Inst.Tag,
3129 lhs_bind: ReadArg.Bind,3130 lhs_bind: ReadArg.Bind,
...@@ -3196,7 +3197,7 @@ fn binOpRegisterNew(...@@ -3196,7 +3197,7 @@ fn binOpRegisterNew(
3196/// an immediate3197/// an immediate
3197///3198///
3198/// Returns the destination register3199/// Returns the destination register
3199fn binOpImmediateNew(3200fn binOpImmediate(
3200 self: *Self,3201 self: *Self,
3201 mir_tag: Mir.Inst.Tag,3202 mir_tag: Mir.Inst.Tag,
3202 lhs_bind: ReadArg.Bind,3203 lhs_bind: ReadArg.Bind,
...@@ -3265,11 +3266,11 @@ fn addSub(...@@ -3265,11 +3266,11 @@ fn addSub(
3265 rhs_ty: Type,3266 rhs_ty: Type,
3266 maybe_inst: ?Air.Inst.Index,3267 maybe_inst: ?Air.Inst.Index,
3267) InnerError!MCValue {3268) InnerError!MCValue {
3269 const mod = self.bin_file.options.module.?;
3268 switch (lhs_ty.zigTypeTag()) {3270 switch (lhs_ty.zigTypeTag()) {
3269 .Float => return self.fail("TODO ARM binary operations on floats", .{}),3271 .Float => return self.fail("TODO ARM binary operations on floats", .{}),
3270 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),3272 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3271 .Int => {3273 .Int => {
3272 const mod = self.bin_file.options.module.?;
3273 assert(lhs_ty.eql(rhs_ty, mod));3274 assert(lhs_ty.eql(rhs_ty, mod));
3274 const int_info = lhs_ty.intInfo(self.target.*);3275 const int_info = lhs_ty.intInfo(self.target.*);
3275 if (int_info.bits <= 32) {3276 if (int_info.bits <= 32) {
...@@ -3298,12 +3299,12 @@ fn addSub(...@@ -3298,12 +3299,12 @@ fn addSub(
3298 };3299 };
32993300
3300 if (rhs_immediate_ok) {3301 if (rhs_immediate_ok) {
3301 return try self.binOpImmediateNew(mir_tag, lhs_bind, rhs_immediate.?, lhs_ty, false, maybe_inst);3302 return try self.binOpImmediate(mir_tag, lhs_bind, rhs_immediate.?, lhs_ty, false, maybe_inst);
3302 } else if (lhs_immediate_ok) {3303 } else if (lhs_immediate_ok) {
3303 // swap lhs and rhs3304 // swap lhs and rhs
3304 return try self.binOpImmediateNew(mir_tag, rhs_bind, lhs_immediate.?, rhs_ty, true, maybe_inst);3305 return try self.binOpImmediate(mir_tag, rhs_bind, lhs_immediate.?, rhs_ty, true, maybe_inst);
3305 } else {3306 } else {
3306 return try self.binOpRegisterNew(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);3307 return try self.binOpRegister(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
3307 }3308 }
3308 } else {3309 } else {
3309 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});3310 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
...@@ -3321,18 +3322,18 @@ fn mul(...@@ -3321,18 +3322,18 @@ fn mul(
3321 rhs_ty: Type,3322 rhs_ty: Type,
3322 maybe_inst: ?Air.Inst.Index,3323 maybe_inst: ?Air.Inst.Index,
3323) InnerError!MCValue {3324) InnerError!MCValue {
3325 const mod = self.bin_file.options.module.?;
3324 switch (lhs_ty.zigTypeTag()) {3326 switch (lhs_ty.zigTypeTag()) {
3325 .Float => return self.fail("TODO ARM binary operations on floats", .{}),3327 .Float => return self.fail("TODO ARM binary operations on floats", .{}),
3326 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),3328 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3327 .Int => {3329 .Int => {
3328 const mod = self.bin_file.options.module.?;
3329 assert(lhs_ty.eql(rhs_ty, mod));3330 assert(lhs_ty.eql(rhs_ty, mod));
3330 const int_info = lhs_ty.intInfo(self.target.*);3331 const int_info = lhs_ty.intInfo(self.target.*);
3331 if (int_info.bits <= 32) {3332 if (int_info.bits <= 32) {
3332 // TODO add optimisations for multiplication3333 // TODO add optimisations for multiplication
3333 // with immediates, for example a * 2 can be3334 // with immediates, for example a * 2 can be
3334 // lowered to a << 13335 // lowered to a << 1
3335 return try self.binOpRegisterNew(.mul, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);3336 return try self.binOpRegister(.mul, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
3336 } else {3337 } else {
3337 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});3338 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
3338 }3339 }
...@@ -3361,22 +3362,62 @@ fn divFloat(...@@ -3361,22 +3362,62 @@ fn divFloat(
3361 }3362 }
3362}3363}
33633364
3364fn div(3365fn divTrunc(
3365 self: *Self,3366 self: *Self,
3366 tag: Air.Inst.Tag,
3367 lhs_bind: ReadArg.Bind,3367 lhs_bind: ReadArg.Bind,
3368 rhs_bind: ReadArg.Bind,3368 rhs_bind: ReadArg.Bind,
3369 lhs_ty: Type,3369 lhs_ty: Type,
3370 rhs_ty: Type,3370 rhs_ty: Type,
3371 maybe_inst: ?Air.Inst.Index,3371 maybe_inst: ?Air.Inst.Index,
3372) InnerError!MCValue {3372) InnerError!MCValue {
3373 _ = tag;3373 const mod = self.bin_file.options.module.?;
3374 switch (lhs_ty.zigTypeTag()) {
3375 .Float => return self.fail("TODO ARM binary operations on floats", .{}),
3376 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3377 .Int => {
3378 assert(lhs_ty.eql(rhs_ty, mod));
3379 const int_info = lhs_ty.intInfo(self.target.*);
3380 if (int_info.bits <= 32) {
3381 switch (int_info.signedness) {
3382 .signed => {
3383 return self.fail("TODO ARM signed integer division", .{});
3384 },
3385 .unsigned => {
3386 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
3387
3388 if (rhs_immediate) |imm| {
3389 if (std.math.isPowerOfTwo(imm)) {
3390 const shift = std.math.log2_int(u32, imm);
3391 return try self.binOpImmediate(.lsr, lhs_bind, shift, lhs_ty, false, maybe_inst);
3392 } else {
3393 return self.fail("TODO ARM integer division by constants", .{});
3394 }
3395 } else {
3396 return self.fail("TODO ARM integer division", .{});
3397 }
3398 },
3399 }
3400 } else {
3401 return self.fail("TODO ARM integer division for integers > u32/i32", .{});
3402 }
3403 },
3404 else => unreachable,
3405 }
3406}
33743407
3408fn divFloor(
3409 self: *Self,
3410 lhs_bind: ReadArg.Bind,
3411 rhs_bind: ReadArg.Bind,
3412 lhs_ty: Type,
3413 rhs_ty: Type,
3414 maybe_inst: ?Air.Inst.Index,
3415) InnerError!MCValue {
3416 const mod = self.bin_file.options.module.?;
3375 switch (lhs_ty.zigTypeTag()) {3417 switch (lhs_ty.zigTypeTag()) {
3376 .Float => return self.fail("TODO ARM binary operations on floats", .{}),3418 .Float => return self.fail("TODO ARM binary operations on floats", .{}),
3377 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),3419 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3378 .Int => {3420 .Int => {
3379 const mod = self.bin_file.options.module.?;
3380 assert(lhs_ty.eql(rhs_ty, mod));3421 assert(lhs_ty.eql(rhs_ty, mod));
3381 const int_info = lhs_ty.intInfo(self.target.*);3422 const int_info = lhs_ty.intInfo(self.target.*);
3382 if (int_info.bits <= 32) {3423 if (int_info.bits <= 32) {
...@@ -3390,7 +3431,7 @@ fn div(...@@ -3390,7 +3431,7 @@ fn div(
3390 if (rhs_immediate) |imm| {3431 if (rhs_immediate) |imm| {
3391 if (std.math.isPowerOfTwo(imm)) {3432 if (std.math.isPowerOfTwo(imm)) {
3392 const shift = std.math.log2_int(u32, imm);3433 const shift = std.math.log2_int(u32, imm);
3393 return try self.binOpImmediateNew(.lsr, lhs_bind, shift, lhs_ty, false, maybe_inst);3434 return try self.binOpImmediate(.lsr, lhs_bind, shift, lhs_ty, false, maybe_inst);
3394 } else {3435 } else {
3395 return self.fail("TODO ARM integer division by constants", .{});3436 return self.fail("TODO ARM integer division by constants", .{});
3396 }3437 }
...@@ -3436,11 +3477,11 @@ fn rem(...@@ -3436,11 +3477,11 @@ fn rem(
3436 rhs_ty: Type,3477 rhs_ty: Type,
3437 maybe_inst: ?Air.Inst.Index,3478 maybe_inst: ?Air.Inst.Index,
3438) InnerError!MCValue {3479) InnerError!MCValue {
3480 const mod = self.bin_file.options.module.?;
3439 switch (lhs_ty.zigTypeTag()) {3481 switch (lhs_ty.zigTypeTag()) {
3440 .Float => return self.fail("TODO ARM binary operations on floats", .{}),3482 .Float => return self.fail("TODO ARM binary operations on floats", .{}),
3441 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),3483 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3442 .Int => {3484 .Int => {
3443 const mod = self.bin_file.options.module.?;
3444 assert(lhs_ty.eql(rhs_ty, mod));3485 assert(lhs_ty.eql(rhs_ty, mod));
3445 const int_info = lhs_ty.intInfo(self.target.*);3486 const int_info = lhs_ty.intInfo(self.target.*);
3446 if (int_info.bits <= 32) {3487 if (int_info.bits <= 32) {
...@@ -3522,28 +3563,26 @@ fn wrappingArithmetic(...@@ -3522,28 +3563,26 @@ fn wrappingArithmetic(
3522 rhs_ty: Type,3563 rhs_ty: Type,
3523 maybe_inst: ?Air.Inst.Index,3564 maybe_inst: ?Air.Inst.Index,
3524) InnerError!MCValue {3565) InnerError!MCValue {
3525 const base_tag: Air.Inst.Tag = switch (tag) {
3526 .addwrap => .add,
3527 .subwrap => .sub,
3528 .mulwrap => .mul,
3529 else => unreachable,
3530 };
3531
3532 // Generate an add/sub/mul
3533 const result = try self.addSub(base_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
3534
3535 // Truncate if necessary
3536 switch (lhs_ty.zigTypeTag()) {3566 switch (lhs_ty.zigTypeTag()) {
3537 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),3567 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3538 .Int => {3568 .Int => {
3539 const int_info = lhs_ty.intInfo(self.target.*);3569 const int_info = lhs_ty.intInfo(self.target.*);
3540 if (int_info.bits <= 32) {3570 if (int_info.bits <= 32) {
3541 const result_reg = result.register;3571 // Generate an add/sub/mul
3572 const result: MCValue = switch (tag) {
3573 .addwrap => try self.addSub(.add, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
3574 .subwrap => try self.addSub(.sub, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
3575 .mulwrap => try self.mul(lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
3576 else => unreachable,
3577 };
35423578
3579 // Truncate if necessary
3580 const result_reg = result.register;
3543 if (int_info.bits < 32) {3581 if (int_info.bits < 32) {
3544 try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits);3582 try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits);
3545 return result;3583 }
3546 } else return result;3584
3585 return result;
3547 } else {3586 } else {
3548 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});3587 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
3549 }3588 }
...@@ -3582,12 +3621,12 @@ fn bitwise(...@@ -3582,12 +3621,12 @@ fn bitwise(
3582 };3621 };
35833622
3584 if (rhs_immediate_ok) {3623 if (rhs_immediate_ok) {
3585 return try self.binOpImmediateNew(mir_tag, lhs_bind, rhs_immediate.?, lhs_ty, false, maybe_inst);3624 return try self.binOpImmediate(mir_tag, lhs_bind, rhs_immediate.?, lhs_ty, false, maybe_inst);
3586 } else if (lhs_immediate_ok) {3625 } else if (lhs_immediate_ok) {
3587 // swap lhs and rhs3626 // swap lhs and rhs
3588 return try self.binOpImmediateNew(mir_tag, rhs_bind, lhs_immediate.?, rhs_ty, true, maybe_inst);3627 return try self.binOpImmediate(mir_tag, rhs_bind, lhs_immediate.?, rhs_ty, true, maybe_inst);
3589 } else {3628 } else {
3590 return try self.binOpRegisterNew(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);3629 return try self.binOpRegister(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
3591 }3630 }
3592 } else {3631 } else {
3593 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});3632 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
...@@ -3623,9 +3662,9 @@ fn shiftExact(...@@ -3623,9 +3662,9 @@ fn shiftExact(
3623 };3662 };
36243663
3625 if (rhs_immediate) |imm| {3664 if (rhs_immediate) |imm| {
3626 return try self.binOpImmediateNew(mir_tag, lhs_bind, imm, lhs_ty, false, maybe_inst);3665 return try self.binOpImmediate(mir_tag, lhs_bind, imm, lhs_ty, false, maybe_inst);
3627 } else {3666 } else {
3628 return try self.binOpRegisterNew(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);3667 return try self.binOpRegister(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
3629 }3668 }
3630 } else {3669 } else {
3631 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});3670 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
...@@ -3644,34 +3683,34 @@ fn shiftNormal(...@@ -3644,34 +3683,34 @@ fn shiftNormal(
3644 rhs_ty: Type,3683 rhs_ty: Type,
3645 maybe_inst: ?Air.Inst.Index,3684 maybe_inst: ?Air.Inst.Index,
3646) InnerError!MCValue {3685) InnerError!MCValue {
3647 const base_tag: Air.Inst.Tag = switch (tag) {3686 switch (lhs_ty.zigTypeTag()) {
3648 .shl => .shl_exact,3687 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3649 .shr => .shr_exact,3688 .Int => {
3650 else => unreachable,3689 const int_info = lhs_ty.intInfo(self.target.*);
3651 };3690 if (int_info.bits <= 32) {
36523691 // Generate a shl_exact/shr_exact
3653 // Generate a shl_exact/shr_exact3692 const result: MCValue = switch (tag) {
3654 const result = try self.shiftExact(base_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);3693 .shl => try self.shiftExact(.shl_exact, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
3694 .shr => try self.shiftExact(.shr_exact, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
3695 else => unreachable,
3696 };
36553697
3656 // Truncate if necessary3698 // Truncate if necessary
3657 switch (tag) {3699 switch (tag) {
3658 .shr => return result,3700 .shr => return result,
3659 .shl => switch (lhs_ty.zigTypeTag()) {3701 .shl => {
3660 .Vector => return self.fail("TODO ARM binary operations on vectors", .{}),3702 const result_reg = result.register;
3661 .Int => {3703 if (int_info.bits < 32) {
3662 const int_info = lhs_ty.intInfo(self.target.*);3704 try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits);
3663 if (int_info.bits <= 32) {3705 }
3664 const result_reg = result.register;
36653706
3666 if (int_info.bits < 32) {
3667 try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits);
3668 return result;3707 return result;
3669 } else return result;3708 },
3670 } else {3709 else => unreachable,
3671 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
3672 }3710 }
3673 },3711 } else {
3674 else => unreachable,3712 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
3713 }
3675 },3714 },
3676 else => unreachable,3715 else => unreachable,
3677 }3716 }
...@@ -3698,12 +3737,12 @@ fn booleanOp(...@@ -3698,12 +3737,12 @@ fn booleanOp(
3698 };3737 };
36993738
3700 if (rhs_immediate) |imm| {3739 if (rhs_immediate) |imm| {
3701 return try self.binOpImmediateNew(mir_tag, lhs_bind, imm, lhs_ty, false, maybe_inst);3740 return try self.binOpImmediate(mir_tag, lhs_bind, imm, lhs_ty, false, maybe_inst);
3702 } else if (lhs_immediate) |imm| {3741 } else if (lhs_immediate) |imm| {
3703 // swap lhs and rhs3742 // swap lhs and rhs
3704 return try self.binOpImmediateNew(mir_tag, rhs_bind, imm, rhs_ty, true, maybe_inst);3743 return try self.binOpImmediate(mir_tag, rhs_bind, imm, rhs_ty, true, maybe_inst);
3705 } else {3744 } else {
3706 return try self.binOpRegisterNew(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);3745 return try self.binOpRegister(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
3707 }3746 }
3708 },3747 },
3709 else => unreachable,3748 else => unreachable,