authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-20 23:02:31-04:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-03-21 08:49:54+01:00
logc58b5732f381fe4f145537501e29347be01e2a44
tree10dbb755d5323d9ca4fd3ddfb88c78a8bd980ce7
parentf316cb29cc094c37be191f1eb72ee70eb0dc99ee

x86_64: implement @byteSwap and @bitReverse

7 files changed, 388 insertions(+), 150 deletions(-)

src/arch/x86_64/CodeGen.zig+243-74
......@@ -2595,10 +2595,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
25952595 try self.asmRegisterMemory(
25962596 .mov,
25972597 registerAlias(dst_mcv.register, elem_abi_size),
2598 Memory.sib(Memory.PtrSize.fromSize(elem_abi_size), .{
2599 .base = dst_mcv.register,
2600 .disp = 0,
2601 }),
2598 Memory.sib(Memory.PtrSize.fromSize(elem_abi_size), .{ .base = dst_mcv.register }),
26022599 );
26032600 break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
26042601 }
......@@ -2956,21 +2953,197 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
29562953 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
29572954}
29582955
2956fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, mem_ok: bool) !MCValue {
2957 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2958
2959 const src_bits = self.regBitSize(src_ty);
2960 const src_lock = switch (src_mcv) {
2961 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2962 else => null,
2963 };
2964 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
2965
2966 switch (src_bits) {
2967 else => unreachable,
2968 8 => return if ((mem_ok or src_mcv.isRegister()) and
2969 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2970 src_mcv
2971 else
2972 try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv),
2973 16 => if ((mem_ok or src_mcv.isRegister()) and
2974 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2975 {
2976 try self.genBinOpMir(.rol, src_ty, src_mcv, .{ .immediate = 8 });
2977 return src_mcv;
2978 },
2979 32, 64 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
2980 try self.genUnOpMir(.bswap, src_ty, src_mcv);
2981 return src_mcv;
2982 },
2983 }
2984
2985 if (src_mcv.isRegister()) {
2986 const dst_mcv: MCValue = if (mem_ok)
2987 try self.allocRegOrMem(inst, true)
2988 else
2989 .{ .register = try self.register_manager.allocReg(inst, gp) };
2990 if (dst_mcv.isRegister()) {
2991 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);
2992 defer self.register_manager.unlockReg(dst_lock);
2993
2994 try self.genSetReg(src_ty, dst_mcv.register, src_mcv);
2995 switch (src_bits) {
2996 else => unreachable,
2997 16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),
2998 32, 64 => try self.genUnOpMir(.bswap, src_ty, dst_mcv),
2999 }
3000 } else try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
3001 return dst_mcv;
3002 }
3003
3004 const dst_reg = try self.register_manager.allocReg(inst, gp);
3005 const dst_mcv = MCValue{ .register = dst_reg };
3006 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
3007 defer self.register_manager.unlockReg(dst_lock);
3008
3009 try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
3010 return dst_mcv;
3011}
3012
29593013fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
29603014 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2961 const result: MCValue = if (self.liveness.isUnused(inst))
2962 .dead
2963 else
2964 return self.fail("TODO implement airByteSwap for {}", .{self.target.cpu.arch});
3015 const result = result: {
3016 if (self.liveness.isUnused(inst)) break :result .dead;
3017
3018 const src_ty = self.air.typeOf(ty_op.operand);
3019 const src_mcv = try self.resolveInst(ty_op.operand);
3020
3021 const dst_mcv = try self.byteSwap(inst, src_ty, src_mcv, true);
3022 switch (self.regExtraBits(src_ty)) {
3023 0 => {},
3024 else => |extra| try self.genBinOpMir(
3025 if (src_ty.isSignedInt()) .sar else .shr,
3026 src_ty,
3027 dst_mcv,
3028 .{ .immediate = extra },
3029 ),
3030 }
3031 break :result dst_mcv;
3032 };
3033
29653034 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
29663035}
29673036
29683037fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
29693038 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2970 const result: MCValue = if (self.liveness.isUnused(inst))
2971 .dead
2972 else
2973 return self.fail("TODO implement airBitReverse for {}", .{self.target.cpu.arch});
3039 const result = result: {
3040 if (self.liveness.isUnused(inst)) break :result .dead;
3041
3042 const src_ty = self.air.typeOf(ty_op.operand);
3043 const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
3044 const src_mcv = try self.resolveInst(ty_op.operand);
3045
3046 const dst_mcv = try self.byteSwap(inst, src_ty, src_mcv, false);
3047 const dst_reg = dst_mcv.register;
3048 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
3049 defer self.register_manager.unlockReg(dst_lock);
3050
3051 const tmp_reg = try self.register_manager.allocReg(null, gp);
3052 const tmp_lock = self.register_manager.lockReg(tmp_reg);
3053 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
3054
3055 {
3056 const dst = registerAlias(dst_reg, src_abi_size);
3057 const tmp = registerAlias(tmp_reg, src_abi_size);
3058 const imm = if (src_abi_size > 4)
3059 try self.register_manager.allocReg(null, gp)
3060 else
3061 undefined;
3062
3063 const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - src_abi_size * 8);
3064 const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
3065 const imm_00_11 = Immediate.u(mask / 0b01_01);
3066 const imm_0_1 = Immediate.u(mask / 0b1_1);
3067
3068 // dst = temp1 = bswap(operand)
3069 try self.asmRegisterRegister(.mov, tmp, dst);
3070 // tmp = temp1
3071 try self.asmRegisterImmediate(.shr, dst, Immediate.u(4));
3072 // dst = temp1 >> 4
3073 if (src_abi_size > 4) {
3074 try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
3075 try self.asmRegisterRegister(.@"and", tmp, imm);
3076 try self.asmRegisterRegister(.@"and", dst, imm);
3077 } else {
3078 try self.asmRegisterImmediate(.@"and", tmp, imm_0000_1111);
3079 try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
3080 }
3081 // tmp = temp1 & 0x0F...0F
3082 // dst = (temp1 >> 4) & 0x0F...0F
3083 try self.asmRegisterImmediate(.shl, tmp, Immediate.u(4));
3084 // tmp = (temp1 & 0x0F...0F) << 4
3085 try self.asmRegisterRegister(.@"or", dst, tmp);
3086 // dst = temp2 = ((temp1 >> 4) & 0x0F...0F) | ((temp1 & 0x0F...0F) << 4)
3087 try self.asmRegisterRegister(.mov, tmp, dst);
3088 // tmp = temp2
3089 try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
3090 // dst = temp2 >> 2
3091 if (src_abi_size > 4) {
3092 try self.asmRegisterImmediate(.mov, imm, imm_00_11);
3093 try self.asmRegisterRegister(.@"and", tmp, imm);
3094 try self.asmRegisterRegister(.@"and", dst, imm);
3095 } else {
3096 try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
3097 try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
3098 }
3099 // tmp = temp2 & 0x33...33
3100 // dst = (temp2 >> 2) & 0x33...33
3101 try self.asmRegisterMemory(
3102 .lea,
3103 if (src_abi_size > 4) tmp.to64() else tmp.to32(),
3104 Memory.sib(.qword, .{
3105 .base = dst.to64(),
3106 .scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 },
3107 }),
3108 );
3109 // tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
3110 try self.asmRegisterRegister(.mov, dst, tmp);
3111 // dst = temp3
3112 try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
3113 // tmp = temp3 >> 1
3114 if (src_abi_size > 4) {
3115 try self.asmRegisterImmediate(.mov, imm, imm_0_1);
3116 try self.asmRegisterRegister(.@"and", dst, imm);
3117 try self.asmRegisterRegister(.@"and", tmp, imm);
3118 } else {
3119 try self.asmRegisterImmediate(.@"and", dst, imm_0_1);
3120 try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
3121 }
3122 // dst = temp3 & 0x55...55
3123 // tmp = (temp3 >> 1) & 0x55...55
3124 try self.asmRegisterMemory(
3125 .lea,
3126 if (src_abi_size > 4) dst.to64() else dst.to32(),
3127 Memory.sib(.qword, .{
3128 .base = tmp.to64(),
3129 .scale_index = .{ .index = dst.to64(), .scale = 1 << 1 },
3130 }),
3131 );
3132 // dst = ((temp3 >> 1) & 0x55...55) + ((temp3 & 0x55...55) << 1)
3133 }
3134
3135 switch (self.regExtraBits(src_ty)) {
3136 0 => {},
3137 else => |extra| try self.genBinOpMir(
3138 if (src_ty.isSignedInt()) .sar else .shr,
3139 src_ty,
3140 dst_mcv,
3141 .{ .immediate = extra },
3142 ),
3143 }
3144 break :result dst_mcv;
3145 };
3146
29743147 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
29753148}
29763149
......@@ -3052,7 +3225,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
30523225 try self.asmRegisterMemory(
30533226 .mov,
30543227 registerAlias(dst_reg, abi_size),
3055 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg, .disp = 0 }),
3228 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg }),
30563229 );
30573230 },
30583231 .stack_offset => |off| {
......@@ -3167,7 +3340,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
31673340 .eflags => |cc| {
31683341 try self.asmSetccMemory(Memory.sib(
31693342 Memory.PtrSize.fromSize(abi_size),
3170 .{ .base = reg.to64(), .disp = 0 },
3343 .{ .base = reg.to64() },
31713344 ), cc);
31723345 },
31733346 .undef => {
......@@ -3187,10 +3360,10 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
31873360 Immediate.s(@intCast(i32, @bitCast(i64, imm)))
31883361 else
31893362 Immediate.u(@truncate(u32, imm));
3190 try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
3191 .base = reg.to64(),
3192 .disp = 0,
3193 }), immediate);
3363 try self.asmMemoryImmediate(.mov, Memory.sib(
3364 Memory.PtrSize.fromSize(abi_size),
3365 .{ .base = reg.to64() },
3366 ), immediate);
31943367 },
31953368 8 => {
31963369 // TODO: optimization: if the imm is only using the lower
......@@ -3262,10 +3435,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
32623435 try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);
32633436
32643437 // To get the actual address of the value we want to modify we have to go through the GOT
3265 try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
3266 .base = addr_reg.to64(),
3267 .disp = 0,
3268 }));
3438 try self.asmRegisterMemory(
3439 .mov,
3440 addr_reg.to64(),
3441 Memory.sib(.qword, .{ .base = addr_reg.to64() }),
3442 );
32693443
32703444 const new_ptr = MCValue{ .register = addr_reg.to64() };
32713445
......@@ -3287,10 +3461,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
32873461 return self.fail("TODO imm64 would get incorrectly sign extended", .{});
32883462 }
32893463 }
3290 try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
3291 .base = addr_reg.to64(),
3292 .disp = 0,
3293 }), Immediate.u(@intCast(u32, imm)));
3464 try self.asmMemoryImmediate(
3465 .mov,
3466 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg.to64() }),
3467 Immediate.u(@intCast(u32, imm)),
3468 );
32943469 },
32953470 .register => {
32963471 return self.store(new_ptr, value, ptr_ty, value_ty);
......@@ -3302,10 +3477,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
33023477 defer self.register_manager.unlockReg(tmp_reg_lock);
33033478
33043479 try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);
3305 try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(.qword, .{
3306 .base = tmp_reg,
3307 .disp = 0,
3308 }));
3480 try self.asmRegisterMemory(
3481 .mov,
3482 tmp_reg,
3483 Memory.sib(.qword, .{ .base = tmp_reg }),
3484 );
33093485
33103486 return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
33113487 }
......@@ -3604,15 +3780,16 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue
36043780 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_mcv);
36053781
36063782 // To get the actual address of the value we want to modify we have to go through the GOT
3607 try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{
3608 .base = addr_reg,
3609 .disp = 0,
3610 }));
3783 try self.asmRegisterMemory(
3784 .mov,
3785 addr_reg,
3786 Memory.sib(.qword, .{ .base = addr_reg }),
3787 );
36113788
3612 try self.asmMemory(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
3613 .base = addr_reg,
3614 .disp = 0,
3615 }));
3789 try self.asmMemory(
3790 mir_tag,
3791 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
3792 );
36163793 },
36173794 }
36183795}
......@@ -4117,17 +4294,15 @@ fn genBinOp(
41174294
41184295 // To get the actual address of the value we want to modify we
41194296 // we have to go through the GOT
4120 try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{
4121 .base = addr_reg,
4122 .disp = 0,
4123 }));
4297 try self.asmRegisterMemory(
4298 .mov,
4299 addr_reg,
4300 Memory.sib(.qword, .{ .base = addr_reg }),
4301 );
41244302
41254303 try self.asmCmovccRegisterMemory(
41264304 registerAlias(dst_reg, abi_size),
4127 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
4128 .base = addr_reg,
4129 .disp = 0,
4130 }),
4305 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
41314306 cc,
41324307 );
41334308 },
......@@ -5175,10 +5350,11 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
51755350 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, opt_mcv);
51765351
51775352 // To get the actual address of the value we want to modify we have to go through the GOT
5178 try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{
5179 .base = addr_reg,
5180 .disp = 0,
5181 }));
5353 try self.asmRegisterMemory(
5354 .mov,
5355 addr_reg,
5356 Memory.sib(.qword, .{ .base = addr_reg }),
5357 );
51825358
51835359 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
51845360 try self.asmMemoryImmediate(.cmp, Memory.sib(
......@@ -6374,10 +6550,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
63746550 .f64 => .qword,
63756551 else => unreachable,
63766552 };
6377 return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
6378 .base = base_reg.to64(),
6379 .disp = 0,
6380 }));
6553 return self.asmRegisterMemory(
6554 tag,
6555 reg.to128(),
6556 Memory.sib(ptr_size, .{ .base = base_reg.to64() }),
6557 );
63816558 }
63826559
63836560 return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
......@@ -6387,7 +6564,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
63876564 try self.asmRegisterMemory(
63886565 .mov,
63896566 registerAlias(reg, abi_size),
6390 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }),
6567 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
63916568 );
63926569 },
63936570 }
......@@ -6408,10 +6585,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
64086585 .f64 => .qword,
64096586 else => unreachable,
64106587 };
6411 return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
6412 .base = base_reg.to64(),
6413 .disp = 0,
6414 }));
6588 return self.asmRegisterMemory(
6589 tag,
6590 reg.to128(),
6591 Memory.sib(ptr_size, .{ .base = base_reg.to64() }),
6592 );
64156593 }
64166594
64176595 return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
......@@ -6447,7 +6625,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
64476625 try self.asmRegisterMemory(
64486626 .mov,
64496627 registerAlias(reg, abi_size),
6450 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }),
6628 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
64516629 );
64526630 }
64536631 }
......@@ -6638,12 +6816,9 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
66386816 const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
66396817 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
66406818 const ptr_mem: Memory = switch (ptr_mcv) {
6641 .register => |reg| Memory.sib(ptr_size, .{ .base = reg, .disp = 0 }),
6819 .register => |reg| Memory.sib(ptr_size, .{ .base = reg }),
66426820 .ptr_stack_offset => |off| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
6643 else => Memory.sib(ptr_size, .{
6644 .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
6645 .disp = 0,
6646 }),
6821 else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),
66476822 };
66486823 const mem_lock = if (ptr_mem.base()) |reg| self.register_manager.lockReg(reg) else null;
66496824 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
......@@ -6692,12 +6867,9 @@ fn atomicOp(
66926867 const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
66936868 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
66946869 const ptr_mem: Memory = switch (ptr_mcv) {
6695 .register => |reg| Memory.sib(ptr_size, .{ .base = reg, .disp = 0 }),
6870 .register => |reg| Memory.sib(ptr_size, .{ .base = reg }),
66966871 .ptr_stack_offset => |off| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
6697 else => Memory.sib(ptr_size, .{
6698 .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
6699 .disp = 0,
6700 }),
6872 else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),
67016873 };
67026874 const mem_lock = if (ptr_mem.base()) |reg| self.register_manager.lockReg(reg) else null;
67036875 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
......@@ -6861,10 +7033,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
68617033 .linker_load, .memory => {
68627034 const reg = try self.register_manager.allocReg(null, gp);
68637035 try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
6864 try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{
6865 .base = reg,
6866 .disp = 0,
6867 }));
7036 try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{ .base = reg }));
68687037 break :blk MCValue{ .register = reg };
68697038 },
68707039 else => break :blk src_ptr,
src/arch/x86_64/Emit.zig+6
......@@ -75,6 +75,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
7575 .@"and",
7676 .bsf,
7777 .bsr,
78 .bswap,
7879 .bt,
7980 .btc,
8081 .btr,
......@@ -100,6 +101,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
100101 .lzcnt,
101102 .mfence,
102103 .mov,
104 .movbe,
103105 .movzx,
104106 .mul,
105107 .neg,
......@@ -109,7 +111,11 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
109111 .pop,
110112 .popcnt,
111113 .push,
114 .rcl,
115 .rcr,
112116 .ret,
117 .rol,
118 .ror,
113119 .sal,
114120 .sar,
115121 .sbb,
src/arch/x86_64/Encoding.zig+3-3
......@@ -307,7 +307,7 @@ pub const Mnemonic = enum {
307307 // zig fmt: off
308308 // General-purpose
309309 adc, add, @"and",
310 bsf, bsr, bt, btc, btr, bts,
310 bsf, bsr, bswap, bt, btc, btr, bts,
311311 call, cbw, cdq, cdqe,
312312 cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
313313 cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
......@@ -325,13 +325,13 @@ pub const Mnemonic = enum {
325325 lea, lfence,
326326 lods, lodsb, lodsd, lodsq, lodsw,
327327 lzcnt,
328 mfence, mov,
328 mfence, mov, movbe,
329329 movs, movsb, movsd, movsq, movsw,
330330 movsx, movsxd, movzx, mul,
331331 neg, nop, not,
332332 @"or",
333333 pop, popcnt, push,
334 ret,
334 rcl, rcr, ret, rol, ror,
335335 sal, sar, sbb,
336336 scas, scasb, scasd, scasq, scasw,
337337 shl, shr, sub, syscall,
src/arch/x86_64/Mir.zig+12
......@@ -42,6 +42,8 @@ pub const Inst = struct {
4242 bsf,
4343 /// Bit scan reverse
4444 bsr,
45 /// Byte swap
46 bswap,
4547 /// Bit test
4648 bt,
4749 /// Bit test and complement
......@@ -94,6 +96,8 @@ pub const Inst = struct {
9496 mfence,
9597 /// Move
9698 mov,
99 /// Move data after swapping bytes
100 movbe,
97101 /// Move with sign extension
98102 movsx,
99103 /// Move with zero extension
......@@ -114,8 +118,16 @@ pub const Inst = struct {
114118 popcnt,
115119 /// Push
116120 push,
121 /// Rotate left through carry
122 rcl,
123 /// Rotate right through carry
124 rcr,
117125 /// Return
118126 ret,
127 /// Rotate left
128 rol,
129 /// Rotate right
130 ror,
119131 /// Arithmetic shift left
120132 sal,
121133 /// Arithmetic shift right
src/arch/x86_64/bits.zig+1-1
......@@ -472,7 +472,7 @@ pub const Memory = union(enum) {
472472 }
473473
474474 pub fn sib(ptr_size: PtrSize, args: struct {
475 disp: i32,
475 disp: i32 = 0,
476476 base: ?Register = null,
477477 scale_index: ?ScaleIndex = null,
478478 }) Memory {
src/arch/x86_64/encoder.zig+49-72
......@@ -211,14 +211,12 @@ pub const Instruction = struct {
211211
212212 fn encodeOpcode(inst: Instruction, encoder: anytype) !void {
213213 const opcode = inst.encoding.opcode();
214 const first = @boolToInt(inst.encoding.mandatoryPrefix() != null);
215 const final = opcode.len - 1;
216 for (opcode[first..final]) |byte| try encoder.opcode_1byte(byte);
214217 switch (inst.encoding.op_en) {
215 .o, .oi => try encoder.opcode_withReg(opcode[0], inst.op1.reg.lowEnc()),
216 else => {
217 const index: usize = if (inst.encoding.mandatoryPrefix()) |_| 1 else 0;
218 for (opcode[index..]) |byte| {
219 try encoder.opcode_1byte(byte);
220 }
221 },
218 .o, .oi => try encoder.opcode_withReg(opcode[final], inst.op1.reg.lowEnc()),
219 else => try encoder.opcode_1byte(opcode[final]),
222220 }
223221 }
224222
......@@ -896,10 +894,10 @@ test "lower MI encoding" {
896894 try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
897895 try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
898896
899 try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
900 .base = .r12,
901 .disp = 0,
902 }) }, .op2 = .{ .imm = Immediate.u(0x10) } });
897 try enc.encode(.mov, .{
898 .op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12 }) },
899 .op2 = .{ .imm = Immediate.u(0x10) },
900 });
903901 try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10");
904902
905903 try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
......@@ -911,10 +909,10 @@ test "lower MI encoding" {
911909 try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } });
912910 try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10");
913911
914 try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
915 .base = .r11,
916 .disp = 0,
917 }) }, .op2 = .{ .imm = Immediate.u(0x10) } });
912 try enc.encode(.mov, .{
913 .op1 = .{ .mem = Memory.sib(.dword, .{ .base = .r11 }) },
914 .op2 = .{ .imm = Immediate.u(0x10) },
915 });
918916 try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10");
919917
920918 try enc.encode(.mov, .{
......@@ -1030,10 +1028,10 @@ test "lower MI encoding" {
10301028test "lower RM encoding" {
10311029 var enc = TestEncode{};
10321030
1033 try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{
1034 .base = .r11,
1035 .disp = 0,
1036 }) } });
1031 try enc.encode(.mov, .{
1032 .op1 = .{ .reg = .rax },
1033 .op2 = .{ .mem = Memory.sib(.qword, .{ .base = .r11 }) },
1034 });
10371035 try expectEqualHexStrings("\x49\x8b\x03", enc.code(), "mov rax, QWORD PTR [r11]");
10381036
10391037 try enc.encode(.mov, .{ .op1 = .{ .reg = .rbx }, .op2 = .{ .mem = Memory.sib(.qword, .{
......@@ -1116,20 +1114,16 @@ test "lower RM encoding" {
11161114 try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .reg = .bl } });
11171115 try expectEqualHexStrings("\x66\x0F\xBE\xC3", enc.code(), "movsx ax, bl");
11181116
1119 try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.word, .{
1120 .base = .rbp,
1121 .disp = 0,
1122 }) } });
1117 try enc.encode(.movsx, .{
1118 .op1 = .{ .reg = .eax },
1119 .op2 = .{ .mem = Memory.sib(.word, .{ .base = .rbp }) },
1120 });
11231121 try expectEqualHexStrings("\x0F\xBF\x45\x00", enc.code(), "movsx eax, BYTE PTR [rbp]");
11241122
1125 try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.byte, .{
1126 .base = null,
1127 .scale_index = .{
1128 .index = .rax,
1129 .scale = 2,
1130 },
1131 .disp = 0,
1132 }) } });
1123 try enc.encode(.movsx, .{
1124 .op1 = .{ .reg = .eax },
1125 .op2 = .{ .mem = Memory.sib(.byte, .{ .scale_index = .{ .index = .rax, .scale = 2 } }) },
1126 });
11331127 try expectEqualHexStrings("\x0F\xBE\x04\x45\x00\x00\x00\x00", enc.code(), "movsx eax, BYTE PTR [rax * 2]");
11341128
11351129 try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } });
......@@ -1156,14 +1150,13 @@ test "lower RM encoding" {
11561150 try enc.encode(.lea, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } });
11571151 try expectEqualHexStrings("\x66\x8D\x05\x10\x00\x00\x00", enc.code(), "lea ax, BYTE PTR [rip + 0x10]");
11581152
1159 try enc.encode(.lea, .{ .op1 = .{ .reg = .rsi }, .op2 = .{ .mem = Memory.sib(.qword, .{
1160 .base = .rbp,
1161 .scale_index = .{
1162 .scale = 1,
1163 .index = .rcx,
1164 },
1165 .disp = 0,
1166 }) } });
1153 try enc.encode(.lea, .{
1154 .op1 = .{ .reg = .rsi },
1155 .op2 = .{ .mem = Memory.sib(.qword, .{
1156 .base = .rbp,
1157 .scale_index = .{ .scale = 1, .index = .rcx },
1158 }) },
1159 });
11671160 try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", enc.code(), "lea rsi, QWORD PTR [rbp + rcx*1 + 0]");
11681161
11691162 try enc.encode(.add, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .mem = Memory.sib(.qword, .{
......@@ -1319,51 +1312,35 @@ test "lower M encoding" {
13191312 try enc.encode(.call, .{ .op1 = .{ .reg = .r12 } });
13201313 try expectEqualHexStrings("\x41\xFF\xD4", enc.code(), "call r12");
13211314
1322 try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1323 .base = .r12,
1324 .disp = 0,
1325 }) } });
1315 try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .r12 }) } });
13261316 try expectEqualHexStrings("\x41\xFF\x14\x24", enc.code(), "call QWORD PTR [r12]");
13271317
1328 try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1329 .base = null,
1330 .scale_index = .{
1331 .index = .r11,
1332 .scale = 2,
1333 },
1334 .disp = 0,
1335 }) } });
1318 try enc.encode(.call, .{
1319 .op1 = .{ .mem = Memory.sib(.qword, .{
1320 .base = null,
1321 .scale_index = .{ .index = .r11, .scale = 2 },
1322 }) },
1323 });
13361324 try expectEqualHexStrings("\x42\xFF\x14\x5D\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r11 * 2]");
13371325
1338 try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1339 .base = null,
1340 .scale_index = .{
1341 .index = .r12,
1342 .scale = 2,
1343 },
1344 .disp = 0,
1345 }) } });
1326 try enc.encode(.call, .{
1327 .op1 = .{ .mem = Memory.sib(.qword, .{
1328 .base = null,
1329 .scale_index = .{ .index = .r12, .scale = 2 },
1330 }) },
1331 });
13461332 try expectEqualHexStrings("\x42\xFF\x14\x65\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r12 * 2]");
13471333
1348 try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1349 .base = .gs,
1350 .disp = 0,
1351 }) } });
1334 try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .gs }) } });
13521335 try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0");
13531336
13541337 try enc.encode(.call, .{ .op1 = .{ .imm = Immediate.s(0) } });
13551338 try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0");
13561339
1357 try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1358 .base = .rbp,
1359 .disp = 0,
1360 }) } });
1340 try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .rbp }) } });
13611341 try expectEqualHexStrings("\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
13621342
1363 try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.word, .{
1364 .base = .rbp,
1365 .disp = 0,
1366 }) } });
1343 try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.word, .{ .base = .rbp }) } });
13671344 try expectEqualHexStrings("\x66\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
13681345
13691346 try enc.encode(.pop, .{ .op1 = .{ .mem = Memory.rip(.qword, 0) } });
......@@ -1491,7 +1468,7 @@ fn cannotEncode(mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void {
14911468
14921469test "cannot encode" {
14931470 try cannotEncode(.@"test", .{
1494 .op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12, .disp = 0 }) },
1471 .op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12 }) },
14951472 .op2 = .{ .reg = .ah },
14961473 });
14971474 try cannotEncode(.@"test", .{
src/arch/x86_64/encodings.zig+74
......@@ -89,6 +89,9 @@ pub const table = &[_]Entry{
8989 .{ .bsr, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xbd }, 0, .none },
9090 .{ .bsr, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xbd }, 0, .long },
9191
92 .{ .bswap, .o, .r32, .none, .none, .none, &.{ 0x0f, 0xc8 }, 0, .none },
93 .{ .bswap, .o, .r64, .none, .none, .none, &.{ 0x0f, 0xc8 }, 0, .long },
94
9295 .{ .bt, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xa3 }, 0, .none },
9396 .{ .bt, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xa3 }, 0, .none },
9497 .{ .bt, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xa3 }, 0, .long },
......@@ -387,6 +390,13 @@ pub const table = &[_]Entry{
387390 .{ .mov, .mi, .rm32, .imm32, .none, .none, &.{ 0xc7 }, 0, .none },
388391 .{ .mov, .mi, .rm64, .imm32s, .none, .none, &.{ 0xc7 }, 0, .long },
389392
393 .{ .movbe, .rm, .r16, .m16, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },
394 .{ .movbe, .rm, .r32, .m32, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },
395 .{ .movbe, .rm, .r64, .m64, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .long },
396 .{ .movbe, .mr, .m16, .r16, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },
397 .{ .movbe, .mr, .m32, .r32, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },
398 .{ .movbe, .mr, .m64, .r64, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .long },
399
390400 .{ .movs, .np, .m8, .m8, .none, .none, &.{ 0xa4 }, 0, .none },
391401 .{ .movs, .np, .m16, .m16, .none, .none, &.{ 0xa5 }, 0, .none },
392402 .{ .movs, .np, .m32, .m32, .none, .none, &.{ 0xa5 }, 0, .none },
......@@ -476,6 +486,70 @@ pub const table = &[_]Entry{
476486
477487 .{ .ret, .np, .none, .none, .none, .none, &.{ 0xc3 }, 0, .none },
478488
489 .{ .rcl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 2, .none },
490 .{ .rcl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 2, .rex },
491 .{ .rcl, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 2, .none },
492 .{ .rcl, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 2, .rex },
493 .{ .rcl, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 2, .none },
494 .{ .rcl, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 2, .rex },
495 .{ .rcl, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 2, .none },
496 .{ .rcl, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 2, .none },
497 .{ .rcl, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 2, .none },
498 .{ .rcl, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 2, .none },
499 .{ .rcl, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 2, .long },
500 .{ .rcl, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 2, .none },
501 .{ .rcl, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 2, .long },
502 .{ .rcl, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 2, .none },
503 .{ .rcl, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 2, .long },
504
505 .{ .rcr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 3, .none },
506 .{ .rcr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 3, .rex },
507 .{ .rcr, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 3, .none },
508 .{ .rcr, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 3, .rex },
509 .{ .rcr, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 3, .none },
510 .{ .rcr, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 3, .rex },
511 .{ .rcr, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 3, .none },
512 .{ .rcr, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 3, .none },
513 .{ .rcr, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 3, .none },
514 .{ .rcr, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 3, .none },
515 .{ .rcr, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 3, .long },
516 .{ .rcr, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 3, .none },
517 .{ .rcr, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 3, .long },
518 .{ .rcr, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 3, .none },
519 .{ .rcr, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 3, .long },
520
521 .{ .rol, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 0, .none },
522 .{ .rol, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 0, .rex },
523 .{ .rol, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 0, .none },
524 .{ .rol, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 0, .rex },
525 .{ .rol, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 0, .none },
526 .{ .rol, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 0, .rex },
527 .{ .rol, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 0, .none },
528 .{ .rol, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 0, .none },
529 .{ .rol, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 0, .none },
530 .{ .rol, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 0, .none },
531 .{ .rol, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 0, .long },
532 .{ .rol, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 0, .none },
533 .{ .rol, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 0, .long },
534 .{ .rol, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 0, .none },
535 .{ .rol, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 0, .long },
536
537 .{ .ror, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 1, .none },
538 .{ .ror, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 1, .rex },
539 .{ .ror, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 1, .none },
540 .{ .ror, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 1, .rex },
541 .{ .ror, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 1, .none },
542 .{ .ror, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 1, .rex },
543 .{ .ror, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 1, .none },
544 .{ .ror, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 1, .none },
545 .{ .ror, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 1, .none },
546 .{ .ror, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 1, .none },
547 .{ .ror, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 1, .long },
548 .{ .ror, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 1, .none },
549 .{ .ror, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 1, .long },
550 .{ .ror, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 1, .none },
551 .{ .ror, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 1, .long },
552
479553 .{ .sal, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .none },
480554 .{ .sal, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .rex },
481555 .{ .sal, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 4, .none },