| author |  Jacob Young <jacobly@ziglang.org> 2025-02-15 03:45:08-05:00 |
| committer | Jacob Young <jacobly@ziglang.org> 2025-02-15 04:13:21-05:00 |
| log | dcc9fe322e16fd01248a3fe5848604c65980354e |
| tree | b3b99935a516b3aff1e58d5d9cdae7b52393eefe |
| parent | 5db585fcde56e78eccd79444ba53b59656c857ae |
x86_64: rewrite unsafe scalar int multiplication
8 files changed, 616 insertions(+), 151 deletions(-)
src/arch/x86_64/CodeGen.zig+385-5
| ... | ... | @@ -6036,10 +6036,377 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 6036 | 6036 | .sub_safe => unreachable, |
| 6037 | 6037 | .mul, .mul_optimized => |air_tag| if (use_old) try cg.airMulDivBinOp(inst, .mul) else fallback: { |
| 6038 | 6038 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; |
| 6039 | if (cg.floatBits(cg.typeOf(bin_op.lhs).scalarType(zcu)) == null) break :fallback try cg.airMulDivBinOp(inst, .mul); | |
| 6039 | const ty = cg.typeOf(bin_op.lhs); | |
| 6040 | if (ty.isVector(zcu) and cg.floatBits(ty.childType(zcu)) == null) break :fallback try cg.airMulDivBinOp(inst, .mul); | |
| 6040 | 6041 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 6041 | 6042 | var res: [1]Temp = undefined; |
| 6042 | cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ | |
| 6043 | cg.select(&res, &.{ty}, &ops, comptime &.{ .{ | |
| 6044 | .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, | |
| 6045 | .patterns = &.{ | |
| 6046 | .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, | |
| 6047 | .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, | |
| 6048 | .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, | |
| 6049 | }, | |
| 6050 | .dst_temps = .{ .{ .ref = .src0 }, .unused }, | |
| 6051 | .clobbers = .{ .eflags = true }, | |
| 6052 | .each = .{ .once = &.{ | |
| 6053 | .{ ._, .i_, .mul, .src1b, ._, ._, ._ }, | |
| 6054 | } }, | |
| 6055 | }, .{ | |
| 6056 | .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, | |
| 6057 | .patterns = &.{ | |
| 6058 | .{ .src = .{ .{ .to_reg = .al }, .mem, .none } }, | |
| 6059 | .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } }, | |
| 6060 | .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } }, | |
| 6061 | }, | |
| 6062 | .dst_temps = .{ .{ .ref = .src0 }, .unused }, | |
| 6063 | .clobbers = .{ .eflags = true }, | |
| 6064 | .each = .{ .once = &.{ | |
| 6065 | .{ ._, ._, .mul, .src1b, ._, ._, ._ }, | |
| 6066 | } }, | |
| 6067 | }, .{ | |
| 6068 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, | |
| 6069 | .patterns = &.{ | |
| 6070 | .{ .src = .{ .mem, .imm16, .none } }, | |
| 6071 | .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } }, | |
| 6072 | .{ .src = .{ .to_gpr, .imm16, .none } }, | |
| 6073 | .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } }, | |
| 6074 | }, | |
| 6075 | .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, | |
| 6076 | .clobbers = .{ .eflags = true }, | |
| 6077 | .each = .{ .once = &.{ | |
| 6078 | .{ ._, .i_, .mul, .dst0w, .src0w, .src1w, ._ }, | |
| 6079 | } }, | |
| 6080 | }, .{ | |
| 6081 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any }, | |
| 6082 | .patterns = &.{ | |
| 6083 | .{ .src = .{ .to_mut_gpr, .mem, .none } }, | |
| 6084 | .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, | |
| 6085 | .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, | |
| 6086 | }, | |
| 6087 | .dst_temps = .{ .{ .ref = .src0 }, .unused }, | |
| 6088 | .clobbers = .{ .eflags = true }, | |
| 6089 | .each = .{ .once = &.{ | |
| 6090 | .{ ._, .i_, .mul, .dst0w, .src1w, ._, ._ }, | |
| 6091 | } }, | |
| 6092 | }, .{ | |
| 6093 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, | |
| 6094 | .patterns = &.{ | |
| 6095 | .{ .src = .{ .mem, .imm32, .none } }, | |
| 6096 | .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } }, | |
| 6097 | .{ .src = .{ .to_gpr, .imm32, .none } }, | |
| 6098 | .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, | |
| 6099 | }, | |
| 6100 | .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, | |
| 6101 | .clobbers = .{ .eflags = true }, | |
| 6102 | .each = .{ .once = &.{ | |
| 6103 | .{ ._, .i_, .mul, .dst0d, .src0d, .src1d, ._ }, | |
| 6104 | } }, | |
| 6105 | }, .{ | |
| 6106 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any }, | |
| 6107 | .patterns = &.{ | |
| 6108 | .{ .src = .{ .to_mut_gpr, .mem, .none } }, | |
| 6109 | .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, | |
| 6110 | .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, | |
| 6111 | }, | |
| 6112 | .dst_temps = .{ .{ .ref = .src0 }, .unused }, | |
| 6113 | .clobbers = .{ .eflags = true }, | |
| 6114 | .each = .{ .once = &.{ | |
| 6115 | .{ ._, .i_, .mul, .dst0d, .src1d, ._, ._ }, | |
| 6116 | } }, | |
| 6117 | }, .{ | |
| 6118 | .required_features = .{ .@"64bit", null, null, null }, | |
| 6119 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, | |
| 6120 | .patterns = &.{ | |
| 6121 | .{ .src = .{ .mem, .simm32, .none } }, | |
| 6122 | .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } }, | |
| 6123 | .{ .src = .{ .to_gpr, .simm32, .none } }, | |
| 6124 | .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } }, | |
| 6125 | }, | |
| 6126 | .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused }, | |
| 6127 | .clobbers = .{ .eflags = true }, | |
| 6128 | .each = .{ .once = &.{ | |
| 6129 | .{ ._, .i_, .mul, .dst0q, .src0q, .src1q, ._ }, | |
| 6130 | } }, | |
| 6131 | }, .{ | |
| 6132 | .required_features = .{ .@"64bit", null, null, null }, | |
| 6133 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any }, | |
| 6134 | .patterns = &.{ | |
| 6135 | .{ .src = .{ .to_mut_gpr, .mem, .none } }, | |
| 6136 | .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, | |
| 6137 | .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, | |
| 6138 | }, | |
| 6139 | .dst_temps = .{ .{ .ref = .src0 }, .unused }, | |
| 6140 | .clobbers = .{ .eflags = true }, | |
| 6141 | .each = .{ .once = &.{ | |
| 6142 | .{ ._, .i_, .mul, .dst0q, .src1q, ._, ._ }, | |
| 6143 | } }, | |
| 6144 | }, .{ | |
| 6145 | .required_features = .{ .@"64bit", null, null, null }, | |
| 6146 | .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any }, | |
| 6147 | .patterns = &.{ | |
| 6148 | .{ .src = .{ .to_mem, .to_mem, .none } }, | |
| 6149 | }, | |
| 6150 | .extra_temps = .{ | |
| 6151 | .{ .type = .u64, .kind = .{ .reg = .rax } }, | |
| 6152 | .{ .type = .u64, .kind = .{ .reg = .rdx } }, | |
| 6153 | .unused, | |
| 6154 | .unused, | |
| 6155 | .unused, | |
| 6156 | .unused, | |
| 6157 | .unused, | |
| 6158 | .unused, | |
| 6159 | .unused, | |
| 6160 | }, | |
| 6161 | .dst_temps = .{ .mem, .unused }, | |
| 6162 | .clobbers = .{ .eflags = true }, | |
| 6163 | .each = .{ .once = &.{ | |
| 6164 | .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, | |
| 6165 | .{ ._, ._, .mul, .src1q, ._, ._, ._ }, | |
| 6166 | .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ }, | |
| 6167 | .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ }, | |
| 6168 | .{ ._, .i_, .mul, .tmp0q, .memd(.src1q, 8), ._, ._ }, | |
| 6169 | .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ }, | |
| 6170 | .{ ._, ._, .mov, .tmp0q, .src1q, ._, ._ }, | |
| 6171 | .{ ._, .i_, .mul, .tmp0q, .memd(.src0q, 8), ._, ._ }, | |
| 6172 | .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ }, | |
| 6173 | .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp1q, ._, ._ }, | |
| 6174 | } }, | |
| 6175 | }, .{ | |
| 6176 | .required_features = .{ .@"64bit", .bmi2, .adx, null }, | |
| 6177 | .src_constraints = .{ | |
| 6178 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, | |
| 6179 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, | |
| 6180 | .any, | |
| 6181 | }, | |
| 6182 | .patterns = &.{ | |
| 6183 | .{ .src = .{ .to_mem, .to_mem, .none } }, | |
| 6184 | }, | |
| 6185 | .extra_temps = .{ | |
| 6186 | .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, | |
| 6187 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | |
| 6188 | .{ .type = .u64, .kind = .{ .reg = .rdx } }, | |
| 6189 | .{ .type = .isize, .kind = .{ .reg = .rcx } }, | |
| 6190 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6191 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6192 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6193 | .unused, | |
| 6194 | .unused, | |
| 6195 | }, | |
| 6196 | .dst_temps = .{ .mem, .unused }, | |
| 6197 | .clobbers = .{ .eflags = true }, | |
| 6198 | .each = .{ .once = &.{ | |
| 6199 | .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ }, | |
| 6200 | .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, | |
| 6201 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, | |
| 6202 | .{ ._, ._, .@"or", .tmp2q, .memi(.src0q, .tmp0), ._, ._ }, | |
| 6203 | .{ ._, ._z, .j, .@"2f", ._, ._, ._ }, | |
| 6204 | .{ ._, ._, .lea, .tmp3p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ }, | |
| 6205 | .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, | |
| 6206 | .{ .@"1:", ._x, .mul, .tmp6q, .tmp5q, .leai(.tmp1q, .tmp3), ._ }, | |
| 6207 | .{ ._, ._x, .adc, .tmp5q, .tmp4q, ._, ._ }, | |
| 6208 | .{ ._, ._, .mov, .memiad(.dst0q, .tmp3, .add_size, -8), .tmp5q, ._, ._ }, | |
| 6209 | .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ }, | |
| 6210 | .{ ._, ._x, .ado, .tmp6q, .memia(.dst0q, .tmp3, .add_size), ._, ._ }, | |
| 6211 | .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ }, | |
| 6212 | .{ ._, ._, .lea, .tmp3p, .lead(.tmp3, 8), ._, ._ }, | |
| 6213 | .{ ._, ._mp, .j, .@"1b", ._, ._, ._ }, | |
| 6214 | .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ }, | |
| 6215 | .{ .@"1:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, | |
| 6216 | .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ }, | |
| 6217 | .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, | |
| 6218 | } }, | |
| 6219 | }, .{ | |
| 6220 | .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null }, | |
| 6221 | .src_constraints = .{ | |
| 6222 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, | |
| 6223 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, | |
| 6224 | .any, | |
| 6225 | }, | |
| 6226 | .patterns = &.{ | |
| 6227 | .{ .src = .{ .to_mem, .to_mem, .none } }, | |
| 6228 | }, | |
| 6229 | .extra_temps = .{ | |
| 6230 | .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, | |
| 6231 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | |
| 6232 | .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, | |
| 6233 | .{ .type = .u64, .kind = .{ .reg = .rdx } }, | |
| 6234 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, | |
| 6235 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6236 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6237 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6238 | .unused, | |
| 6239 | }, | |
| 6240 | .dst_temps = .{ .mem, .unused }, | |
| 6241 | .clobbers = .{ .eflags = true }, | |
| 6242 | .each = .{ .once = &.{ | |
| 6243 | .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, | |
| 6244 | .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, | |
| 6245 | .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, | |
| 6246 | .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, | |
| 6247 | .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, | |
| 6248 | .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, | |
| 6249 | .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, | |
| 6250 | .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, | |
| 6251 | .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, | |
| 6252 | .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, | |
| 6253 | .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, | |
| 6254 | .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, | |
| 6255 | .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, | |
| 6256 | .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, | |
| 6257 | .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ }, | |
| 6258 | .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, | |
| 6259 | .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, | |
| 6260 | .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, | |
| 6261 | .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, | |
| 6262 | .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, | |
| 6263 | .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, | |
| 6264 | .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, | |
| 6265 | } }, | |
| 6266 | }, .{ | |
| 6267 | .required_features = .{ .@"64bit", .bmi2, null, null }, | |
| 6268 | .src_constraints = .{ | |
| 6269 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, | |
| 6270 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, | |
| 6271 | .any, | |
| 6272 | }, | |
| 6273 | .patterns = &.{ | |
| 6274 | .{ .src = .{ .to_mem, .to_mem, .none } }, | |
| 6275 | }, | |
| 6276 | .extra_temps = .{ | |
| 6277 | .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, | |
| 6278 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | |
| 6279 | .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, | |
| 6280 | .{ .type = .u64, .kind = .{ .reg = .rdx } }, | |
| 6281 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, | |
| 6282 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6283 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6284 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6285 | .unused, | |
| 6286 | }, | |
| 6287 | .dst_temps = .{ .mem, .unused }, | |
| 6288 | .clobbers = .{ .eflags = true }, | |
| 6289 | .each = .{ .once = &.{ | |
| 6290 | .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, | |
| 6291 | .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, | |
| 6292 | .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, | |
| 6293 | .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, | |
| 6294 | .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, | |
| 6295 | .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, | |
| 6296 | .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, | |
| 6297 | .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, | |
| 6298 | .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, | |
| 6299 | .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, | |
| 6300 | .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, | |
| 6301 | .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, | |
| 6302 | .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, | |
| 6303 | .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, | |
| 6304 | .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ }, | |
| 6305 | .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, | |
| 6306 | .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, | |
| 6307 | .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, | |
| 6308 | .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, | |
| 6309 | .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, | |
| 6310 | .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, | |
| 6311 | .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, | |
| 6312 | } }, | |
| 6313 | }, .{ | |
| 6314 | .required_features = .{ .@"64bit", .slow_incdec, null, null }, | |
| 6315 | .src_constraints = .{ | |
| 6316 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, | |
| 6317 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, | |
| 6318 | .any, | |
| 6319 | }, | |
| 6320 | .patterns = &.{ | |
| 6321 | .{ .src = .{ .to_mem, .to_mem, .none } }, | |
| 6322 | }, | |
| 6323 | .extra_temps = .{ | |
| 6324 | .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, | |
| 6325 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | |
| 6326 | .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, | |
| 6327 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6328 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, | |
| 6329 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6330 | .{ .type = .u64, .kind = .{ .reg = .rax } }, | |
| 6331 | .{ .type = .u64, .kind = .{ .reg = .rdx } }, | |
| 6332 | .unused, | |
| 6333 | }, | |
| 6334 | .dst_temps = .{ .mem, .unused }, | |
| 6335 | .clobbers = .{ .eflags = true }, | |
| 6336 | .each = .{ .once = &.{ | |
| 6337 | .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, | |
| 6338 | .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, | |
| 6339 | .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, | |
| 6340 | .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, | |
| 6341 | .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, | |
| 6342 | .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, | |
| 6343 | .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, | |
| 6344 | .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, | |
| 6345 | .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, | |
| 6346 | .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, | |
| 6347 | .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, | |
| 6348 | .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, | |
| 6349 | .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, | |
| 6350 | .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ }, | |
| 6351 | .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ }, | |
| 6352 | .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, | |
| 6353 | .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, | |
| 6354 | .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, | |
| 6355 | .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, | |
| 6356 | .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, | |
| 6357 | .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, | |
| 6358 | .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ }, | |
| 6359 | .{ ._, ._ae, .j, .@"0b", ._, ._, ._ }, | |
| 6360 | } }, | |
| 6361 | }, .{ | |
| 6362 | .required_features = .{ .@"64bit", null, null, null }, | |
| 6363 | .src_constraints = .{ | |
| 6364 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, | |
| 6365 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, | |
| 6366 | .any, | |
| 6367 | }, | |
| 6368 | .patterns = &.{ | |
| 6369 | .{ .src = .{ .to_mem, .to_mem, .none } }, | |
| 6370 | }, | |
| 6371 | .extra_temps = .{ | |
| 6372 | .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, | |
| 6373 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | |
| 6374 | .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, | |
| 6375 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6376 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, | |
| 6377 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 6378 | .{ .type = .u64, .kind = .{ .reg = .rax } }, | |
| 6379 | .{ .type = .u64, .kind = .{ .reg = .rdx } }, | |
| 6380 | .unused, | |
| 6381 | }, | |
| 6382 | .dst_temps = .{ .mem, .unused }, | |
| 6383 | .clobbers = .{ .eflags = true }, | |
| 6384 | .each = .{ .once = &.{ | |
| 6385 | .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ }, | |
| 6386 | .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ }, | |
| 6387 | .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ }, | |
| 6388 | .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, | |
| 6389 | .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, | |
| 6390 | .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ }, | |
| 6391 | .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ }, | |
| 6392 | .{ ._, ._nz, .j, .@"2f", ._, ._, ._ }, | |
| 6393 | .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ }, | |
| 6394 | .{ ._, ._mp, .j, .@"3f", ._, ._, ._ }, | |
| 6395 | .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ }, | |
| 6396 | .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ }, | |
| 6397 | .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ }, | |
| 6398 | .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ }, | |
| 6399 | .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ }, | |
| 6400 | .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, | |
| 6401 | .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ }, | |
| 6402 | .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ }, | |
| 6403 | .{ ._, ._c, .in, .tmp2p, ._, ._, ._ }, | |
| 6404 | .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, | |
| 6405 | .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ }, | |
| 6406 | .{ ._, ._c, .de, .tmp0d, ._, ._, ._ }, | |
| 6407 | .{ ._, ._ns, .j, .@"0b", ._, ._, ._ }, | |
| 6408 | } }, | |
| 6409 | }, .{ | |
| 6043 | 6410 | .required_features = .{ .f16c, null, null, null }, |
| 6044 | 6411 | .src_constraints = .{ |
| 6045 | 6412 | .{ .scalar_float = .{ .of = .word, .is = .word } }, |
| ... | ... | @@ -6890,7 +7257,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 6890 | 7257 | } }) catch |err| switch (err) { |
| 6891 | 7258 | error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ |
| 6892 | 7259 | @tagName(air_tag), |
| 6893 | cg.typeOf(bin_op.lhs).fmt(pt), | |
| 7260 | ty.fmt(pt), | |
| 6894 | 7261 | ops[0].tracking(cg), |
| 6895 | 7262 | ops[1].tracking(cg), |
| 6896 | 7263 | }), |
| ... | ... | @@ -92700,7 +93067,7 @@ const Select = struct { |
| 92700 | 93067 | const mir_tag: Mir.Inst.FixedTag = .{ inst[1], inst[2] }; |
| 92701 | 93068 | pseudo: { |
| 92702 | 93069 | switch (inst[0]) { |
| 92703 | .@"0:", .@"1:", .@"2:" => |label| s.emitLabel(label), | |
| 93070 | .@"0:", .@"1:", .@"2:", .@"3:" => |label| s.emitLabel(label), | |
| 92704 | 93071 | ._ => {}, |
| 92705 | 93072 | .pseudo => break :pseudo, |
| 92706 | 93073 | } |
| ... | ... | @@ -93578,7 +93945,7 @@ const Select = struct { |
| 93578 | 93945 | Select.Operand, |
| 93579 | 93946 | Select.Operand, |
| 93580 | 93947 | }; |
| 93581 | const Label = enum { @"0:", @"1:", @"2:", @"_", pseudo }; | |
| 93948 | const Label = enum { @"0:", @"1:", @"2:", @"3:", @"_", pseudo }; | |
| 93582 | 93949 | const Operand = struct { |
| 93583 | 93950 | flags: packed struct(u16) { |
| 93584 | 93951 | tag: Tag, |
| ... | ... | @@ -93609,6 +93976,7 @@ const Select = struct { |
| 93609 | 93976 | ptr_size, |
| 93610 | 93977 | ptr_bit_size, |
| 93611 | 93978 | size, |
| 93979 | src0_size, | |
| 93612 | 93980 | delta_size, |
| 93613 | 93981 | delta_elem_size, |
| 93614 | 93982 | size_add_elem_size, |
| ... | ... | @@ -93641,6 +94009,8 @@ const Select = struct { |
| 93641 | 94009 | const sub_size_div_8: Adjust = .{ .sign = .neg, .lhs = .size, .op = .div, .rhs = .@"8" }; |
| 93642 | 94010 | const sub_size_div_4: Adjust = .{ .sign = .neg, .lhs = .size, .op = .div, .rhs = .@"4" }; |
| 93643 | 94011 | const sub_size: Adjust = .{ .sign = .neg, .lhs = .size, .op = .mul, .rhs = .@"1" }; |
| 94012 | const sub_src0_size_div_8: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .div, .rhs = .@"8" }; | |
| 94013 | const sub_src0_size: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .mul, .rhs = .@"1" }; | |
| 93644 | 94014 | const add_delta_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_size, .op = .div, .rhs = .@"8" }; |
| 93645 | 94015 | const add_delta_elem_size: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .mul, .rhs = .@"1" }; |
| 93646 | 94016 | const add_delta_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .div, .rhs = .@"8" }; |
| ... | ... | @@ -93882,6 +94252,8 @@ const Select = struct { |
| 93882 | 94252 | const @"1f": Select.Operand = .{ .flags = .{ .tag = .forward_label }, .base = .{ .ref = .tmp1, .size = .none } }; |
| 93883 | 94253 | const @"2b": Select.Operand = .{ .flags = .{ .tag = .backward_label }, .base = .{ .ref = .tmp2, .size = .none } }; |
| 93884 | 94254 | const @"2f": Select.Operand = .{ .flags = .{ .tag = .forward_label }, .base = .{ .ref = .tmp2, .size = .none } }; |
| 94255 | const @"3b": Select.Operand = .{ .flags = .{ .tag = .backward_label }, .base = .{ .ref = .tmp3, .size = .none } }; | |
| 94256 | const @"3f": Select.Operand = .{ .flags = .{ .tag = .forward_label }, .base = .{ .ref = .tmp3, .size = .none } }; | |
| 93885 | 94257 | |
| 93886 | 94258 | const tmp0b: Select.Operand = .{ .flags = .{ .tag = .ref }, .base = .tmp0b }; |
| 93887 | 94259 | const tmp0w: Select.Operand = .{ .flags = .{ .tag = .ref }, .base = .tmp0w }; |
| ... | ... | @@ -94070,6 +94442,13 @@ const Select = struct { |
| 94070 | 94442 | .base = base, |
| 94071 | 94443 | }; |
| 94072 | 94444 | } |
| 94445 | fn leaad(base: Ref.Sized, adjust: Adjust, disp: i32) Select.Operand { | |
| 94446 | return .{ | |
| 94447 | .flags = .{ .tag = .lea, .adjust = adjust }, | |
| 94448 | .base = base, | |
| 94449 | .imm = disp, | |
| 94450 | }; | |
| 94451 | } | |
| 94073 | 94452 | fn lead(base: Ref.Sized, disp: i32) Select.Operand { |
| 94074 | 94453 | return .{ |
| 94075 | 94454 | .flags = .{ .tag = .lea }, |
| ... | ... | @@ -94226,6 +94605,7 @@ const Select = struct { |
| 94226 | 94605 | .ptr_size => @divExact(s.cg.target.ptrBitWidth(), 8), |
| 94227 | 94606 | .ptr_bit_size => s.cg.target.ptrBitWidth(), |
| 94228 | 94607 | .size => @intCast(op.base.ref.typeOf(s).abiSize(s.cg.pt.zcu)), |
| 94608 | .src0_size => @intCast(Select.Operand.Ref.src0.typeOf(s).abiSize(s.cg.pt.zcu)), | |
| 94229 | 94609 | .delta_size => @intCast(@as(SignedImm, @intCast(op.base.ref.typeOf(s).abiSize(s.cg.pt.zcu))) - |
| 94230 | 94610 | @as(SignedImm, @intCast(op.index.ref.typeOf(s).abiSize(s.cg.pt.zcu)))), |
| 94231 | 94611 | .delta_elem_size => @intCast(@as(SignedImm, @intCast(op.base.ref.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu))) - |
src/arch/x86_64/Emit.zig+46-22
| ... | ... | @@ -88,13 +88,32 @@ pub fn emitMir(emit: *Emit) Error!void { |
| 88 | 88 | lowered_relocs[0].lowered_inst_index == lowered_index) : ({ |
| 89 | 89 | lowered_relocs = lowered_relocs[1..]; |
| 90 | 90 | }) switch (lowered_relocs[0].target) { |
| 91 | .inst => |target| try relocs.append(emit.lower.allocator, .{ | |
| 92 | .source = start_offset, | |
| 93 | .source_offset = end_offset - 4, | |
| 94 | .target = target, | |
| 95 | .target_offset = lowered_relocs[0].off, | |
| 96 | .length = @intCast(end_offset - start_offset), | |
| 97 | }), | |
| 91 | .inst => |target| { | |
| 92 | const inst_length: u4 = @intCast(end_offset - start_offset); | |
| 93 | const reloc_offset, const reloc_length = reloc_offset_length: { | |
| 94 | var reloc_offset = inst_length; | |
| 95 | var op_index: usize = lowered_inst.ops.len; | |
| 96 | while (true) { | |
| 97 | op_index -= 1; | |
| 98 | const op = lowered_inst.encoding.data.ops[op_index]; | |
| 99 | if (op == .none) continue; | |
| 100 | const enc_length: u4 = @intCast( | |
| 101 | std.math.divCeil(u7, @intCast(op.immBitSize()), 8) catch unreachable, | |
| 102 | ); | |
| 103 | reloc_offset -= enc_length; | |
| 104 | if (op_index == lowered_relocs[0].op_index) | |
| 105 | break :reloc_offset_length .{ reloc_offset, enc_length }; | |
| 106 | } | |
| 107 | }; | |
| 108 | try relocs.append(emit.lower.allocator, .{ | |
| 109 | .inst_offset = start_offset, | |
| 110 | .inst_length = inst_length, | |
| 111 | .source_offset = reloc_offset, | |
| 112 | .source_length = reloc_length, | |
| 113 | .target = target, | |
| 114 | .target_offset = lowered_relocs[0].off, | |
| 115 | }); | |
| 116 | }, | |
| 98 | 117 | .table => try table_relocs.append(emit.lower.allocator, .{ |
| 99 | 118 | .source_offset = end_offset - 4, |
| 100 | 119 | .target_offset = lowered_relocs[0].off, |
| ... | ... | @@ -409,7 +428,7 @@ pub fn emitMir(emit: *Emit) Error!void { |
| 409 | 428 | } } }; |
| 410 | 429 | }, |
| 411 | 430 | .pseudo_dbg_local_am => loc: { |
| 412 | const mem = emit.lower.mem(mir_inst.data.ax.payload); | |
| 431 | const mem = emit.lower.mem(undefined, mir_inst.data.ax.payload); | |
| 413 | 432 | break :loc .{ mir_inst.data.ax.air_inst, .{ .plus = .{ |
| 414 | 433 | base: { |
| 415 | 434 | loc_buf[0] = switch (mem.base()) { |
| ... | ... | @@ -466,15 +485,18 @@ pub fn emitMir(emit: *Emit) Error!void { |
| 466 | 485 | } |
| 467 | 486 | } |
| 468 | 487 | } |
| 469 | { | |
| 470 | // TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size. | |
| 471 | // This should be reversed like it is done in aarch64 MIR emit code: start with the smallest | |
| 472 | // possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution | |
| 473 | // until the entire decl is correctly emitted with all JMP/CALL instructions within range. | |
| 474 | for (relocs.items) |reloc| { | |
| 475 | const target = code_offset_mapping[reloc.target]; | |
| 476 | const disp = @as(i64, @intCast(target)) - @as(i64, @intCast(reloc.source + reloc.length)) + reloc.target_offset; | |
| 477 | std.mem.writeInt(i32, emit.code.items[reloc.source_offset..][0..4], @intCast(disp), .little); | |
| 488 | for (relocs.items) |reloc| { | |
| 489 | const target = code_offset_mapping[reloc.target]; | |
| 490 | const disp = @as(i64, @intCast(target)) - @as(i64, @intCast(reloc.inst_offset + reloc.inst_length)) + reloc.target_offset; | |
| 491 | const inst_bytes = emit.code.items[reloc.inst_offset..][0..reloc.inst_length]; | |
| 492 | switch (reloc.source_length) { | |
| 493 | else => unreachable, | |
| 494 | inline 1, 4 => |source_length| std.mem.writeInt( | |
| 495 | @Type(.{ .int = .{ .signedness = .signed, .bits = @as(u16, 8) * source_length } }), | |
| 496 | inst_bytes[reloc.source_offset..][0..source_length], | |
| 497 | @intCast(disp), | |
| 498 | .little, | |
| 499 | ), | |
| 478 | 500 | } |
| 479 | 501 | } |
| 480 | 502 | if (emit.lower.mir.table.len > 0) { |
| ... | ... | @@ -511,15 +533,17 @@ fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error { |
| 511 | 533 | |
| 512 | 534 | const Reloc = struct { |
| 513 | 535 | /// Offset of the instruction. |
| 514 | source: u32, | |
| 536 | inst_offset: u32, | |
| 537 | /// Length of the instruction. | |
| 538 | inst_length: u4, | |
| 515 | 539 | /// Offset of the relocation within the instruction. |
| 516 | source_offset: u32, | |
| 540 | source_offset: u4, | |
| 541 | /// Length of the relocation. | |
| 542 | source_length: u4, | |
| 517 | 543 | /// Target of the relocation. |
| 518 | 544 | target: Mir.Inst.Index, |
| 519 | /// Offset from the target instruction. | |
| 545 | /// Offset from the target. | |
| 520 | 546 | target_offset: i32, |
| 521 | /// Length of the instruction. | |
| 522 | length: u5, | |
| 523 | 547 | }; |
| 524 | 548 | |
| 525 | 549 | const TableReloc = struct { |
src/arch/x86_64/Encoding.zig+11-6
| ... | ... | @@ -304,20 +304,20 @@ pub const Mnemonic = enum { |
| 304 | 304 | jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, jrcxz, js, jz, |
| 305 | 305 | lahf, lar, lea, leave, lfence, lgdt, lidt, lldt, lmsw, loop, loope, loopne, |
| 306 | 306 | lods, lodsb, lodsd, lodsq, lodsw, |
| 307 | lsl, ltr, lzcnt, | |
| 307 | lsl, ltr, | |
| 308 | 308 | mfence, mov, movbe, |
| 309 | 309 | movs, movsb, movsd, movsq, movsw, |
| 310 | 310 | movsx, movsxd, movzx, mul, |
| 311 | 311 | neg, nop, not, |
| 312 | 312 | @"or", out, outs, outsb, outsd, outsw, |
| 313 | pause, pop, popcnt, popf, popfd, popfq, push, pushfq, | |
| 313 | pause, pop, popf, popfd, popfq, push, pushfq, | |
| 314 | 314 | rcl, rcr, |
| 315 | 315 | rdfsbase, rdgsbase, rdmsr, rdpid, rdpkru, rdpmc, rdrand, rdseed, rdssd, rdssq, rdtsc, rdtscp, |
| 316 | ret, rol, ror, rorx, rsm, | |
| 317 | sahf, sal, sar, sarx, sbb, | |
| 316 | ret, rol, ror, rsm, | |
| 317 | sahf, sal, sar, sbb, | |
| 318 | 318 | scas, scasb, scasd, scasq, scasw, |
| 319 | 319 | senduipi, serialize, |
| 320 | shl, shld, shlx, shr, shrd, shrx, | |
| 320 | shl, shld, shr, shrd, | |
| 321 | 321 | stac, stc, std, sti, str, stui, |
| 322 | 322 | sub, swapgs, syscall, sysenter, sysexit, sysret, |
| 323 | 323 | seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae, |
| ... | ... | @@ -433,6 +433,8 @@ pub const Mnemonic = enum { |
| 433 | 433 | roundpd, roundps, roundsd, roundss, |
| 434 | 434 | // SSE4.2 |
| 435 | 435 | crc32, pcmpgtq, |
| 436 | // ABM | |
| 437 | lzcnt, popcnt, | |
| 436 | 438 | // PCLMUL |
| 437 | 439 | pclmulqdq, |
| 438 | 440 | // AES |
| ... | ... | @@ -440,7 +442,6 @@ pub const Mnemonic = enum { |
| 440 | 442 | // SHA |
| 441 | 443 | sha1rnds4, sha1nexte, sha1msg1, sha1msg2, sha256msg1, sha256msg2, sha256rnds2, |
| 442 | 444 | // AVX |
| 443 | andn, bextr, blsi, blsmsk, blsr, bzhi, tzcnt, | |
| 444 | 445 | vaddpd, vaddps, vaddsd, vaddss, vaddsubpd, vaddsubps, |
| 445 | 446 | vaesdec, vaesdeclast, vaesenc, vaesenclast, vaesimc, vaeskeygenassist, |
| 446 | 447 | vandnpd, vandnps, vandpd, vandps, |
| ... | ... | @@ -506,6 +507,10 @@ pub const Mnemonic = enum { |
| 506 | 507 | vtestpd, vtestps, |
| 507 | 508 | vucomisd, vucomiss, vunpckhpd, vunpckhps, vunpcklpd, vunpcklps, |
| 508 | 509 | vxorpd, vxorps, |
| 510 | // BMI | |
| 511 | andn, bextr, blsi, blsmsk, blsr, tzcnt, | |
| 512 | // BMI2 | |
| 513 | bzhi, mulx, pdep, pext, rorx, sarx, shlx, shrx, | |
| 509 | 514 | // F16C |
| 510 | 515 | vcvtph2ps, vcvtps2ph, |
| 511 | 516 | // FMA |
src/arch/x86_64/Lower.zig+88-72
| ... | ... | @@ -10,32 +10,38 @@ mir: Mir, |
| 10 | 10 | cc: std.builtin.CallingConvention, |
| 11 | 11 | err_msg: ?*Zcu.ErrorMsg = null, |
| 12 | 12 | src_loc: Zcu.LazySrcLoc, |
| 13 | result_insts_len: u8 = undefined, | |
| 14 | result_relocs_len: u8 = undefined, | |
| 15 | result_insts: [ | |
| 16 | @max( | |
| 17 | 1, // non-pseudo instructions | |
| 18 | 3, // (ELF only) TLS local dynamic (LD) sequence in PIC mode | |
| 19 | 2, // cmovcc: cmovcc \ cmovcc | |
| 20 | 3, // setcc: setcc \ setcc \ logicop | |
| 21 | 2, // jcc: jcc \ jcc | |
| 22 | pseudo_probe_align_insts, | |
| 23 | pseudo_probe_adjust_unrolled_max_insts, | |
| 24 | pseudo_probe_adjust_setup_insts, | |
| 25 | pseudo_probe_adjust_loop_insts, | |
| 26 | abi.Win64.callee_preserved_regs.len * 2, // push_regs/pop_regs | |
| 27 | abi.SysV.callee_preserved_regs.len * 2, // push_regs/pop_regs | |
| 28 | ) | |
| 29 | ]Instruction = undefined, | |
| 30 | result_relocs: [ | |
| 31 | @max( | |
| 32 | 1, // jmp/jcc/call/mov/lea: jmp/jcc/call/mov/lea | |
| 33 | 2, // jcc: jcc \ jcc | |
| 34 | 2, // test \ jcc \ probe \ sub \ jmp | |
| 35 | 1, // probe \ sub \ jcc | |
| 36 | 3, // (ELF only) TLS local dynamic (LD) sequence in PIC mode | |
| 37 | ) | |
| 38 | ]Reloc = undefined, | |
| 13 | result_insts_len: ResultInstIndex = undefined, | |
| 14 | result_insts: [max_result_insts]Instruction = undefined, | |
| 15 | result_relocs_len: ResultRelocIndex = undefined, | |
| 16 | result_relocs: [max_result_relocs]Reloc = undefined, | |
| 17 | ||
| 18 | const max_result_insts = @max( | |
| 19 | 1, // non-pseudo instructions | |
| 20 | 3, // (ELF only) TLS local dynamic (LD) sequence in PIC mode | |
| 21 | 2, // cmovcc: cmovcc \ cmovcc | |
| 22 | 3, // setcc: setcc \ setcc \ logicop | |
| 23 | 2, // jcc: jcc \ jcc | |
| 24 | pseudo_probe_align_insts, | |
| 25 | pseudo_probe_adjust_unrolled_max_insts, | |
| 26 | pseudo_probe_adjust_setup_insts, | |
| 27 | pseudo_probe_adjust_loop_insts, | |
| 28 | abi.Win64.callee_preserved_regs.len * 2, // push_regs/pop_regs | |
| 29 | abi.SysV.callee_preserved_regs.len * 2, // push_regs/pop_regs | |
| 30 | ); | |
| 31 | const max_result_relocs = @max( | |
| 32 | 1, // jmp/jcc/call/mov/lea: jmp/jcc/call/mov/lea | |
| 33 | 2, // jcc: jcc \ jcc | |
| 34 | 2, // test \ jcc \ probe \ sub \ jmp | |
| 35 | 1, // probe \ sub \ jcc | |
| 36 | 3, // (ELF only) TLS local dynamic (LD) sequence in PIC mode | |
| 37 | ); | |
| 38 | ||
| 39 | const ResultInstIndex = std.math.IntFittingRange(0, max_result_insts - 1); | |
| 40 | const ResultRelocIndex = std.math.IntFittingRange(0, max_result_relocs - 1); | |
| 41 | const InstOpIndex = std.math.IntFittingRange( | |
| 42 | 0, | |
| 43 | @typeInfo(@FieldType(Instruction, "ops")).array.len - 1, | |
| 44 | ); | |
| 39 | 45 | |
| 40 | 46 | pub const pseudo_probe_align_insts = 5; // test \ jcc \ probe \ sub \ jmp |
| 41 | 47 | pub const pseudo_probe_adjust_unrolled_max_insts = |
| ... | ... | @@ -51,7 +57,8 @@ pub const Error = error{ |
| 51 | 57 | }; |
| 52 | 58 | |
| 53 | 59 | pub const Reloc = struct { |
| 54 | lowered_inst_index: u8, | |
| 60 | lowered_inst_index: ResultInstIndex, | |
| 61 | op_index: InstOpIndex, | |
| 55 | 62 | target: Target, |
| 56 | 63 | off: i32, |
| 57 | 64 | |
| ... | ... | @@ -114,11 +121,11 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct { |
| 114 | 121 | assert(inst.data.rx.fixes == ._); |
| 115 | 122 | try lower.emit(.none, .cmovnz, &.{ |
| 116 | 123 | .{ .reg = inst.data.rx.r1 }, |
| 117 | .{ .mem = lower.mem(inst.data.rx.payload) }, | |
| 124 | .{ .mem = lower.mem(1, inst.data.rx.payload) }, | |
| 118 | 125 | }); |
| 119 | 126 | try lower.emit(.none, .cmovp, &.{ |
| 120 | 127 | .{ .reg = inst.data.rx.r1 }, |
| 121 | .{ .mem = lower.mem(inst.data.rx.payload) }, | |
| 128 | .{ .mem = lower.mem(1, inst.data.rx.payload) }, | |
| 122 | 129 | }); |
| 123 | 130 | }, |
| 124 | 131 | .pseudo_set_z_and_np_r => { |
| ... | ... | @@ -137,13 +144,13 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct { |
| 137 | 144 | .pseudo_set_z_and_np_m => { |
| 138 | 145 | assert(inst.data.rx.fixes == ._); |
| 139 | 146 | try lower.emit(.none, .setz, &.{ |
| 140 | .{ .mem = lower.mem(inst.data.rx.payload) }, | |
| 147 | .{ .mem = lower.mem(0, inst.data.rx.payload) }, | |
| 141 | 148 | }); |
| 142 | 149 | try lower.emit(.none, .setnp, &.{ |
| 143 | 150 | .{ .reg = inst.data.rx.r1 }, |
| 144 | 151 | }); |
| 145 | 152 | try lower.emit(.none, .@"and", &.{ |
| 146 | .{ .mem = lower.mem(inst.data.rx.payload) }, | |
| 153 | .{ .mem = lower.mem(0, inst.data.rx.payload) }, | |
| 147 | 154 | .{ .reg = inst.data.rx.r1 }, |
| 148 | 155 | }); |
| 149 | 156 | }, |
| ... | ... | @@ -163,32 +170,32 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct { |
| 163 | 170 | .pseudo_set_nz_or_p_m => { |
| 164 | 171 | assert(inst.data.rx.fixes == ._); |
| 165 | 172 | try lower.emit(.none, .setnz, &.{ |
| 166 | .{ .mem = lower.mem(inst.data.rx.payload) }, | |
| 173 | .{ .mem = lower.mem(0, inst.data.rx.payload) }, | |
| 167 | 174 | }); |
| 168 | 175 | try lower.emit(.none, .setp, &.{ |
| 169 | 176 | .{ .reg = inst.data.rx.r1 }, |
| 170 | 177 | }); |
| 171 | 178 | try lower.emit(.none, .@"or", &.{ |
| 172 | .{ .mem = lower.mem(inst.data.rx.payload) }, | |
| 179 | .{ .mem = lower.mem(0, inst.data.rx.payload) }, | |
| 173 | 180 | .{ .reg = inst.data.rx.r1 }, |
| 174 | 181 | }); |
| 175 | 182 | }, |
| 176 | 183 | .pseudo_j_z_and_np_inst => { |
| 177 | 184 | assert(inst.data.inst.fixes == ._); |
| 178 | 185 | try lower.emit(.none, .jnz, &.{ |
| 179 | .{ .imm = lower.reloc(.{ .inst = index + 1 }, 0) }, | |
| 186 | .{ .imm = lower.reloc(0, .{ .inst = index + 1 }, 0) }, | |
| 180 | 187 | }); |
| 181 | 188 | try lower.emit(.none, .jnp, &.{ |
| 182 | .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }, 0) }, | |
| 189 | .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, | |
| 183 | 190 | }); |
| 184 | 191 | }, |
| 185 | 192 | .pseudo_j_nz_or_p_inst => { |
| 186 | 193 | assert(inst.data.inst.fixes == ._); |
| 187 | 194 | try lower.emit(.none, .jnz, &.{ |
| 188 | .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }, 0) }, | |
| 195 | .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, | |
| 189 | 196 | }); |
| 190 | 197 | try lower.emit(.none, .jp, &.{ |
| 191 | .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }, 0) }, | |
| 198 | .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, | |
| 192 | 199 | }); |
| 193 | 200 | }, |
| 194 | 201 | |
| ... | ... | @@ -198,7 +205,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct { |
| 198 | 205 | .{ .imm = .s(@bitCast(inst.data.ri.i)) }, |
| 199 | 206 | }); |
| 200 | 207 | try lower.emit(.none, .jz, &.{ |
| 201 | .{ .imm = lower.reloc(.{ .inst = index + 1 }, 0) }, | |
| 208 | .{ .imm = lower.reloc(0, .{ .inst = index + 1 }, 0) }, | |
| 202 | 209 | }); |
| 203 | 210 | try lower.emit(.none, .lea, &.{ |
| 204 | 211 | .{ .reg = inst.data.ri.r1 }, |
| ... | ... | @@ -214,7 +221,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct { |
| 214 | 221 | .{ .reg = inst.data.ri.r1.to32() }, |
| 215 | 222 | }); |
| 216 | 223 | try lower.emit(.none, .jmp, &.{ |
| 217 | .{ .imm = lower.reloc(.{ .inst = index }, 0) }, | |
| 224 | .{ .imm = lower.reloc(0, .{ .inst = index }, 0) }, | |
| 218 | 225 | }); |
| 219 | 226 | assert(lower.result_insts_len == pseudo_probe_align_insts); |
| 220 | 227 | }, |
| ... | ... | @@ -260,7 +267,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct { |
| 260 | 267 | .{ .imm = .s(page_size) }, |
| 261 | 268 | }); |
| 262 | 269 | try lower.emit(.none, .jae, &.{ |
| 263 | .{ .imm = lower.reloc(.{ .inst = index }, 0) }, | |
| 270 | .{ .imm = lower.reloc(0, .{ .inst = index }, 0) }, | |
| 264 | 271 | }); |
| 265 | 272 | assert(lower.result_insts_len == pseudo_probe_adjust_loop_insts); |
| 266 | 273 | }, |
| ... | ... | @@ -382,21 +389,22 @@ pub fn imm(lower: *const Lower, ops: Mir.Inst.Ops, i: u32) Immediate { |
| 382 | 389 | }; |
| 383 | 390 | } |
| 384 | 391 | |
| 385 | pub fn mem(lower: *Lower, payload: u32) Memory { | |
| 392 | pub fn mem(lower: *Lower, op_index: InstOpIndex, payload: u32) Memory { | |
| 386 | 393 | var m = lower.mir.resolveFrameLoc(lower.mir.extraData(Mir.Memory, payload).data).decode(); |
| 387 | 394 | switch (m) { |
| 388 | 395 | .sib => |*sib| switch (sib.base) { |
| 389 | 396 | else => {}, |
| 390 | .table => sib.disp = lower.reloc(.table, sib.disp).signed, | |
| 397 | .table => sib.disp = lower.reloc(op_index, .table, sib.disp).signed, | |
| 391 | 398 | }, |
| 392 | 399 | else => {}, |
| 393 | 400 | } |
| 394 | 401 | return m; |
| 395 | 402 | } |
| 396 | 403 | |
| 397 | fn reloc(lower: *Lower, target: Reloc.Target, off: i32) Immediate { | |
| 404 | fn reloc(lower: *Lower, op_index: InstOpIndex, target: Reloc.Target, off: i32) Immediate { | |
| 398 | 405 | lower.result_relocs[lower.result_relocs_len] = .{ |
| 399 | 406 | .lowered_inst_index = lower.result_insts_len, |
| 407 | .op_index = op_index, | |
| 400 | 408 | .target = target, |
| 401 | 409 | .off = off, |
| 402 | 410 | }; |
| ... | ... | @@ -409,7 +417,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) |
| 409 | 417 | var emit_mnemonic = mnemonic; |
| 410 | 418 | var emit_ops_storage: [4]Operand = undefined; |
| 411 | 419 | const emit_ops = emit_ops_storage[0..ops.len]; |
| 412 | for (emit_ops, ops) |*emit_op, op| { | |
| 420 | for (emit_ops, ops, 0..) |*emit_op, op, op_index| { | |
| 413 | 421 | emit_op.* = switch (op) { |
| 414 | 422 | else => op, |
| 415 | 423 | .mem => |mem_op| switch (mem_op.base()) { |
| ... | ... | @@ -428,20 +436,20 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) |
| 428 | 436 | if (lower.pic) { |
| 429 | 437 | // Here, we currently assume local dynamic TLS vars, and so |
| 430 | 438 | // we emit LD model. |
| 431 | _ = lower.reloc(.{ .linker_tlsld = sym_index }, 0); | |
| 439 | _ = lower.reloc(1, .{ .linker_tlsld = sym_index }, 0); | |
| 432 | 440 | lower.result_insts[lower.result_insts_len] = try .new(.none, .lea, &.{ |
| 433 | 441 | .{ .reg = .rdi }, |
| 434 | 442 | .{ .mem = Memory.initRip(.none, 0) }, |
| 435 | 443 | }, lower.target); |
| 436 | 444 | lower.result_insts_len += 1; |
| 437 | _ = lower.reloc(.{ | |
| 445 | _ = lower.reloc(0, .{ | |
| 438 | 446 | .linker_extern_fn = try elf_file.getGlobalSymbol("__tls_get_addr", null), |
| 439 | 447 | }, 0); |
| 440 | 448 | lower.result_insts[lower.result_insts_len] = try .new(.none, .call, &.{ |
| 441 | 449 | .{ .imm = .s(0) }, |
| 442 | 450 | }, lower.target); |
| 443 | 451 | lower.result_insts_len += 1; |
| 444 | _ = lower.reloc(.{ .linker_dtpoff = sym_index }, 0); | |
| 452 | _ = lower.reloc(@intCast(op_index), .{ .linker_dtpoff = sym_index }, 0); | |
| 445 | 453 | emit_mnemonic = .lea; |
| 446 | 454 | break :op .{ .mem = Memory.initSib(.none, .{ |
| 447 | 455 | .base = .{ .reg = .rax }, |
| ... | ... | @@ -454,7 +462,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) |
| 454 | 462 | .{ .mem = Memory.initSib(.qword, .{ .base = .{ .reg = .fs } }) }, |
| 455 | 463 | }, lower.target); |
| 456 | 464 | lower.result_insts_len += 1; |
| 457 | _ = lower.reloc(.{ .linker_reloc = sym_index }, 0); | |
| 465 | _ = lower.reloc(@intCast(op_index), .{ .linker_reloc = sym_index }, 0); | |
| 458 | 466 | emit_mnemonic = .lea; |
| 459 | 467 | break :op .{ .mem = Memory.initSib(.none, .{ |
| 460 | 468 | .base = .{ .reg = .rax }, |
| ... | ... | @@ -463,15 +471,17 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) |
| 463 | 471 | } |
| 464 | 472 | } |
| 465 | 473 | |
| 466 | _ = lower.reloc(.{ .linker_reloc = sym_index }, 0); | |
| 467 | 474 | if (lower.pic) switch (mnemonic) { |
| 468 | .lea => if (elf_sym.flags.is_extern_ptr) { | |
| 475 | .lea => { | |
| 476 | _ = lower.reloc(@intCast(op_index), .{ .linker_reloc = sym_index }, 0); | |
| 477 | if (!elf_sym.flags.is_extern_ptr) break :op .{ .mem = Memory.initRip(.none, 0) }; | |
| 469 | 478 | emit_mnemonic = .mov; |
| 470 | 479 | break :op .{ .mem = Memory.initRip(.ptr, 0) }; |
| 471 | } else break :op .{ .mem = Memory.initRip(.none, 0) }, | |
| 480 | }, | |
| 472 | 481 | .mov => { |
| 473 | 482 | if (elf_sym.flags.is_extern_ptr) { |
| 474 | 483 | const reg = ops[0].reg; |
| 484 | _ = lower.reloc(1, .{ .linker_reloc = sym_index }, 0); | |
| 475 | 485 | lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{ |
| 476 | 486 | .{ .reg = reg.to64() }, |
| 477 | 487 | .{ .mem = Memory.initRip(.qword, 0) }, |
| ... | ... | @@ -481,10 +491,13 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) |
| 481 | 491 | .reg = reg.to64(), |
| 482 | 492 | } }) }; |
| 483 | 493 | } |
| 494 | _ = lower.reloc(@intCast(op_index), .{ .linker_reloc = sym_index }, 0); | |
| 484 | 495 | break :op .{ .mem = Memory.initRip(mem_op.sib.ptr_size, 0) }; |
| 485 | 496 | }, |
| 486 | 497 | else => unreachable, |
| 487 | } else switch (mnemonic) { | |
| 498 | }; | |
| 499 | _ = lower.reloc(@intCast(op_index), .{ .linker_reloc = sym_index }, 0); | |
| 500 | switch (mnemonic) { | |
| 488 | 501 | .call => break :op .{ .mem = Memory.initSib(mem_op.sib.ptr_size, .{ |
| 489 | 502 | .base = .{ .reg = .ds }, |
| 490 | 503 | }) }, |
| ... | ... | @@ -502,7 +515,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) |
| 502 | 515 | const macho_sym = zo.symbols.items[sym_index]; |
| 503 | 516 | |
| 504 | 517 | if (macho_sym.flags.tlv) { |
| 505 | _ = lower.reloc(.{ .linker_reloc = sym_index }, 0); | |
| 518 | _ = lower.reloc(1, .{ .linker_reloc = sym_index }, 0); | |
| 506 | 519 | lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{ |
| 507 | 520 | .{ .reg = .rdi }, |
| 508 | 521 | .{ .mem = Memory.initRip(.ptr, 0) }, |
| ... | ... | @@ -516,15 +529,17 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) |
| 516 | 529 | break :op .{ .reg = .rax }; |
| 517 | 530 | } |
| 518 | 531 | |
| 519 | _ = lower.reloc(.{ .linker_reloc = sym_index }, 0); | |
| 520 | 532 | break :op switch (mnemonic) { |
| 521 | .lea => if (macho_sym.flags.is_extern_ptr) { | |
| 533 | .lea => { | |
| 534 | _ = lower.reloc(@intCast(op_index), .{ .linker_reloc = sym_index }, 0); | |
| 535 | if (!macho_sym.flags.is_extern_ptr) break :op .{ .mem = Memory.initRip(.none, 0) }; | |
| 522 | 536 | emit_mnemonic = .mov; |
| 523 | 537 | break :op .{ .mem = Memory.initRip(.ptr, 0) }; |
| 524 | } else break :op .{ .mem = Memory.initRip(.none, 0) }, | |
| 538 | }, | |
| 525 | 539 | .mov => { |
| 526 | 540 | if (macho_sym.flags.is_extern_ptr) { |
| 527 | 541 | const reg = ops[0].reg; |
| 542 | _ = lower.reloc(1, .{ .linker_reloc = sym_index }, 0); | |
| 528 | 543 | lower.result_insts[lower.result_insts_len] = try .new(.none, .mov, &.{ |
| 529 | 544 | .{ .reg = reg.to64() }, |
| 530 | 545 | .{ .mem = Memory.initRip(.qword, 0) }, |
| ... | ... | @@ -534,6 +549,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) |
| 534 | 549 | .reg = reg.to64(), |
| 535 | 550 | } }) }; |
| 536 | 551 | } |
| 552 | _ = lower.reloc(@intCast(op_index), .{ .linker_reloc = sym_index }, 0); | |
| 537 | 553 | break :op .{ .mem = Memory.initRip(mem_op.sib.ptr_size, 0) }; |
| 538 | 554 | }, |
| 539 | 555 | else => unreachable, |
| ... | ... | @@ -550,7 +566,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) |
| 550 | 566 | } |
| 551 | 567 | |
| 552 | 568 | fn generic(lower: *Lower, inst: Mir.Inst) Error!void { |
| 553 | @setEvalBranchQuota(2_400); | |
| 569 | @setEvalBranchQuota(2_500); | |
| 554 | 570 | const fixes = switch (inst.ops) { |
| 555 | 571 | .none => inst.data.none.fixes, |
| 556 | 572 | .inst => inst.data.inst.fixes, |
| ... | ... | @@ -595,7 +611,7 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void { |
| 595 | 611 | }, switch (inst.ops) { |
| 596 | 612 | .none => &.{}, |
| 597 | 613 | .inst => &.{ |
| 598 | .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }, 0) }, | |
| 614 | .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) }, | |
| 599 | 615 | }, |
| 600 | 616 | .i_s, .i_u => &.{ |
| 601 | 617 | .{ .imm = lower.imm(inst.ops, inst.data.i.i) }, |
| ... | ... | @@ -642,10 +658,10 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void { |
| 642 | 658 | .{ .imm = lower.imm(inst.ops, inst.data.rri.i) }, |
| 643 | 659 | }, |
| 644 | 660 | .m => &.{ |
| 645 | .{ .mem = lower.mem(inst.data.x.payload) }, | |
| 661 | .{ .mem = lower.mem(0, inst.data.x.payload) }, | |
| 646 | 662 | }, |
| 647 | 663 | .mi_s, .mi_u => &.{ |
| 648 | .{ .mem = lower.mem(inst.data.x.payload + 1) }, | |
| 664 | .{ .mem = lower.mem(0, inst.data.x.payload + 1) }, | |
| 649 | 665 | .{ .imm = lower.imm( |
| 650 | 666 | inst.ops, |
| 651 | 667 | lower.mir.extraData(Mir.Imm32, inst.data.x.payload).data.imm, |
| ... | ... | @@ -653,64 +669,64 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void { |
| 653 | 669 | }, |
| 654 | 670 | .rm => &.{ |
| 655 | 671 | .{ .reg = inst.data.rx.r1 }, |
| 656 | .{ .mem = lower.mem(inst.data.rx.payload) }, | |
| 672 | .{ .mem = lower.mem(1, inst.data.rx.payload) }, | |
| 657 | 673 | }, |
| 658 | 674 | .rmr => &.{ |
| 659 | 675 | .{ .reg = inst.data.rrx.r1 }, |
| 660 | .{ .mem = lower.mem(inst.data.rrx.payload) }, | |
| 676 | .{ .mem = lower.mem(1, inst.data.rrx.payload) }, | |
| 661 | 677 | .{ .reg = inst.data.rrx.r2 }, |
| 662 | 678 | }, |
| 663 | 679 | .rmi => &.{ |
| 664 | 680 | .{ .reg = inst.data.rix.r1 }, |
| 665 | .{ .mem = lower.mem(inst.data.rix.payload) }, | |
| 681 | .{ .mem = lower.mem(1, inst.data.rix.payload) }, | |
| 666 | 682 | .{ .imm = lower.imm(inst.ops, inst.data.rix.i) }, |
| 667 | 683 | }, |
| 668 | 684 | .rmi_s, .rmi_u => &.{ |
| 669 | 685 | .{ .reg = inst.data.rx.r1 }, |
| 670 | .{ .mem = lower.mem(inst.data.rx.payload + 1) }, | |
| 686 | .{ .mem = lower.mem(1, inst.data.rx.payload + 1) }, | |
| 671 | 687 | .{ .imm = lower.imm( |
| 672 | 688 | inst.ops, |
| 673 | 689 | lower.mir.extraData(Mir.Imm32, inst.data.rx.payload).data.imm, |
| 674 | 690 | ) }, |
| 675 | 691 | }, |
| 676 | 692 | .mr => &.{ |
| 677 | .{ .mem = lower.mem(inst.data.rx.payload) }, | |
| 693 | .{ .mem = lower.mem(0, inst.data.rx.payload) }, | |
| 678 | 694 | .{ .reg = inst.data.rx.r1 }, |
| 679 | 695 | }, |
| 680 | 696 | .mrr => &.{ |
| 681 | .{ .mem = lower.mem(inst.data.rrx.payload) }, | |
| 697 | .{ .mem = lower.mem(0, inst.data.rrx.payload) }, | |
| 682 | 698 | .{ .reg = inst.data.rrx.r1 }, |
| 683 | 699 | .{ .reg = inst.data.rrx.r2 }, |
| 684 | 700 | }, |
| 685 | 701 | .mri => &.{ |
| 686 | .{ .mem = lower.mem(inst.data.rix.payload) }, | |
| 702 | .{ .mem = lower.mem(0, inst.data.rix.payload) }, | |
| 687 | 703 | .{ .reg = inst.data.rix.r1 }, |
| 688 | 704 | .{ .imm = lower.imm(inst.ops, inst.data.rix.i) }, |
| 689 | 705 | }, |
| 690 | 706 | .rrm => &.{ |
| 691 | 707 | .{ .reg = inst.data.rrx.r1 }, |
| 692 | 708 | .{ .reg = inst.data.rrx.r2 }, |
| 693 | .{ .mem = lower.mem(inst.data.rrx.payload) }, | |
| 709 | .{ .mem = lower.mem(2, inst.data.rrx.payload) }, | |
| 694 | 710 | }, |
| 695 | 711 | .rrmr => &.{ |
| 696 | 712 | .{ .reg = inst.data.rrrx.r1 }, |
| 697 | 713 | .{ .reg = inst.data.rrrx.r2 }, |
| 698 | .{ .mem = lower.mem(inst.data.rrrx.payload) }, | |
| 714 | .{ .mem = lower.mem(2, inst.data.rrrx.payload) }, | |
| 699 | 715 | .{ .reg = inst.data.rrrx.r3 }, |
| 700 | 716 | }, |
| 701 | 717 | .rrmi => &.{ |
| 702 | 718 | .{ .reg = inst.data.rrix.r1 }, |
| 703 | 719 | .{ .reg = inst.data.rrix.r2 }, |
| 704 | .{ .mem = lower.mem(inst.data.rrix.payload) }, | |
| 720 | .{ .mem = lower.mem(2, inst.data.rrix.payload) }, | |
| 705 | 721 | .{ .imm = lower.imm(inst.ops, inst.data.rrix.i) }, |
| 706 | 722 | }, |
| 707 | 723 | .extern_fn_reloc, .rel => &.{ |
| 708 | .{ .imm = lower.reloc(.{ .linker_extern_fn = inst.data.reloc.sym_index }, inst.data.reloc.off) }, | |
| 724 | .{ .imm = lower.reloc(0, .{ .linker_extern_fn = inst.data.reloc.sym_index }, inst.data.reloc.off) }, | |
| 709 | 725 | }, |
| 710 | 726 | .got_reloc, .direct_reloc, .import_reloc => ops: { |
| 711 | 727 | const reg = inst.data.rx.r1; |
| 712 | 728 | const extra = lower.mir.extraData(bits.SymbolOffset, inst.data.rx.payload).data; |
| 713 | _ = lower.reloc(switch (inst.ops) { | |
| 729 | _ = lower.reloc(1, switch (inst.ops) { | |
| 714 | 730 | .got_reloc => .{ .linker_got = extra.sym_index }, |
| 715 | 731 | .direct_reloc => .{ .linker_direct = extra.sym_index }, |
| 716 | 732 | .import_reloc => .{ .linker_import = extra.sym_index }, |
src/arch/x86_64/Mir.zig+4-3
| ... | ... | @@ -100,6 +100,8 @@ pub const Inst = struct { |
| 100 | 100 | /// ___ Division |
| 101 | 101 | _d, |
| 102 | 102 | |
| 103 | /// ___ Without Affecting Flags | |
| 104 | _x, | |
| 103 | 105 | /// ___ Left |
| 104 | 106 | _l, |
| 105 | 107 | /// ___ Left Double |
| ... | ... | @@ -483,6 +485,7 @@ pub const Inst = struct { |
| 483 | 485 | /// ASCII adjust al after subtraction |
| 484 | 486 | aa, |
| 485 | 487 | /// Add with carry |
| 488 | /// Unsigned integer addition of two operands with carry flag | |
| 486 | 489 | adc, |
| 487 | 490 | /// Add |
| 488 | 491 | /// Add packed integers |
| ... | ... | @@ -1162,10 +1165,8 @@ pub const Inst = struct { |
| 1162 | 1165 | fmadd231, |
| 1163 | 1166 | |
| 1164 | 1167 | // ADX |
| 1165 | /// Unsigned integer addition of two operands with carry flag | |
| 1166 | adcx, | |
| 1167 | 1168 | /// Unsigned integer addition of two operands with overflow flag |
| 1168 | adox, | |
| 1169 | ado, | |
| 1169 | 1170 | |
| 1170 | 1171 | // AESKLE |
| 1171 | 1172 | /// Encode 128-bit key with key locker |
src/arch/x86_64/encodings.zig+55-43
| ... | ... | @@ -405,9 +405,9 @@ pub const table = [_]Entry{ |
| 405 | 405 | .{ .jb, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none, .none }, |
| 406 | 406 | .{ .jbe, .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none, .none }, |
| 407 | 407 | .{ .jc, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none, .none }, |
| 408 | .{ .jcxz, .d, &.{ .rel32 }, &.{ 0xe3 }, 0, .short, .@"32bit" }, | |
| 409 | .{ .jecxz, .d, &.{ .rel32 }, &.{ 0xe3 }, 0, .none, .@"32bit" }, | |
| 410 | .{ .jrcxz, .d, &.{ .rel32 }, &.{ 0xe3 }, 0, .none, .@"64bit" }, | |
| 408 | .{ .jcxz, .d, &.{ .rel8 }, &.{ 0xe3 }, 0, .short, .@"32bit" }, | |
| 409 | .{ .jecxz, .d, &.{ .rel8 }, &.{ 0xe3 }, 0, .none, .@"32bit" }, | |
| 410 | .{ .jrcxz, .d, &.{ .rel8 }, &.{ 0xe3 }, 0, .none, .@"64bit" }, | |
| 411 | 411 | .{ .je, .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none, .none }, |
| 412 | 412 | .{ .jg, .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none, .none }, |
| 413 | 413 | .{ .jge, .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none, .none }, |
| ... | ... | @@ -477,10 +477,6 @@ pub const table = [_]Entry{ |
| 477 | 477 | |
| 478 | 478 | .{ .ltr, .m, &.{ .rm16 }, &.{ 0x0f, 0x00 }, 3, .none, .none }, |
| 479 | 479 | |
| 480 | .{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .short, .lzcnt }, | |
| 481 | .{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .lzcnt }, | |
| 482 | .{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .lzcnt }, | |
| 483 | ||
| 484 | 480 | .{ .mfence, .z, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none }, |
| 485 | 481 | |
| 486 | 482 | .{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .none, .none }, |
| ... | ... | @@ -630,10 +626,6 @@ pub const table = [_]Entry{ |
| 630 | 626 | .{ .pop, .m, &.{ .rm16 }, &.{ 0x8f }, 0, .short, .none }, |
| 631 | 627 | .{ .pop, .m, &.{ .rm64 }, &.{ 0x8f }, 0, .none, .none }, |
| 632 | 628 | |
| 633 | .{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .short, .popcnt }, | |
| 634 | .{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .popcnt }, | |
| 635 | .{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .popcnt }, | |
| 636 | ||
| 637 | 629 | .{ .popf, .z, &.{}, &.{ 0x9d }, 0, .short, .none }, |
| 638 | 630 | .{ .popfd, .z, &.{}, &.{ 0x9d }, 0, .none, .@"32bit" }, |
| 639 | 631 | .{ .popfq, .z, &.{}, &.{ 0x9d }, 0, .none, .@"64bit" }, |
| ... | ... | @@ -1738,6 +1730,15 @@ pub const table = [_]Entry{ |
| 1738 | 1730 | |
| 1739 | 1731 | .{ .pcmpgtq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x37 }, 0, .none, .sse4_2 }, |
| 1740 | 1732 | |
| 1733 | // ABM | |
| 1734 | .{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .short, .lzcnt }, | |
| 1735 | .{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .lzcnt }, | |
| 1736 | .{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .lzcnt }, | |
| 1737 | ||
| 1738 | .{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .short, .popcnt }, | |
| 1739 | .{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .popcnt }, | |
| 1740 | .{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .popcnt }, | |
| 1741 | ||
| 1741 | 1742 | // PCLMUL |
| 1742 | 1743 | .{ .pclmulqdq, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .none, .pclmul }, |
| 1743 | 1744 | |
| ... | ... | @@ -1771,38 +1772,6 @@ pub const table = [_]Entry{ |
| 1771 | 1772 | .{ .sha256msg2, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x38, 0xcd }, 0, .none, .sha }, |
| 1772 | 1773 | |
| 1773 | 1774 | // AVX |
| 1774 | .{ .andn, .rvm, &.{ .r32, .r32, .rm32 }, &.{ 0x0f, 0x38, 0xf2 }, 0, .vex_lz_w0, .bmi }, | |
| 1775 | .{ .andn, .rvm, &.{ .r64, .r64, .rm64 }, &.{ 0x0f, 0x38, 0xf2 }, 0, .vex_lz_w1, .bmi }, | |
| 1776 | ||
| 1777 | .{ .bextr, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi }, | |
| 1778 | .{ .bextr, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi }, | |
| 1779 | ||
| 1780 | .{ .blsi, .vm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x38, 0xf3 }, 3, .vex_lz_w0, .bmi }, | |
| 1781 | .{ .blsi, .vm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x38, 0xf3 }, 3, .vex_lz_w1, .bmi }, | |
| 1782 | ||
| 1783 | .{ .blsmsk, .vm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x38, 0xf3 }, 2, .vex_lz_w0, .bmi }, | |
| 1784 | .{ .blsmsk, .vm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x38, 0xf3 }, 2, .vex_lz_w1, .bmi }, | |
| 1785 | ||
| 1786 | .{ .blsr, .vm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x38, 0xf3 }, 1, .vex_lz_w0, .bmi }, | |
| 1787 | .{ .blsr, .vm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x38, 0xf3 }, 1, .vex_lz_w1, .bmi }, | |
| 1788 | ||
| 1789 | .{ .bzhi, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w0, .bmi2 }, | |
| 1790 | .{ .bzhi, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w1, .bmi2 }, | |
| 1791 | ||
| 1792 | .{ .rorx, .rmi, &.{ .r32, .rm32, .imm8 }, &.{ 0xf2, 0x0f, 0x3a }, 0, .vex_lz_w0, .bmi2 }, | |
| 1793 | .{ .rorx, .rmi, &.{ .r64, .rm64, .imm8 }, &.{ 0xf2, 0x0f, 0x3a }, 0, .vex_lz_w1, .bmi2 }, | |
| 1794 | ||
| 1795 | .{ .sarx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, | |
| 1796 | .{ .shlx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, | |
| 1797 | .{ .shrx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, | |
| 1798 | .{ .sarx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, | |
| 1799 | .{ .shlx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, | |
| 1800 | .{ .shrx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, | |
| 1801 | ||
| 1802 | .{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .short, .bmi }, | |
| 1803 | .{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .bmi }, | |
| 1804 | .{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .bmi }, | |
| 1805 | ||
| 1806 | 1775 | .{ .vaddpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_128_wig, .avx }, |
| 1807 | 1776 | .{ .vaddpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_256_wig, .avx }, |
| 1808 | 1777 | |
| ... | ... | @@ -2307,6 +2276,49 @@ pub const table = [_]Entry{ |
| 2307 | 2276 | .{ .vxorps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .vex_128_wig, .avx }, |
| 2308 | 2277 | .{ .vxorps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x57 }, 0, .vex_256_wig, .avx }, |
| 2309 | 2278 | |
| 2279 | // BMI | |
| 2280 | .{ .andn, .rvm, &.{ .r32, .r32, .rm32 }, &.{ 0x0f, 0x38, 0xf2 }, 0, .vex_lz_w0, .bmi }, | |
| 2281 | .{ .andn, .rvm, &.{ .r64, .r64, .rm64 }, &.{ 0x0f, 0x38, 0xf2 }, 0, .vex_lz_w1, .bmi }, | |
| 2282 | ||
| 2283 | .{ .bextr, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi }, | |
| 2284 | .{ .bextr, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi }, | |
| 2285 | ||
| 2286 | .{ .blsi, .vm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x38, 0xf3 }, 3, .vex_lz_w0, .bmi }, | |
| 2287 | .{ .blsi, .vm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x38, 0xf3 }, 3, .vex_lz_w1, .bmi }, | |
| 2288 | ||
| 2289 | .{ .blsmsk, .vm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x38, 0xf3 }, 2, .vex_lz_w0, .bmi }, | |
| 2290 | .{ .blsmsk, .vm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x38, 0xf3 }, 2, .vex_lz_w1, .bmi }, | |
| 2291 | ||
| 2292 | .{ .blsr, .vm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x38, 0xf3 }, 1, .vex_lz_w0, .bmi }, | |
| 2293 | .{ .blsr, .vm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x38, 0xf3 }, 1, .vex_lz_w1, .bmi }, | |
| 2294 | ||
| 2295 | .{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .short, .bmi }, | |
| 2296 | .{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .bmi }, | |
| 2297 | .{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .bmi }, | |
| 2298 | ||
| 2299 | // BMI2 | |
| 2300 | .{ .bzhi, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w0, .bmi2 }, | |
| 2301 | .{ .bzhi, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w1, .bmi2 }, | |
| 2302 | ||
| 2303 | .{ .mulx, .rvm, &.{ .r32, .r32, .rm32 }, &.{ 0xf2, 0x0f, 0x38, 0xf6 }, 0, .vex_lz_w0, .bmi2 }, | |
| 2304 | .{ .mulx, .rvm, &.{ .r64, .r64, .rm64 }, &.{ 0xf2, 0x0f, 0x38, 0xf6 }, 0, .vex_lz_w1, .bmi2 }, | |
| 2305 | ||
| 2306 | .{ .pdep, .rvm, &.{ .r32, .r32, .rm32 }, &.{ 0xf2, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w0, .bmi2 }, | |
| 2307 | .{ .pdep, .rvm, &.{ .r64, .r64, .rm64 }, &.{ 0xf2, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w1, .bmi2 }, | |
| 2308 | ||
| 2309 | .{ .pext, .rvm, &.{ .r32, .r32, .rm32 }, &.{ 0xf3, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w0, .bmi2 }, | |
| 2310 | .{ .pext, .rvm, &.{ .r64, .r64, .rm64 }, &.{ 0xf3, 0x0f, 0x38, 0xf5 }, 0, .vex_lz_w1, .bmi2 }, | |
| 2311 | ||
| 2312 | .{ .rorx, .rmi, &.{ .r32, .rm32, .imm8 }, &.{ 0xf2, 0x0f, 0x3a }, 0, .vex_lz_w0, .bmi2 }, | |
| 2313 | .{ .rorx, .rmi, &.{ .r64, .rm64, .imm8 }, &.{ 0xf2, 0x0f, 0x3a }, 0, .vex_lz_w1, .bmi2 }, | |
| 2314 | ||
| 2315 | .{ .sarx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, | |
| 2316 | .{ .shlx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, | |
| 2317 | .{ .shrx, .rmv, &.{ .r32, .rm32, .r32 }, &.{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w0, .bmi2 }, | |
| 2318 | .{ .sarx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0xf3, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, | |
| 2319 | .{ .shlx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0x66, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, | |
| 2320 | .{ .shrx, .rmv, &.{ .r64, .rm64, .r64 }, &.{ 0xf2, 0x0f, 0x38, 0xf7 }, 0, .vex_lz_w1, .bmi2 }, | |
| 2321 | ||
| 2310 | 2322 | // F16C |
| 2311 | 2323 | .{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128_w0, .f16c }, |
| 2312 | 2324 | .{ .vcvtph2ps, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_256_w0, .f16c }, |
test/behavior/x86_64/build.zig+5
| ... | ... | @@ -93,6 +93,11 @@ pub fn build(b: *std.Build) void { |
| 93 | 93 | .cpu_arch = .x86_64, |
| 94 | 94 | .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v3 }, |
| 95 | 95 | }, |
| 96 | .{ | |
| 97 | .cpu_arch = .x86_64, | |
| 98 | .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v3 }, | |
| 99 | .cpu_features_add = std.Target.x86.featureSet(&.{.adx}), | |
| 100 | }, | |
| 96 | 101 | .{ |
| 97 | 102 | .cpu_arch = .x86_64, |
| 98 | 103 | .cpu_model = .{ .explicit = &std.Target.x86.cpu.x86_64_v4 }, |
test/behavior/x86_64/math.zig+22
| ... | ... | @@ -44,6 +44,17 @@ fn AddOneBit(comptime Type: type) type { |
| 44 | 44 | .vector => |vector| @Vector(vector.len, ResultScalar), |
| 45 | 45 | }; |
| 46 | 46 | } |
| 47 | fn DoubleBits(comptime Type: type) type { | |
| 48 | const ResultScalar = switch (@typeInfo(Scalar(Type))) { | |
| 49 | .int => |int| @Type(.{ .int = .{ .signedness = int.signedness, .bits = int.bits * 2 } }), | |
| 50 | .float => Scalar(Type), | |
| 51 | else => @compileError(@typeName(Type)), | |
| 52 | }; | |
| 53 | return switch (@typeInfo(Type)) { | |
| 54 | else => ResultScalar, | |
| 55 | .vector => |vector| @Vector(vector.len, ResultScalar), | |
| 56 | }; | |
| 57 | } | |
| 47 | 58 | // inline to avoid a runtime `@splat` |
| 48 | 59 | inline fn splat(comptime Type: type, scalar: Scalar(Type)) Type { |
| 49 | 60 | return switch (@typeInfo(Type)) { |
| ... | ... | @@ -16216,6 +16227,8 @@ fn binary(comptime op: anytype, comptime opts: struct { compare: Compare = .rela |
| 16216 | 16227 | ); |
| 16217 | 16228 | } |
| 16218 | 16229 | fn testInts() !void { |
| 16230 | try testArgs(i4, 0x3, 0x2); | |
| 16231 | try testArgs(u4, 0xe, 0x6); | |
| 16219 | 16232 | try testArgs(i8, 0x48, 0x6c); |
| 16220 | 16233 | try testArgs(u8, 0xbb, 0x43); |
| 16221 | 16234 | try testArgs(i16, -0x0fdf, 0x302e); |
| ... | ... | @@ -18993,6 +19006,15 @@ test subUnsafe { |
| 18993 | 19006 | try test_sub_unsafe.testFloatVectors(); |
| 18994 | 19007 | } |
| 18995 | 19008 | |
| 19009 | inline fn mulUnsafe(comptime Type: type, lhs: Type, rhs: Type) DoubleBits(Type) { | |
| 19010 | @setRuntimeSafety(false); | |
| 19011 | return @as(DoubleBits(Type), lhs) * rhs; | |
| 19012 | } | |
| 19013 | test mulUnsafe { | |
| 19014 | const test_mul_unsafe = binary(mulUnsafe, .{}); | |
| 19015 | try test_mul_unsafe.testInts(); | |
| 19016 | } | |
| 19017 | ||
| 18996 | 19018 | inline fn multiply(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs * rhs) { |
| 18997 | 19019 | if (@inComptime() and @typeInfo(Type) == .vector) { |
| 18998 | 19020 | // workaround https://github.com/ziglang/zig/issues/22743 |