| ... | @@ -135,6 +135,8 @@ const Owner = union(enum) { | ... | @@ -135,6 +135,8 @@ const Owner = union(enum) { |
| 135 | } | 135 | } |
| 136 | }; | 136 | }; |
| 137 | | 137 | |
| | 138 | const MaskKind = enum { sign, all }; |
| | 139 | |
| 138 | pub const MCValue = union(enum) { | 140 | pub const MCValue = union(enum) { |
| 139 | /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. | 141 | /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. |
| 140 | /// TODO Look into deleting this tag and using `dead` instead, since every use | 142 | /// TODO Look into deleting this tag and using `dead` instead, since every use |
| ... | @@ -156,10 +158,16 @@ pub const MCValue = union(enum) { | ... | @@ -156,10 +158,16 @@ pub const MCValue = union(enum) { |
| 156 | register: Register, | 158 | register: Register, |
| 157 | /// The value is split across two registers. | 159 | /// The value is split across two registers. |
| 158 | register_pair: [2]Register, | 160 | register_pair: [2]Register, |
| | 161 | /// The value is split across three registers. |
| | 162 | register_triple: [3]Register, |
| | 163 | /// The value is split across four registers. |
| | 164 | register_quadruple: [4]Register, |
| 159 | /// The value is a constant offset from the value in a register. | 165 | /// The value is a constant offset from the value in a register. |
| 160 | register_offset: bits.RegisterOffset, | 166 | register_offset: bits.RegisterOffset, |
| 161 | /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register. | 167 | /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register. |
| 162 | register_overflow: struct { reg: Register, eflags: Condition }, | 168 | register_overflow: struct { reg: Register, eflags: Condition }, |
| | 169 | /// The value is a bool vector stored in a vector register with a different scalar type. |
| | 170 | register_mask: struct { reg: Register, kind: MaskKind, inverted: bool, scalar: Memory.Size }, |
| 163 | /// The value is in memory at a hard-coded address. | 171 | /// The value is in memory at a hard-coded address. |
| 164 | /// If the type is a pointer, it means the pointer address is stored at this memory location. | 172 | /// If the type is a pointer, it means the pointer address is stored at this memory location. |
| 165 | memory: u64, | 173 | memory: u64, |
| ... | @@ -195,7 +203,7 @@ pub const MCValue = union(enum) { | ... | @@ -195,7 +203,7 @@ pub const MCValue = union(enum) { |
| 195 | /// Payload is a frame address. | 203 | /// Payload is a frame address. |
| 196 | lea_frame: bits.FrameAddr, | 204 | lea_frame: bits.FrameAddr, |
| 197 | /// Supports integer_per_element abi | 205 | /// Supports integer_per_element abi |
| 198 | elementwise_regs_then_frame: packed struct { regs: u3 = 0, frame_off: i29 = 0, frame_index: FrameIndex }, | 206 | elementwise_regs_then_frame: packed struct { regs: u3, frame_off: i29, frame_index: FrameIndex }, |
| 199 | /// This indicates that we have already allocated a frame index for this instruction, | 207 | /// This indicates that we have already allocated a frame index for this instruction, |
| 200 | /// but it has not been spilled there yet in the current control flow. | 208 | /// but it has not been spilled there yet in the current control flow. |
| 201 | /// Payload is a frame index. | 209 | /// Payload is a frame index. |
| ... | @@ -210,6 +218,7 @@ pub const MCValue = union(enum) { | ... | @@ -210,6 +218,7 @@ pub const MCValue = union(enum) { |
| 210 | .undef, | 218 | .undef, |
| 211 | .immediate, | 219 | .immediate, |
| 212 | .register_offset, | 220 | .register_offset, |
| | 221 | .register_mask, |
| 213 | .eflags, | 222 | .eflags, |
| 214 | .register_overflow, | 223 | .register_overflow, |
| 215 | .lea_symbol, | 224 | .lea_symbol, |
| ... | @@ -223,6 +232,8 @@ pub const MCValue = union(enum) { | ... | @@ -223,6 +232,8 @@ pub const MCValue = union(enum) { |
| 223 | => false, | 232 | => false, |
| 224 | .register, | 233 | .register, |
| 225 | .register_pair, | 234 | .register_pair, |
| | 235 | .register_triple, |
| | 236 | .register_quadruple, |
| 226 | .memory, | 237 | .memory, |
| 227 | .load_symbol, | 238 | .load_symbol, |
| 228 | .load_got, | 239 | .load_got, |
| ... | @@ -234,13 +245,20 @@ pub const MCValue = union(enum) { | ... | @@ -234,13 +245,20 @@ pub const MCValue = union(enum) { |
| 234 | }; | 245 | }; |
| 235 | } | 246 | } |
| 236 | | 247 | |
| 237 | fn isMemory(mcv: MCValue) bool { | 248 | fn isBase(mcv: MCValue) bool { |
| 238 | return switch (mcv) { | 249 | return switch (mcv) { |
| 239 | .memory, .indirect, .load_frame => true, | 250 | .memory, .indirect, .load_frame => true, |
| 240 | else => false, | 251 | else => false, |
| 241 | }; | 252 | }; |
| 242 | } | 253 | } |
| 243 | | 254 | |
| | 255 | fn isMemory(mcv: MCValue) bool { |
| | 256 | return switch (mcv) { |
| | 257 | .memory, .indirect, .load_frame, .load_symbol => true, |
| | 258 | else => false, |
| | 259 | }; |
| | 260 | } |
| | 261 | |
| 244 | fn isImmediate(mcv: MCValue) bool { | 262 | fn isImmediate(mcv: MCValue) bool { |
| 245 | return switch (mcv) { | 263 | return switch (mcv) { |
| 246 | .immediate => true, | 264 | .immediate => true, |
| ... | @@ -268,16 +286,23 @@ pub const MCValue = union(enum) { | ... | @@ -268,16 +286,23 @@ pub const MCValue = union(enum) { |
| 268 | .register => |reg| reg, | 286 | .register => |reg| reg, |
| 269 | .register_offset, .indirect => |ro| ro.reg, | 287 | .register_offset, .indirect => |ro| ro.reg, |
| 270 | .register_overflow => |ro| ro.reg, | 288 | .register_overflow => |ro| ro.reg, |
| | 289 | .register_mask => |rm| rm.reg, |
| 271 | else => null, | 290 | else => null, |
| 272 | }; | 291 | }; |
| 273 | } | 292 | } |
| 274 | | 293 | |
| 275 | fn getRegs(mcv: *const MCValue) []const Register { | 294 | fn getRegs(mcv: *const MCValue) []const Register { |
| 276 | return switch (mcv.*) { | 295 | return switch (mcv.*) { |
| 277 | .register => |*reg| @as(*const [1]Register, reg), | 296 | .register => |*reg| reg[0..1], |
| 278 | .register_pair => |*regs| regs, | 297 | inline .register_pair, |
| 279 | .register_offset, .indirect => |*ro| @as(*const [1]Register, &ro.reg), | 298 | .register_triple, |
| 280 | .register_overflow => |*ro| @as(*const [1]Register, &ro.reg), | 299 | .register_quadruple, |
| | 300 | => |*regs| regs, |
| | 301 | inline .register_offset, |
| | 302 | .indirect, |
| | 303 | .register_overflow, |
| | 304 | .register_mask, |
| | 305 | => |*pl| (&pl.reg)[0..1], |
| 281 | else => &.{}, | 306 | else => &.{}, |
| 282 | }; | 307 | }; |
| 283 | } | 308 | } |
| ... | @@ -300,8 +325,11 @@ pub const MCValue = union(enum) { | ... | @@ -300,8 +325,11 @@ pub const MCValue = union(enum) { |
| 300 | .eflags, | 325 | .eflags, |
| 301 | .register, | 326 | .register, |
| 302 | .register_pair, | 327 | .register_pair, |
| | 328 | .register_triple, |
| | 329 | .register_quadruple, |
| 303 | .register_offset, | 330 | .register_offset, |
| 304 | .register_overflow, | 331 | .register_overflow, |
| | 332 | .register_mask, |
| 305 | .lea_symbol, | 333 | .lea_symbol, |
| 306 | .lea_direct, | 334 | .lea_direct, |
| 307 | .lea_got, | 335 | .lea_got, |
| ... | @@ -332,7 +360,10 @@ pub const MCValue = union(enum) { | ... | @@ -332,7 +360,10 @@ pub const MCValue = union(enum) { |
| 332 | .undef, | 360 | .undef, |
| 333 | .eflags, | 361 | .eflags, |
| 334 | .register_pair, | 362 | .register_pair, |
| | 363 | .register_triple, |
| | 364 | .register_quadruple, |
| 335 | .register_overflow, | 365 | .register_overflow, |
| | 366 | .register_mask, |
| 336 | .memory, | 367 | .memory, |
| 337 | .indirect, | 368 | .indirect, |
| 338 | .load_direct, | 369 | .load_direct, |
| ... | @@ -367,7 +398,10 @@ pub const MCValue = union(enum) { | ... | @@ -367,7 +398,10 @@ pub const MCValue = union(enum) { |
| 367 | => unreachable, // not valid | 398 | => unreachable, // not valid |
| 368 | .eflags, | 399 | .eflags, |
| 369 | .register_pair, | 400 | .register_pair, |
| | 401 | .register_triple, |
| | 402 | .register_quadruple, |
| 370 | .register_overflow, | 403 | .register_overflow, |
| | 404 | .register_mask, |
| 371 | .memory, | 405 | .memory, |
| 372 | .indirect, | 406 | .indirect, |
| 373 | .load_direct, | 407 | .load_direct, |
| ... | @@ -404,8 +438,11 @@ pub const MCValue = union(enum) { | ... | @@ -404,8 +438,11 @@ pub const MCValue = union(enum) { |
| 404 | .eflags, | 438 | .eflags, |
| 405 | .register, | 439 | .register, |
| 406 | .register_pair, | 440 | .register_pair, |
| | 441 | .register_triple, |
| | 442 | .register_quadruple, |
| 407 | .register_offset, | 443 | .register_offset, |
| 408 | .register_overflow, | 444 | .register_overflow, |
| | 445 | .register_mask, |
| 409 | .load_direct, | 446 | .load_direct, |
| 410 | .lea_direct, | 447 | .lea_direct, |
| 411 | .load_got, | 448 | .load_got, |
| ... | @@ -472,10 +509,11 @@ pub const MCValue = union(enum) { | ... | @@ -472,10 +509,11 @@ pub const MCValue = union(enum) { |
| 472 | .memory => |pl| try writer.print("[ds:0x{x}]", .{pl}), | 509 | .memory => |pl| try writer.print("[ds:0x{x}]", .{pl}), |
| 473 | inline .eflags, .register => |pl| try writer.print("{s}", .{@tagName(pl)}), | 510 | inline .eflags, .register => |pl| try writer.print("{s}", .{@tagName(pl)}), |
| 474 | .register_pair => |pl| try writer.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }), | 511 | .register_pair => |pl| try writer.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }), |
| | 512 | .register_triple => |pl| try writer.print("{s}:{s}:{s}", .{ @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]) }), |
| | 513 | .register_quadruple => |pl| try writer.print("{s}:{s}:{s}:{s}", .{ @tagName(pl[3]), @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]) }), |
| 475 | .register_offset => |pl| try writer.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }), | 514 | .register_offset => |pl| try writer.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }), |
| 476 | .register_overflow => |pl| try writer.print("{s}:{s}", .{ | 515 | .register_overflow => |pl| try writer.print("{s}:{s}", .{ @tagName(pl.eflags), @tagName(pl.reg) }), |
| 477 | @tagName(pl.eflags), @tagName(pl.reg), | 516 | .register_mask => |pl| try writer.print("mask({s},{}):{s}", .{ @tagName(pl.kind), pl.scalar, @tagName(pl.reg) }), |
| 478 | }), | | |
| 479 | .load_symbol => |pl| try writer.print("[sym:{} + 0x{x}]", .{ pl.sym_index, pl.off }), | 517 | .load_symbol => |pl| try writer.print("[sym:{} + 0x{x}]", .{ pl.sym_index, pl.off }), |
| 480 | .lea_symbol => |pl| try writer.print("sym:{} + 0x{x}", .{ pl.sym_index, pl.off }), | 518 | .lea_symbol => |pl| try writer.print("sym:{} + 0x{x}", .{ pl.sym_index, pl.off }), |
| 481 | .indirect => |pl| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }), | 519 | .indirect => |pl| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }), |
| ... | @@ -526,8 +564,11 @@ const InstTracking = struct { | ... | @@ -526,8 +564,11 @@ const InstTracking = struct { |
| 526 | .eflags, | 564 | .eflags, |
| 527 | .register, | 565 | .register, |
| 528 | .register_pair, | 566 | .register_pair, |
| | 567 | .register_triple, |
| | 568 | .register_quadruple, |
| 529 | .register_offset, | 569 | .register_offset, |
| 530 | .register_overflow, | 570 | .register_overflow, |
| | 571 | .register_mask, |
| 531 | .indirect, | 572 | .indirect, |
| 532 | => .none, | 573 | => .none, |
| 533 | }, .short = result }; | 574 | }, .short = result }; |
| ... | @@ -545,17 +586,17 @@ const InstTracking = struct { | ... | @@ -545,17 +586,17 @@ const InstTracking = struct { |
| 545 | return self.short.getCondition(); | 586 | return self.short.getCondition(); |
| 546 | } | 587 | } |
| 547 | | 588 | |
| 548 | fn spill(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void { | 589 | fn spill(self: *InstTracking, cg: *CodeGen, inst: Air.Inst.Index) !void { |
| 549 | if (std.meta.eql(self.long, self.short)) return; // Already spilled | 590 | if (std.meta.eql(self.long, self.short)) return; // Already spilled |
| 550 | // Allocate or reuse frame index | 591 | // Allocate or reuse frame index |
| 551 | switch (self.long) { | 592 | switch (self.long) { |
| 552 | .none => self.long = try function.allocRegOrMem(inst, false), | 593 | .none => self.long = try cg.allocRegOrMem(inst, false), |
| 553 | .load_frame => {}, | 594 | .load_frame => {}, |
| 554 | .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } }, | 595 | .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } }, |
| 555 | else => unreachable, | 596 | else => unreachable, |
| 556 | } | 597 | } |
| 557 | tracking_log.debug("spill {} from {} to {}", .{ inst, self.short, self.long }); | 598 | tracking_log.debug("spill {} from {} to {}", .{ inst, self.short, self.long }); |
| 558 | try function.genCopy(function.typeOfIndex(inst), self.long, self.short, .{}); | 599 | try cg.genCopy(cg.typeOfIndex(inst), self.long, self.short, .{}); |
| 559 | } | 600 | } |
| 560 | | 601 | |
| 561 | fn reuseFrame(self: *InstTracking) void { | 602 | fn reuseFrame(self: *InstTracking) void { |
| ... | @@ -584,8 +625,11 @@ const InstTracking = struct { | ... | @@ -584,8 +625,11 @@ const InstTracking = struct { |
| 584 | .eflags, | 625 | .eflags, |
| 585 | .register, | 626 | .register, |
| 586 | .register_pair, | 627 | .register_pair, |
| | 628 | .register_triple, |
| | 629 | .register_quadruple, |
| 587 | .register_offset, | 630 | .register_offset, |
| 588 | .register_overflow, | 631 | .register_overflow, |
| | 632 | .register_mask, |
| 589 | .indirect, | 633 | .indirect, |
| 590 | .elementwise_regs_then_frame, | 634 | .elementwise_regs_then_frame, |
| 591 | .reserved_frame, | 635 | .reserved_frame, |
| ... | @@ -630,8 +674,11 @@ const InstTracking = struct { | ... | @@ -630,8 +674,11 @@ const InstTracking = struct { |
| 630 | .eflags, | 674 | .eflags, |
| 631 | .register, | 675 | .register, |
| 632 | .register_pair, | 676 | .register_pair, |
| | 677 | .register_triple, |
| | 678 | .register_quadruple, |
| 633 | .register_offset, | 679 | .register_offset, |
| 634 | .register_overflow, | 680 | .register_overflow, |
| | 681 | .register_mask, |
| 635 | .indirect, | 682 | .indirect, |
| 636 | .elementwise_regs_then_frame, | 683 | .elementwise_regs_then_frame, |
| 637 | .air_ref, | 684 | .air_ref, |
| ... | @@ -735,13 +782,13 @@ const InstTracking = struct { | ... | @@ -735,13 +782,13 @@ const InstTracking = struct { |
| 735 | } | 782 | } |
| 736 | | 783 | |
| 737 | pub fn format( | 784 | pub fn format( |
| 738 | self: InstTracking, | 785 | tracking: InstTracking, |
| 739 | comptime _: []const u8, | 786 | comptime _: []const u8, |
| 740 | _: std.fmt.FormatOptions, | 787 | _: std.fmt.FormatOptions, |
| 741 | writer: anytype, | 788 | writer: anytype, |
| 742 | ) @TypeOf(writer).Error!void { | 789 | ) @TypeOf(writer).Error!void { |
| 743 | if (!std.meta.eql(self.long, self.short)) try writer.print("|{}| ", .{self.long}); | 790 | if (!std.meta.eql(tracking.long, tracking.short)) try writer.print("|{}| ", .{tracking.long}); |
| 744 | try writer.print("{}", .{self.short}); | 791 | try writer.print("{}", .{tracking.short}); |
| 745 | } | 792 | } |
| 746 | }; | 793 | }; |
| 747 | | 794 | |
| ... | @@ -2259,44 +2306,44 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2259,44 +2306,44 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2259 | try self.asmPseudo(.pseudo_dbg_leave_block_none); | 2306 | try self.asmPseudo(.pseudo_dbg_leave_block_none); |
| 2260 | } | 2307 | } |
| 2261 | | 2308 | |
| 2262 | fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | 2309 | fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2263 | const pt = self.pt; | 2310 | const pt = cg.pt; |
| 2264 | const zcu = pt.zcu; | 2311 | const zcu = pt.zcu; |
| 2265 | const ip = &zcu.intern_pool; | 2312 | const ip = &zcu.intern_pool; |
| 2266 | const air_tags = self.air.instructions.items(.tag); | 2313 | const air_tags = cg.air.instructions.items(.tag); |
| 2267 | const air_datas = self.air.instructions.items(.data); | 2314 | const air_datas = cg.air.instructions.items(.data); |
| 2268 | const use_old = self.target.ofmt == .coff; | 2315 | const use_old = cg.target.ofmt == .coff; |
| 2269 | | 2316 | |
| 2270 | self.arg_index = 0; | 2317 | cg.arg_index = 0; |
| 2271 | for (body) |inst| switch (air_tags[@intFromEnum(inst)]) { | 2318 | for (body) |inst| switch (air_tags[@intFromEnum(inst)]) { |
| 2272 | .arg => { | 2319 | .arg => { |
| 2273 | wip_mir_log.debug("{}", .{self.fmtAir(inst)}); | 2320 | wip_mir_log.debug("{}", .{cg.fmtAir(inst)}); |
| 2274 | verbose_tracking_log.debug("{}", .{self.fmtTracking()}); | 2321 | verbose_tracking_log.debug("{}", .{cg.fmtTracking()}); |
| 2275 | | 2322 | |
| 2276 | self.reused_operands = .initEmpty(); | 2323 | cg.reused_operands = .initEmpty(); |
| 2277 | try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1); | 2324 | try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1); |
| 2278 | | 2325 | |
| 2279 | try self.airArg(inst); | 2326 | try cg.airArg(inst); |
| 2280 | | 2327 | |
| 2281 | self.resetTemps(); | 2328 | cg.resetTemps(); |
| 2282 | self.checkInvariantsAfterAirInst(); | 2329 | cg.checkInvariantsAfterAirInst(); |
| 2283 | }, | 2330 | }, |
| 2284 | else => break, | 2331 | else => break, |
| 2285 | }; | 2332 | }; |
| 2286 | | 2333 | |
| 2287 | if (self.arg_index == 0) try self.airDbgVarArgs(); | 2334 | if (cg.arg_index == 0) try cg.airDbgVarArgs(); |
| 2288 | self.arg_index = 0; | 2335 | cg.arg_index = 0; |
| 2289 | for (body) |inst| { | 2336 | for (body) |inst| { |
| 2290 | if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue; | 2337 | if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) continue; |
| 2291 | wip_mir_log.debug("{}", .{self.fmtAir(inst)}); | 2338 | wip_mir_log.debug("{}", .{cg.fmtAir(inst)}); |
| 2292 | verbose_tracking_log.debug("{}", .{self.fmtTracking()}); | 2339 | verbose_tracking_log.debug("{}", .{cg.fmtTracking()}); |
| 2293 | | 2340 | |
| 2294 | self.reused_operands = .initEmpty(); | 2341 | cg.reused_operands = .initEmpty(); |
| 2295 | try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1); | 2342 | try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1); |
| 2296 | switch (air_tags[@intFromEnum(inst)]) { | 2343 | switch (air_tags[@intFromEnum(inst)]) { |
| 2297 | // zig fmt: off | 2344 | // zig fmt: off |
| 2298 | .not, | 2345 | .not, |
| 2299 | => |tag| try self.airUnOp(inst, tag), | 2346 | => |air_tag| try cg.airUnOp(inst, air_tag), |
| 2300 | | 2347 | |
| 2301 | .add, | 2348 | .add, |
| 2302 | .add_wrap, | 2349 | .add_wrap, |
| ... | @@ -2306,22 +2353,22 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2306,22 +2353,22 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2306 | .bool_or, | 2353 | .bool_or, |
| 2307 | .min, | 2354 | .min, |
| 2308 | .max, | 2355 | .max, |
| 2309 | => |tag| try self.airBinOp(inst, tag), | 2356 | => |air_tag| try cg.airBinOp(inst, air_tag), |
| 2310 | | 2357 | |
| 2311 | .ptr_add, .ptr_sub => |tag| try self.airPtrArithmetic(inst, tag), | 2358 | .ptr_add, .ptr_sub => |air_tag| try cg.airPtrArithmetic(inst, air_tag), |
| 2312 | | 2359 | |
| 2313 | .shr, .shr_exact => try self.airShlShrBinOp(inst), | 2360 | .shr, .shr_exact => try cg.airShlShrBinOp(inst), |
| 2314 | .shl, .shl_exact => try self.airShlShrBinOp(inst), | 2361 | .shl, .shl_exact => try cg.airShlShrBinOp(inst), |
| 2315 | | 2362 | |
| 2316 | .mul => try self.airMulDivBinOp(inst), | 2363 | .mul => try cg.airMulDivBinOp(inst), |
| 2317 | .mul_wrap => try self.airMulDivBinOp(inst), | 2364 | .mul_wrap => try cg.airMulDivBinOp(inst), |
| 2318 | .rem => try self.airMulDivBinOp(inst), | 2365 | .rem => try cg.airMulDivBinOp(inst), |
| 2319 | .mod => try self.airMulDivBinOp(inst), | 2366 | .mod => try cg.airMulDivBinOp(inst), |
| 2320 | | 2367 | |
| 2321 | .add_sat => try self.airAddSat(inst), | 2368 | .add_sat => try cg.airAddSat(inst), |
| 2322 | .sub_sat => try self.airSubSat(inst), | 2369 | .sub_sat => try cg.airSubSat(inst), |
| 2323 | .mul_sat => try self.airMulSat(inst), | 2370 | .mul_sat => try cg.airMulSat(inst), |
| 2324 | .shl_sat => try self.airShlSat(inst), | 2371 | .shl_sat => try cg.airShlSat(inst), |
| 2325 | | 2372 | |
| 2326 | .sin, | 2373 | .sin, |
| 2327 | .cos, | 2374 | .cos, |
| ... | @@ -2332,98 +2379,97 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2332,98 +2379,97 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2332 | .log2, | 2379 | .log2, |
| 2333 | .log10, | 2380 | .log10, |
| 2334 | .round, | 2381 | .round, |
| 2335 | => |tag| try self.airUnaryMath(inst, tag), | 2382 | => |air_tag| try cg.airUnaryMath(inst, air_tag), |
| 2336 | | 2383 | |
| 2337 | .floor => try self.airRound(inst, .{ .mode = .down, .precision = .inexact }), | 2384 | .floor => try cg.airRound(inst, .{ .mode = .down, .precision = .inexact }), |
| 2338 | .ceil => try self.airRound(inst, .{ .mode = .up, .precision = .inexact }), | 2385 | .ceil => try cg.airRound(inst, .{ .mode = .up, .precision = .inexact }), |
| 2339 | .trunc_float => try self.airRound(inst, .{ .mode = .zero, .precision = .inexact }), | 2386 | .trunc_float => try cg.airRound(inst, .{ .mode = .zero, .precision = .inexact }), |
| 2340 | .sqrt => try self.airSqrt(inst), | 2387 | .sqrt => try cg.airSqrt(inst), |
| 2341 | .neg => try self.airFloatSign(inst), | 2388 | .neg => try cg.airFloatSign(inst), |
| 2342 | | 2389 | |
| 2343 | .abs => try self.airAbs(inst), | 2390 | .abs => try cg.airAbs(inst), |
| 2344 | | 2391 | |
| 2345 | .add_with_overflow => try self.airAddSubWithOverflow(inst), | 2392 | .add_with_overflow => try cg.airAddSubWithOverflow(inst), |
| 2346 | .sub_with_overflow => try self.airAddSubWithOverflow(inst), | 2393 | .sub_with_overflow => try cg.airAddSubWithOverflow(inst), |
| 2347 | .mul_with_overflow => try self.airMulWithOverflow(inst), | 2394 | .mul_with_overflow => try cg.airMulWithOverflow(inst), |
| 2348 | .shl_with_overflow => try self.airShlWithOverflow(inst), | 2395 | .shl_with_overflow => try cg.airShlWithOverflow(inst), |
| 2349 | | 2396 | |
| 2350 | .div_float, .div_trunc, .div_floor, .div_exact => try self.airMulDivBinOp(inst), | 2397 | .div_float, .div_trunc, .div_floor, .div_exact => try cg.airMulDivBinOp(inst), |
| 2351 | | 2398 | |
| 2352 | .cmp_lt => try self.airCmp(inst, .lt), | 2399 | .cmp_lt => try cg.airCmp(inst, .lt), |
| 2353 | .cmp_lte => try self.airCmp(inst, .lte), | 2400 | .cmp_lte => try cg.airCmp(inst, .lte), |
| 2354 | .cmp_eq => try self.airCmp(inst, .eq), | 2401 | .cmp_eq => try cg.airCmp(inst, .eq), |
| 2355 | .cmp_gte => try self.airCmp(inst, .gte), | 2402 | .cmp_gte => try cg.airCmp(inst, .gte), |
| 2356 | .cmp_gt => try self.airCmp(inst, .gt), | 2403 | .cmp_gt => try cg.airCmp(inst, .gt), |
| 2357 | .cmp_neq => try self.airCmp(inst, .neq), | 2404 | .cmp_neq => try cg.airCmp(inst, .neq), |
| 2358 | | 2405 | |
| 2359 | .cmp_vector => try self.airCmpVector(inst), | 2406 | .cmp_lt_errors_len => try cg.airCmpLtErrorsLen(inst), |
| 2360 | .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), | 2407 | |
| 2361 | | 2408 | .bitcast => try cg.airBitCast(inst), |
| 2362 | .bitcast => try self.airBitCast(inst), | 2409 | .fptrunc => try cg.airFptrunc(inst), |
| 2363 | .fptrunc => try self.airFptrunc(inst), | 2410 | .fpext => try cg.airFpext(inst), |
| 2364 | .fpext => try self.airFpext(inst), | 2411 | .intcast => try cg.airIntCast(inst), |
| 2365 | .intcast => try self.airIntCast(inst), | 2412 | .trunc => try cg.airTrunc(inst), |
| 2366 | .trunc => try self.airTrunc(inst), | 2413 | .is_non_null => try cg.airIsNonNull(inst), |
| 2367 | .is_non_null => try self.airIsNonNull(inst), | 2414 | .is_null => try cg.airIsNull(inst), |
| 2368 | .is_null => try self.airIsNull(inst), | 2415 | .is_non_err => try cg.airIsNonErr(inst), |
| 2369 | .is_non_err => try self.airIsNonErr(inst), | 2416 | .is_err => try cg.airIsErr(inst), |
| 2370 | .is_err => try self.airIsErr(inst), | 2417 | .load => try cg.airLoad(inst), |
| 2371 | .load => try self.airLoad(inst), | 2418 | .store => try cg.airStore(inst, false), |
| 2372 | .store => try self.airStore(inst, false), | 2419 | .store_safe => try cg.airStore(inst, true), |
| 2373 | .store_safe => try self.airStore(inst, true), | 2420 | .struct_field_val => try cg.airStructFieldVal(inst), |
| 2374 | .struct_field_val => try self.airStructFieldVal(inst), | 2421 | .float_from_int => try cg.airFloatFromInt(inst), |
| 2375 | .float_from_int => try self.airFloatFromInt(inst), | 2422 | .int_from_float => try cg.airIntFromFloat(inst), |
| 2376 | .int_from_float => try self.airIntFromFloat(inst), | 2423 | .cmpxchg_strong => try cg.airCmpxchg(inst), |
| 2377 | .cmpxchg_strong => try self.airCmpxchg(inst), | 2424 | .cmpxchg_weak => try cg.airCmpxchg(inst), |
| 2378 | .cmpxchg_weak => try self.airCmpxchg(inst), | 2425 | .atomic_rmw => try cg.airAtomicRmw(inst), |
| 2379 | .atomic_rmw => try self.airAtomicRmw(inst), | 2426 | .atomic_load => try cg.airAtomicLoad(inst), |
| 2380 | .atomic_load => try self.airAtomicLoad(inst), | 2427 | .memcpy => try cg.airMemcpy(inst), |
| 2381 | .memcpy => try self.airMemcpy(inst), | 2428 | .memset => try cg.airMemset(inst, false), |
| 2382 | .memset => try self.airMemset(inst, false), | 2429 | .memset_safe => try cg.airMemset(inst, true), |
| 2383 | .memset_safe => try self.airMemset(inst, true), | 2430 | .set_union_tag => try cg.airSetUnionTag(inst), |
| 2384 | .set_union_tag => try self.airSetUnionTag(inst), | 2431 | .get_union_tag => try cg.airGetUnionTag(inst), |
| 2385 | .get_union_tag => try self.airGetUnionTag(inst), | 2432 | .clz => try cg.airClz(inst), |
| 2386 | .clz => try self.airClz(inst), | 2433 | .ctz => try cg.airCtz(inst), |
| 2387 | .ctz => try self.airCtz(inst), | 2434 | .popcount => try cg.airPopCount(inst), |
| 2388 | .popcount => try self.airPopCount(inst), | 2435 | .byte_swap => try cg.airByteSwap(inst), |
| 2389 | .byte_swap => try self.airByteSwap(inst), | 2436 | .bit_reverse => try cg.airBitReverse(inst), |
| 2390 | .bit_reverse => try self.airBitReverse(inst), | 2437 | .tag_name => try cg.airTagName(inst), |
| 2391 | .tag_name => try self.airTagName(inst), | 2438 | .error_name => try cg.airErrorName(inst), |
| 2392 | .error_name => try self.airErrorName(inst), | 2439 | .splat => try cg.airSplat(inst), |
| 2393 | .splat => try self.airSplat(inst), | 2440 | .select => try cg.airSelect(inst), |
| 2394 | .select => try self.airSelect(inst), | 2441 | .shuffle => try cg.airShuffle(inst), |
| 2395 | .shuffle => try self.airShuffle(inst), | 2442 | .reduce => try cg.airReduce(inst), |
| 2396 | .reduce => try self.airReduce(inst), | 2443 | .aggregate_init => try cg.airAggregateInit(inst), |
| 2397 | .aggregate_init => try self.airAggregateInit(inst), | 2444 | .union_init => try cg.airUnionInit(inst), |
| 2398 | .union_init => try self.airUnionInit(inst), | 2445 | .prefetch => try cg.airPrefetch(inst), |
| 2399 | .prefetch => try self.airPrefetch(inst), | 2446 | .mul_add => try cg.airMulAdd(inst), |
| 2400 | .mul_add => try self.airMulAdd(inst), | 2447 | |
| 2401 | | 2448 | .atomic_store_unordered => try cg.airAtomicStore(inst, .unordered), |
| 2402 | .atomic_store_unordered => try self.airAtomicStore(inst, .unordered), | 2449 | .atomic_store_monotonic => try cg.airAtomicStore(inst, .monotonic), |
| 2403 | .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic), | 2450 | .atomic_store_release => try cg.airAtomicStore(inst, .release), |
| 2404 | .atomic_store_release => try self.airAtomicStore(inst, .release), | 2451 | .atomic_store_seq_cst => try cg.airAtomicStore(inst, .seq_cst), |
| 2405 | .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst), | 2452 | |
| 2406 | | 2453 | .array_elem_val => try cg.airArrayElemVal(inst), |
| 2407 | .array_elem_val => try self.airArrayElemVal(inst), | 2454 | .slice_elem_val => try cg.airSliceElemVal(inst), |
| 2408 | .slice_elem_val => try self.airSliceElemVal(inst), | 2455 | .ptr_elem_val => try cg.airPtrElemVal(inst), |
| 2409 | .ptr_elem_val => try self.airPtrElemVal(inst), | 2456 | |
| 2410 | | 2457 | .optional_payload => try cg.airOptionalPayload(inst), |
| 2411 | .optional_payload => try self.airOptionalPayload(inst), | 2458 | .unwrap_errunion_err => try cg.airUnwrapErrUnionErr(inst), |
| 2412 | .unwrap_errunion_err => try self.airUnwrapErrUnionErr(inst), | 2459 | .unwrap_errunion_payload => try cg.airUnwrapErrUnionPayload(inst), |
| 2413 | .unwrap_errunion_payload => try self.airUnwrapErrUnionPayload(inst), | 2460 | .err_return_trace => try cg.airErrReturnTrace(inst), |
| 2414 | .err_return_trace => try self.airErrReturnTrace(inst), | 2461 | .set_err_return_trace => try cg.airSetErrReturnTrace(inst), |
| 2415 | .set_err_return_trace => try self.airSetErrReturnTrace(inst), | 2462 | .save_err_return_trace_index=> try cg.airSaveErrReturnTraceIndex(inst), |
| 2416 | .save_err_return_trace_index=> try self.airSaveErrReturnTraceIndex(inst), | 2463 | |
| 2417 | | 2464 | .wrap_optional => try cg.airWrapOptional(inst), |
| 2418 | .wrap_optional => try self.airWrapOptional(inst), | 2465 | .wrap_errunion_payload => try cg.airWrapErrUnionPayload(inst), |
| 2419 | .wrap_errunion_payload => try self.airWrapErrUnionPayload(inst), | 2466 | .wrap_errunion_err => try cg.airWrapErrUnionErr(inst), |
| 2420 | .wrap_errunion_err => try self.airWrapErrUnionErr(inst), | | |
| 2421 | // zig fmt: on | 2467 | // zig fmt: on |
| 2422 | | 2468 | |
| 2423 | .add_safe, | 2469 | .add_safe, |
| 2424 | .sub_safe, | 2470 | .sub_safe, |
| 2425 | .mul_safe, | 2471 | .mul_safe, |
| 2426 | => return self.fail("TODO implement safety_checked_instructions", .{}), | 2472 | => return cg.fail("TODO implement safety_checked_instructions", .{}), |
| 2427 | .add_optimized, | 2473 | .add_optimized, |
| 2428 | .sub_optimized, | 2474 | .sub_optimized, |
| 2429 | .mul_optimized, | 2475 | .mul_optimized, |
| ... | @@ -2440,214 +2486,612 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2440,214 +2486,612 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2440 | .cmp_gte_optimized, | 2486 | .cmp_gte_optimized, |
| 2441 | .cmp_gt_optimized, | 2487 | .cmp_gt_optimized, |
| 2442 | .cmp_neq_optimized, | 2488 | .cmp_neq_optimized, |
| 2443 | .cmp_vector_optimized, | | |
| 2444 | .reduce_optimized, | 2489 | .reduce_optimized, |
| 2445 | .int_from_float_optimized, | 2490 | .int_from_float_optimized, |
| 2446 | => return self.fail("TODO implement optimized float mode", .{}), | 2491 | => return cg.fail("TODO implement optimized float mode", .{}), |
| 2447 | | 2492 | |
| 2448 | .arg => try self.airDbgArg(inst), | 2493 | .arg => try cg.airDbgArg(inst), |
| 2449 | .alloc => if (use_old) try self.airAlloc(inst) else { | 2494 | .alloc => if (use_old) try cg.airAlloc(inst) else { |
| 2450 | var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{ | 2495 | var slot = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .lea_frame = .{ |
| 2451 | .index = try self.allocMemPtr(inst), | 2496 | .index = try cg.allocMemPtr(inst), |
| 2452 | } }); | 2497 | } }); |
| 2453 | try slot.moveTo(inst, self); | 2498 | try slot.moveTo(inst, cg); |
| 2454 | }, | 2499 | }, |
| 2455 | .inferred_alloc => unreachable, | 2500 | .inferred_alloc => unreachable, |
| 2456 | .inferred_alloc_comptime => unreachable, | 2501 | .inferred_alloc_comptime => unreachable, |
| 2457 | .ret_ptr => if (use_old) try self.airRetPtr(inst) else { | 2502 | .ret_ptr => if (use_old) try cg.airRetPtr(inst) else { |
| 2458 | var slot = switch (self.ret_mcv.long) { | 2503 | var slot = switch (cg.ret_mcv.long) { |
| 2459 | else => unreachable, | 2504 | else => unreachable, |
| 2460 | .none => try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{ | 2505 | .none => try cg.tempFromValue(cg.typeOfIndex(inst), .{ .lea_frame = .{ |
| 2461 | .index = try self.allocMemPtr(inst), | 2506 | .index = try cg.allocMemPtr(inst), |
| 2462 | } }), | 2507 | } }), |
| 2463 | .load_frame => slot: { | 2508 | .load_frame => slot: { |
| 2464 | var slot = try self.tempFromValue(self.typeOfIndex(inst), self.ret_mcv.long); | 2509 | var slot = try cg.tempFromValue(cg.typeOfIndex(inst), cg.ret_mcv.long); |
| 2465 | try slot.toOffset(self.ret_mcv.short.indirect.off, self); | 2510 | try slot.toOffset(cg.ret_mcv.short.indirect.off, cg); |
| 2466 | break :slot slot; | 2511 | break :slot slot; |
| 2467 | }, | 2512 | }, |
| 2468 | }; | 2513 | }; |
| 2469 | try slot.moveTo(inst, self); | 2514 | try slot.moveTo(inst, cg); |
| 2470 | }, | 2515 | }, |
| 2471 | .assembly => try self.airAsm(inst), | 2516 | .assembly => try cg.airAsm(inst), |
| 2472 | inline .bit_and, .bit_or, .xor => |tag| if (use_old) try self.airBinOp(inst, tag) else { | 2517 | .bit_and, .bit_or, .xor => |air_tag| if (use_old) try cg.airBinOp(inst, air_tag) else { |
| 2473 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; | 2518 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; |
| 2474 | var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); | 2519 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 2475 | try self.spillEflagsIfOccupied(); | | |
| 2476 | const mir_tag: Mir.Inst.Tag = switch (tag) { | | |
| 2477 | else => unreachable, | | |
| 2478 | .bit_and => .@"and", | | |
| 2479 | .bit_or => .@"or", | | |
| 2480 | .xor => .xor, | | |
| 2481 | }; | | |
| 2482 | var res: [1]Temp = undefined; | 2520 | var res: [1]Temp = undefined; |
| 2483 | try self.select(&res, &.{ &ops[0], &ops[1] }, &.{ | 2521 | try cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, pattern_sets: switch (air_tag) { |
| 2484 | .{ | 2522 | else => unreachable, |
| 2485 | .required_features = &.{.avx2}, | 2523 | inline .bit_and, .bit_or, .xor => |ct_air_tag| { |
| 2486 | .mir_tag = .{ .vp_, mir_tag }, | 2524 | const mir_tag: Mir.Inst.Tag = switch (ct_air_tag) { |
| 2487 | .patterns = &.{ | 2525 | else => unreachable, |
| 2488 | .{ .ops = &.{ .ymm, .ymm, .mem } }, | 2526 | .bit_and => .@"and", |
| 2489 | .{ .ops = &.{ .ymm, .mem, .ymm }, .commute = .{ 1, 2 } }, | 2527 | .bit_or => .@"or", |
| 2490 | .{ .ops = &.{ .ymm, .ymm, .ymm } }, | 2528 | .xor => .xor, |
| 2491 | }, | 2529 | }; |
| 2492 | }, | 2530 | break :pattern_sets &.{ |
| 2493 | .{ | 2531 | .{ |
| 2494 | .required_features = &.{.avx}, | 2532 | .required_features = &.{.avx2}, |
| 2495 | .mir_tag = .{ .vp_, mir_tag }, | 2533 | .mir_tag = .{ .vp_, mir_tag }, |
| 2496 | .patterns = &.{ | 2534 | .patterns = &.{ |
| 2497 | .{ .ops = &.{ .xmm, .xmm, .mem } }, | 2535 | .{ .ops = &.{ .ymm, .ymm, .mem } }, |
| 2498 | .{ .ops = &.{ .xmm, .mem, .xmm }, .commute = .{ 1, 2 } }, | 2536 | .{ .ops = &.{ .ymm, .mem, .ymm }, .commute = .{ 1, 2 } }, |
| 2499 | .{ .ops = &.{ .xmm, .xmm, .xmm } }, | 2537 | .{ .ops = &.{ .ymm, .ymm, .ymm } }, |
| 2500 | }, | 2538 | }, |
| 2501 | }, | 2539 | }, |
| 2502 | .{ | 2540 | .{ |
| 2503 | .required_features = &.{.sse2}, | 2541 | .required_features = &.{.avx}, |
| 2504 | .mir_tag = .{ .p_, mir_tag }, | 2542 | .mir_tag = .{ .vp_, mir_tag }, |
| 2505 | .patterns = &.{ | 2543 | .patterns = &.{ |
| 2506 | .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .mem } }, | 2544 | .{ .ops = &.{ .xmm, .xmm, .mem } }, |
| 2507 | .{ .ops = &.{ .xmm, .mem, .{ .implicit = 0 } } }, | 2545 | .{ .ops = &.{ .xmm, .mem, .xmm }, .commute = .{ 1, 2 } }, |
| 2508 | .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .xmm } }, | 2546 | .{ .ops = &.{ .xmm, .xmm, .xmm } }, |
| 2509 | .{ .ops = &.{ .xmm, .xmm, .{ .implicit = 0 } } }, | 2547 | }, |
| 2510 | }, | 2548 | }, |
| 2511 | }, | 2549 | .{ |
| 2512 | .{ | 2550 | .required_features = &.{.sse2}, |
| 2513 | .required_features = &.{.mmx}, | 2551 | .mir_tag = .{ .p_, mir_tag }, |
| 2514 | .mir_tag = .{ .p_, mir_tag }, | 2552 | .patterns = &.{ |
| 2515 | .patterns = &.{ | 2553 | .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .mem } }, |
| 2516 | .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mem } }, | 2554 | .{ .ops = &.{ .xmm, .mem, .{ .implicit = 0 } } }, |
| 2517 | .{ .ops = &.{ .mm, .mem, .{ .implicit = 0 } } }, | 2555 | .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .xmm } }, |
| 2518 | .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mm } }, | 2556 | .{ .ops = &.{ .xmm, .xmm, .{ .implicit = 0 } } }, |
| 2519 | .{ .ops = &.{ .mm, .mm, .{ .implicit = 0 } } }, | 2557 | }, |
| 2520 | }, | 2558 | }, |
| 2521 | }, | 2559 | .{ |
| 2522 | .{ | 2560 | .required_features = &.{.mmx}, |
| 2523 | .mir_tag = .{ ._, mir_tag }, | 2561 | .mir_tag = .{ .p_, mir_tag }, |
| 2524 | .patterns = &.{ | 2562 | .patterns = &.{ |
| 2525 | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .simm32 } }, | 2563 | .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mem } }, |
| 2526 | .{ .ops = &.{ .mem, .simm32, .{ .implicit = 0 } } }, | 2564 | .{ .ops = &.{ .mm, .mem, .{ .implicit = 0 } } }, |
| 2527 | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .gpr } }, | 2565 | .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mm } }, |
| 2528 | .{ .ops = &.{ .mem, .gpr, .{ .implicit = 0 } } }, | 2566 | .{ .ops = &.{ .mm, .mm, .{ .implicit = 0 } } }, |
| 2529 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .simm32 } }, | 2567 | }, |
| 2530 | .{ .ops = &.{ .gpr, .simm32, .{ .implicit = 0 } } }, | 2568 | }, |
| 2531 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .mem } }, | 2569 | .{ |
| 2532 | .{ .ops = &.{ .gpr, .mem, .{ .implicit = 0 } } }, | 2570 | .clobbers = .{ .eflags = true }, |
| 2533 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .gpr } }, | 2571 | .mir_tag = .{ ._, mir_tag }, |
| 2534 | .{ .ops = &.{ .gpr, .gpr, .{ .implicit = 0 } } }, | 2572 | .patterns = &.{ |
| 2535 | }, | 2573 | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .simm32 } }, |
| 2536 | }, | 2574 | .{ .ops = &.{ .mem, .simm32, .{ .implicit = 0 } } }, |
| | 2575 | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .gpr } }, |
| | 2576 | .{ .ops = &.{ .mem, .gpr, .{ .implicit = 0 } } }, |
| | 2577 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .simm32 } }, |
| | 2578 | .{ .ops = &.{ .gpr, .simm32, .{ .implicit = 0 } } }, |
| | 2579 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .mem } }, |
| | 2580 | .{ .ops = &.{ .gpr, .mem, .{ .implicit = 0 } } }, |
| | 2581 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .gpr } }, |
| | 2582 | .{ .ops = &.{ .gpr, .gpr, .{ .implicit = 0 } } }, |
| | 2583 | }, |
| | 2584 | }, |
| 2537 | | 2585 | |
| 2538 | .{ | 2586 | .{ |
| 2539 | .required_features = &.{.avx2}, | 2587 | .required_features = &.{.avx2}, |
| 2540 | .loop = .bitwise, | 2588 | .loop = .bitwise, |
| 2541 | .mir_tag = .{ .vp_, mir_tag }, | 2589 | .mir_tag = .{ .vp_, mir_tag }, |
| 2542 | .patterns = &.{ | 2590 | .patterns = &.{ |
| 2543 | .{ .ops = &.{ .ymm_limb, .{ .explicit = 0 }, .mem_limb } }, | 2591 | .{ .ops = &.{ .ymm_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2544 | .{ .ops = &.{ .ymm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, | 2592 | .{ .ops = &.{ .ymm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2545 | .{ .ops = &.{ .ymm_limb, .ymm_limb, .mem_limb } }, | 2593 | .{ .ops = &.{ .ymm_limb, .ymm_limb, .mem_limb } }, |
| 2546 | }, | 2594 | .{ .ops = &.{ .ymm_limb, .ymm_limb, .ymm_limb } }, |
| 2547 | }, | 2595 | }, |
| 2548 | .{ | 2596 | }, |
| 2549 | .required_features = &.{.avx}, | 2597 | .{ |
| 2550 | .loop = .bitwise, | 2598 | .required_features = &.{.avx}, |
| 2551 | .mir_tag = .{ .vp_, mir_tag }, | 2599 | .loop = .bitwise, |
| 2552 | .patterns = &.{ | 2600 | .mir_tag = .{ .vp_, mir_tag }, |
| 2553 | .{ .ops = &.{ .xmm_limb, .{ .explicit = 0 }, .mem_limb } }, | 2601 | .patterns = &.{ |
| 2554 | .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, | 2602 | .{ .ops = &.{ .xmm_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2555 | .{ .ops = &.{ .xmm_limb, .xmm_limb, .mem_limb } }, | 2603 | .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2556 | }, | 2604 | .{ .ops = &.{ .xmm_limb, .xmm_limb, .mem_limb } }, |
| 2557 | }, | 2605 | .{ .ops = &.{ .xmm_limb, .xmm_limb, .xmm_limb } }, |
| 2558 | .{ | 2606 | }, |
| 2559 | .required_features = &.{.sse2}, | 2607 | }, |
| 2560 | .loop = .bitwise, | 2608 | .{ |
| 2561 | .mir_tag = .{ .p_, mir_tag }, | 2609 | .required_features = &.{.sse2}, |
| 2562 | .patterns = &.{ | 2610 | .loop = .bitwise, |
| 2563 | .{ .ops = &.{ .xmm_limb, .{ .implicit = 0 }, .mem_limb } }, | 2611 | .mir_tag = .{ .p_, mir_tag }, |
| 2564 | .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .implicit = 0 } } }, | 2612 | .patterns = &.{ |
| 2565 | }, | 2613 | .{ .ops = &.{ .xmm_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 2566 | }, | 2614 | .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 2567 | .{ | 2615 | .{ .ops = &.{ .xmm_limb, .{ .implicit = 0 }, .xmm_limb } }, |
| 2568 | .required_features = &.{.mmx}, | 2616 | }, |
| 2569 | .loop = .bitwise, | 2617 | }, |
| 2570 | .mir_tag = .{ .p_, mir_tag }, | 2618 | .{ |
| 2571 | .patterns = &.{ | 2619 | .required_features = &.{.mmx}, |
| 2572 | .{ .ops = &.{ .mm_limb, .{ .implicit = 0 }, .mem_limb } }, | 2620 | .loop = .bitwise, |
| 2573 | .{ .ops = &.{ .mm_limb, .mem_limb, .{ .implicit = 0 } } }, | 2621 | .mir_tag = .{ .p_, mir_tag }, |
| 2574 | }, | 2622 | .patterns = &.{ |
| 2575 | }, | 2623 | .{ .ops = &.{ .mm_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 2576 | .{ | 2624 | .{ .ops = &.{ .mm_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 2577 | .loop = .bitwise, | 2625 | .{ .ops = &.{ .mm_limb, .{ .implicit = 0 }, .mm_limb } }, |
| 2578 | .mir_tag = .{ ._, mir_tag }, | 2626 | }, |
| 2579 | .patterns = &.{ | 2627 | }, |
| 2580 | .{ .ops = &.{ .mem_limb, .{ .implicit = 0 }, .gpr_limb } }, | 2628 | .{ |
| 2581 | .{ .ops = &.{ .mem_limb, .gpr_limb, .{ .implicit = 0 } } }, | 2629 | .clobbers = .{ .eflags = true }, |
| 2582 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .mem_limb } }, | 2630 | .loop = .bitwise, |
| 2583 | .{ .ops = &.{ .gpr_limb, .mem_limb, .{ .implicit = 0 } } }, | 2631 | .mir_tag = .{ ._, mir_tag }, |
| 2584 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .gpr_limb } }, | 2632 | .patterns = &.{ |
| 2585 | }, | 2633 | .{ .ops = &.{ .mem_limb, .{ .implicit = 0 }, .gpr_limb } }, |
| | 2634 | .{ .ops = &.{ .mem_limb, .gpr_limb, .{ .implicit = 0 } } }, |
| | 2635 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .mem_limb } }, |
| | 2636 | .{ .ops = &.{ .gpr_limb, .mem_limb, .{ .implicit = 0 } } }, |
| | 2637 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .gpr_limb } }, |
| | 2638 | }, |
| | 2639 | }, |
| | 2640 | }; |
| 2586 | }, | 2641 | }, |
| 2587 | }); | 2642 | }, .{}); |
| 2588 | if (ops[0].index != res[0].index) try ops[0].die(self); | 2643 | if (ops[0].index != res[0].index) try ops[0].die(cg); |
| 2589 | if (ops[1].index != res[0].index) try ops[1].die(self); | 2644 | if (ops[1].index != res[0].index) try ops[1].die(cg); |
| 2590 | try res[0].moveTo(inst, self); | 2645 | try res[0].moveTo(inst, cg); |
| 2591 | }, | 2646 | }, |
| 2592 | .block => if (use_old) try self.airBlock(inst) else { | 2647 | .block => if (use_old) try cg.airBlock(inst) else { |
| 2593 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; | 2648 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2594 | const extra = self.air.extraData(Air.Block, ty_pl.payload); | 2649 | const extra = cg.air.extraData(Air.Block, ty_pl.payload); |
| 2595 | try self.asmPseudo(.pseudo_dbg_enter_block_none); | 2650 | try cg.asmPseudo(.pseudo_dbg_enter_block_none); |
| 2596 | try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); | 2651 | try cg.lowerBlock(inst, @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len])); |
| 2597 | try self.asmPseudo(.pseudo_dbg_leave_block_none); | 2652 | try cg.asmPseudo(.pseudo_dbg_leave_block_none); |
| 2598 | }, | 2653 | }, |
| 2599 | .loop => if (use_old) try self.airLoop(inst) else { | 2654 | .loop => if (use_old) try cg.airLoop(inst) else { |
| 2600 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; | 2655 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2601 | const extra = self.air.extraData(Air.Block, ty_pl.payload); | 2656 | const extra = cg.air.extraData(Air.Block, ty_pl.payload); |
| 2602 | self.scope_generation += 1; | 2657 | cg.scope_generation += 1; |
| 2603 | try self.loops.putNoClobber(self.gpa, inst, .{ | 2658 | try cg.loops.putNoClobber(cg.gpa, inst, .{ |
| 2604 | .state = try self.saveState(), | 2659 | .state = try cg.saveState(), |
| 2605 | .target = @intCast(self.mir_instructions.len), | 2660 | .target = @intCast(cg.mir_instructions.len), |
| 2606 | }); | 2661 | }); |
| 2607 | defer assert(self.loops.remove(inst)); | 2662 | defer assert(cg.loops.remove(inst)); |
| 2608 | try self.genBodyBlock(@ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); | 2663 | try cg.genBodyBlock(@ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len])); |
| 2609 | }, | 2664 | }, |
| 2610 | .repeat => if (use_old) try self.airRepeat(inst) else { | 2665 | .repeat => if (use_old) try cg.airRepeat(inst) else { |
| 2611 | const repeat = air_datas[@intFromEnum(inst)].repeat; | 2666 | const repeat = air_datas[@intFromEnum(inst)].repeat; |
| 2612 | const loop = self.loops.get(repeat.loop_inst).?; | 2667 | const loop = cg.loops.get(repeat.loop_inst).?; |
| 2613 | try self.restoreState(loop.state, &.{}, .{ | 2668 | try cg.restoreState(loop.state, &.{}, .{ |
| 2614 | .emit_instructions = true, | 2669 | .emit_instructions = true, |
| 2615 | .update_tracking = false, | 2670 | .update_tracking = false, |
| 2616 | .resurrect = false, | 2671 | .resurrect = false, |
| 2617 | .close_scope = true, | 2672 | .close_scope = true, |
| 2618 | }); | 2673 | }); |
| 2619 | _ = try self.asmJmpReloc(loop.target); | 2674 | _ = try cg.asmJmpReloc(loop.target); |
| 2620 | }, | 2675 | }, |
| 2621 | .br => try self.airBr(inst), | 2676 | .br => try cg.airBr(inst), |
| 2622 | .trap => try self.asmOpOnly(.{ ._, .ud2 }), | 2677 | .trap => try cg.asmOpOnly(.{ ._, .ud2 }), |
| 2623 | .breakpoint => try self.asmOpOnly(.{ ._, .int3 }), | 2678 | .breakpoint => try cg.asmOpOnly(.{ ._, .int3 }), |
| 2624 | .ret_addr => if (use_old) try self.airRetAddr(inst) else { | 2679 | .ret_addr => if (use_old) try cg.airRetAddr(inst) else { |
| 2625 | var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .load_frame = .{ | 2680 | var slot = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .load_frame = .{ |
| 2626 | .index = .ret_addr, | 2681 | .index = .ret_addr, |
| 2627 | } }); | 2682 | } }); |
| 2628 | while (try slot.toAnyReg(self)) {} | 2683 | while (try slot.toAnyReg(cg)) {} |
| 2629 | try slot.moveTo(inst, self); | 2684 | try slot.moveTo(inst, cg); |
| 2630 | }, | 2685 | }, |
| 2631 | .frame_addr => if (use_old) try self.airFrameAddress(inst) else { | 2686 | .frame_addr => if (use_old) try cg.airFrameAddress(inst) else { |
| 2632 | var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{ | 2687 | var slot = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .lea_frame = .{ |
| 2633 | .index = .base_ptr, | 2688 | .index = .base_ptr, |
| 2634 | } }); | 2689 | } }); |
| 2635 | try slot.moveTo(inst, self); | 2690 | try slot.moveTo(inst, cg); |
| | 2691 | }, |
| | 2692 | .call => try cg.airCall(inst, .auto), |
| | 2693 | .call_always_tail => try cg.airCall(inst, .always_tail), |
| | 2694 | .call_never_tail => try cg.airCall(inst, .never_tail), |
| | 2695 | .call_never_inline => try cg.airCall(inst, .never_inline), |
| | 2696 | |
| | 2697 | .cmp_vector, .cmp_vector_optimized => if (use_old) try cg.airCmpVector(inst) else fallback: { |
| | 2698 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| | 2699 | const extra = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data; |
| | 2700 | switch (extra.compareOperator()) { |
| | 2701 | .eq, .neq => {}, |
| | 2702 | else => break :fallback try cg.airCmpVector(inst), |
| | 2703 | } |
| | 2704 | var ops = try cg.tempsFromOperands(inst, .{ extra.lhs, extra.rhs }); |
| | 2705 | var res: [1]Temp = undefined; |
| | 2706 | switch (extra.compareOperator()) { |
| | 2707 | .lt => unreachable, |
| | 2708 | .lte => unreachable, |
| | 2709 | .eq, .neq => |cmp_op| try cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, &.{ |
| | 2710 | .{ |
| | 2711 | .required_features = &.{.avx2}, |
| | 2712 | .scalar = .{ .any_int = .byte }, |
| | 2713 | .mir_tag = .{ .vp_b, .cmpeq }, |
| | 2714 | .patterns = &.{ |
| | 2715 | .{ .ops = &.{ .ymm_mask, .ymm, .mem } }, |
| | 2716 | .{ .ops = &.{ .ymm_mask, .mem, .ymm }, .commute = .{ 1, 2 } }, |
| | 2717 | .{ .ops = &.{ .ymm_mask, .ymm, .ymm } }, |
| | 2718 | }, |
| | 2719 | }, |
| | 2720 | .{ |
| | 2721 | .required_features = &.{.avx2}, |
| | 2722 | .scalar = .{ .any_int = .word }, |
| | 2723 | .mir_tag = .{ .vp_w, .cmpeq }, |
| | 2724 | .patterns = &.{ |
| | 2725 | .{ .ops = &.{ .ymm_mask, .ymm, .mem } }, |
| | 2726 | .{ .ops = &.{ .ymm_mask, .mem, .ymm }, .commute = .{ 1, 2 } }, |
| | 2727 | .{ .ops = &.{ .ymm_mask, .ymm, .ymm } }, |
| | 2728 | }, |
| | 2729 | }, |
| | 2730 | .{ |
| | 2731 | .required_features = &.{.avx2}, |
| | 2732 | .scalar = .{ .any_int = .dword }, |
| | 2733 | .mir_tag = .{ .vp_d, .cmpeq }, |
| | 2734 | .patterns = &.{ |
| | 2735 | .{ .ops = &.{ .ymm_mask, .ymm, .mem } }, |
| | 2736 | .{ .ops = &.{ .ymm_mask, .mem, .ymm }, .commute = .{ 1, 2 } }, |
| | 2737 | .{ .ops = &.{ .ymm_mask, .ymm, .ymm } }, |
| | 2738 | }, |
| | 2739 | }, |
| | 2740 | .{ |
| | 2741 | .required_features = &.{.avx2}, |
| | 2742 | .scalar = .{ .any_int = .qword }, |
| | 2743 | .mir_tag = .{ .vp_q, .cmpeq }, |
| | 2744 | .patterns = &.{ |
| | 2745 | .{ .ops = &.{ .ymm_mask, .ymm, .mem } }, |
| | 2746 | .{ .ops = &.{ .ymm_mask, .mem, .ymm }, .commute = .{ 1, 2 } }, |
| | 2747 | .{ .ops = &.{ .ymm_mask, .ymm, .ymm } }, |
| | 2748 | }, |
| | 2749 | }, |
| | 2750 | .{ |
| | 2751 | .required_features = &.{.avx}, |
| | 2752 | .scalar = .{ .any_int = .byte }, |
| | 2753 | .mir_tag = .{ .vp_b, .cmpeq }, |
| | 2754 | .patterns = &.{ |
| | 2755 | .{ .ops = &.{ .xmm_mask, .xmm, .mem } }, |
| | 2756 | .{ .ops = &.{ .xmm_mask, .mem, .xmm }, .commute = .{ 1, 2 } }, |
| | 2757 | .{ .ops = &.{ .xmm_mask, .xmm, .xmm } }, |
| | 2758 | }, |
| | 2759 | }, |
| | 2760 | .{ |
| | 2761 | .required_features = &.{.avx}, |
| | 2762 | .scalar = .{ .any_int = .word }, |
| | 2763 | .mir_tag = .{ .vp_w, .cmpeq }, |
| | 2764 | .patterns = &.{ |
| | 2765 | .{ .ops = &.{ .xmm_mask, .xmm, .mem } }, |
| | 2766 | .{ .ops = &.{ .xmm_mask, .mem, .xmm }, .commute = .{ 1, 2 } }, |
| | 2767 | .{ .ops = &.{ .xmm_mask, .xmm, .xmm } }, |
| | 2768 | }, |
| | 2769 | }, |
| | 2770 | .{ |
| | 2771 | .required_features = &.{.avx}, |
| | 2772 | .scalar = .{ .any_int = .dword }, |
| | 2773 | .mir_tag = .{ .vp_d, .cmpeq }, |
| | 2774 | .patterns = &.{ |
| | 2775 | .{ .ops = &.{ .xmm_mask, .xmm, .mem } }, |
| | 2776 | .{ .ops = &.{ .xmm_mask, .mem, .xmm }, .commute = .{ 1, 2 } }, |
| | 2777 | .{ .ops = &.{ .xmm_mask, .xmm, .xmm } }, |
| | 2778 | }, |
| | 2779 | }, |
| | 2780 | .{ |
| | 2781 | .required_features = &.{.avx}, |
| | 2782 | .scalar = .{ .any_int = .qword }, |
| | 2783 | .mir_tag = .{ .vp_q, .cmpeq }, |
| | 2784 | .patterns = &.{ |
| | 2785 | .{ .ops = &.{ .xmm_mask, .xmm, .mem } }, |
| | 2786 | .{ .ops = &.{ .xmm_mask, .mem, .xmm }, .commute = .{ 1, 2 } }, |
| | 2787 | .{ .ops = &.{ .xmm_mask, .xmm, .xmm } }, |
| | 2788 | }, |
| | 2789 | }, |
| | 2790 | .{ |
| | 2791 | .required_features = &.{.sse2}, |
| | 2792 | .scalar = .{ .any_int = .byte }, |
| | 2793 | .mir_tag = .{ .p_b, .cmpeq }, |
| | 2794 | .patterns = &.{ |
| | 2795 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .mem } }, |
| | 2796 | .{ .ops = &.{ .xmm_mask, .mem, .{ .implicit = 0 } } }, |
| | 2797 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .xmm } }, |
| | 2798 | .{ .ops = &.{ .xmm_mask, .xmm, .{ .implicit = 0 } } }, |
| | 2799 | }, |
| | 2800 | }, |
| | 2801 | .{ |
| | 2802 | .required_features = &.{.sse2}, |
| | 2803 | .scalar = .{ .any_int = .word }, |
| | 2804 | .mir_tag = .{ .p_w, .cmpeq }, |
| | 2805 | .patterns = &.{ |
| | 2806 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .mem } }, |
| | 2807 | .{ .ops = &.{ .xmm_mask, .mem, .{ .implicit = 0 } } }, |
| | 2808 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .xmm } }, |
| | 2809 | .{ .ops = &.{ .xmm_mask, .xmm, .{ .implicit = 0 } } }, |
| | 2810 | }, |
| | 2811 | }, |
| | 2812 | .{ |
| | 2813 | .required_features = &.{.sse2}, |
| | 2814 | .scalar = .{ .any_int = .dword }, |
| | 2815 | .mir_tag = .{ .p_d, .cmpeq }, |
| | 2816 | .patterns = &.{ |
| | 2817 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .mem } }, |
| | 2818 | .{ .ops = &.{ .xmm_mask, .mem, .{ .implicit = 0 } } }, |
| | 2819 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .xmm } }, |
| | 2820 | .{ .ops = &.{ .xmm_mask, .xmm, .{ .implicit = 0 } } }, |
| | 2821 | }, |
| | 2822 | }, |
| | 2823 | .{ |
| | 2824 | .required_features = &.{.sse4_1}, |
| | 2825 | .scalar = .{ .any_int = .qword }, |
| | 2826 | .mir_tag = .{ .p_q, .cmpeq }, |
| | 2827 | .patterns = &.{ |
| | 2828 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .mem } }, |
| | 2829 | .{ .ops = &.{ .xmm_mask, .mem, .{ .implicit = 0 } } }, |
| | 2830 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .xmm } }, |
| | 2831 | .{ .ops = &.{ .xmm_mask, .xmm, .{ .implicit = 0 } } }, |
| | 2832 | }, |
| | 2833 | }, |
| | 2834 | .{ |
| | 2835 | .required_features = &.{.mmx}, |
| | 2836 | .scalar = .{ .any_int = .byte }, |
| | 2837 | .mir_tag = .{ .p_b, .cmpeq }, |
| | 2838 | .patterns = &.{ |
| | 2839 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mem } }, |
| | 2840 | .{ .ops = &.{ .mm_mask, .mem, .{ .implicit = 0 } } }, |
| | 2841 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mm } }, |
| | 2842 | .{ .ops = &.{ .mm_mask, .mm, .{ .implicit = 0 } } }, |
| | 2843 | }, |
| | 2844 | }, |
| | 2845 | .{ |
| | 2846 | .required_features = &.{.mmx}, |
| | 2847 | .scalar = .{ .any_int = .word }, |
| | 2848 | .mir_tag = .{ .p_w, .cmpeq }, |
| | 2849 | .patterns = &.{ |
| | 2850 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mem } }, |
| | 2851 | .{ .ops = &.{ .mm_mask, .mem, .{ .implicit = 0 } } }, |
| | 2852 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mm } }, |
| | 2853 | .{ .ops = &.{ .mm_mask, .mm, .{ .implicit = 0 } } }, |
| | 2854 | }, |
| | 2855 | }, |
| | 2856 | .{ |
| | 2857 | .required_features = &.{.mmx}, |
| | 2858 | .scalar = .{ .any_int = .dword }, |
| | 2859 | .mir_tag = .{ .p_d, .cmpeq }, |
| | 2860 | .patterns = &.{ |
| | 2861 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mem } }, |
| | 2862 | .{ .ops = &.{ .mm_mask, .mem, .{ .implicit = 0 } } }, |
| | 2863 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mm } }, |
| | 2864 | .{ .ops = &.{ .mm_mask, .mm, .{ .implicit = 0 } } }, |
| | 2865 | }, |
| | 2866 | }, |
| | 2867 | .{ |
| | 2868 | .scalar = .bool, |
| | 2869 | .clobbers = .{ .eflags = true }, |
| | 2870 | .invert_result = true, |
| | 2871 | .mir_tag = .{ ._, .xor }, |
| | 2872 | .patterns = &.{ |
| | 2873 | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .simm32 } }, |
| | 2874 | .{ .ops = &.{ .mem, .simm32, .{ .implicit = 0 } } }, |
| | 2875 | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .gpr } }, |
| | 2876 | .{ .ops = &.{ .mem, .gpr, .{ .implicit = 0 } } }, |
| | 2877 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .simm32 } }, |
| | 2878 | .{ .ops = &.{ .gpr, .simm32, .{ .implicit = 0 } } }, |
| | 2879 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .mem } }, |
| | 2880 | .{ .ops = &.{ .gpr, .mem, .{ .implicit = 0 } } }, |
| | 2881 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .gpr } }, |
| | 2882 | .{ .ops = &.{ .gpr, .gpr, .{ .implicit = 0 } } }, |
| | 2883 | }, |
| | 2884 | }, |
| | 2885 | |
| | 2886 | .{ |
| | 2887 | .required_features = &.{.avx2}, |
| | 2888 | .scalar = .{ .any_int = .byte }, |
| | 2889 | .loop = .elementwise, |
| | 2890 | .mir_tag = .{ .vp_b, .cmpeq }, |
| | 2891 | .patterns = &.{ |
| | 2892 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| | 2893 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| | 2894 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, |
| | 2895 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, |
| | 2896 | }, |
| | 2897 | }, |
| | 2898 | .{ |
| | 2899 | .required_features = &.{.avx2}, |
| | 2900 | .scalar = .{ .any_int = .word }, |
| | 2901 | .loop = .elementwise, |
| | 2902 | .mir_tag = .{ .vp_w, .cmpeq }, |
| | 2903 | .patterns = &.{ |
| | 2904 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| | 2905 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| | 2906 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, |
| | 2907 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, |
| | 2908 | }, |
| | 2909 | }, |
| | 2910 | .{ |
| | 2911 | .required_features = &.{.avx2}, |
| | 2912 | .scalar = .{ .any_int = .dword }, |
| | 2913 | .loop = .elementwise, |
| | 2914 | .mir_tag = .{ .vp_d, .cmpeq }, |
| | 2915 | .patterns = &.{ |
| | 2916 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| | 2917 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| | 2918 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, |
| | 2919 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, |
| | 2920 | }, |
| | 2921 | }, |
| | 2922 | .{ |
| | 2923 | .required_features = &.{.avx2}, |
| | 2924 | .scalar = .{ .any_int = .qword }, |
| | 2925 | .loop = .elementwise, |
| | 2926 | .mir_tag = .{ .vp_q, .cmpeq }, |
| | 2927 | .patterns = &.{ |
| | 2928 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| | 2929 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| | 2930 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, |
| | 2931 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, |
| | 2932 | }, |
| | 2933 | }, |
| | 2934 | .{ |
| | 2935 | .required_features = &.{.avx}, |
| | 2936 | .scalar = .{ .any_int = .byte }, |
| | 2937 | .loop = .elementwise, |
| | 2938 | .mir_tag = .{ .vp_b, .cmpeq }, |
| | 2939 | .patterns = &.{ |
| | 2940 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| | 2941 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| | 2942 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, |
| | 2943 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, |
| | 2944 | }, |
| | 2945 | }, |
| | 2946 | .{ |
| | 2947 | .required_features = &.{.avx}, |
| | 2948 | .scalar = .{ .any_int = .word }, |
| | 2949 | .loop = .elementwise, |
| | 2950 | .mir_tag = .{ .vp_w, .cmpeq }, |
| | 2951 | .patterns = &.{ |
| | 2952 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| | 2953 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| | 2954 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, |
| | 2955 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, |
| | 2956 | }, |
| | 2957 | }, |
| | 2958 | .{ |
| | 2959 | .required_features = &.{.avx}, |
| | 2960 | .scalar = .{ .any_int = .dword }, |
| | 2961 | .loop = .elementwise, |
| | 2962 | .mir_tag = .{ .vp_d, .cmpeq }, |
| | 2963 | .patterns = &.{ |
| | 2964 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| | 2965 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| | 2966 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, |
| | 2967 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, |
| | 2968 | }, |
| | 2969 | }, |
| | 2970 | .{ |
| | 2971 | .required_features = &.{.avx}, |
| | 2972 | .scalar = .{ .any_int = .qword }, |
| | 2973 | .loop = .elementwise, |
| | 2974 | .mir_tag = .{ .vp_q, .cmpeq }, |
| | 2975 | .patterns = &.{ |
| | 2976 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| | 2977 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| | 2978 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, |
| | 2979 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, |
| | 2980 | }, |
| | 2981 | }, |
| | 2982 | .{ |
| | 2983 | .required_features = &.{.sse2}, |
| | 2984 | .scalar = .{ .any_int = .byte }, |
| | 2985 | .loop = .elementwise, |
| | 2986 | .mir_tag = .{ .p_b, .cmpeq }, |
| | 2987 | .patterns = &.{ |
| | 2988 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| | 2989 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| | 2990 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, |
| | 2991 | }, |
| | 2992 | }, |
| | 2993 | .{ |
| | 2994 | .required_features = &.{.sse2}, |
| | 2995 | .scalar = .{ .any_int = .word }, |
| | 2996 | .loop = .elementwise, |
| | 2997 | .mir_tag = .{ .p_w, .cmpeq }, |
| | 2998 | .patterns = &.{ |
| | 2999 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| | 3000 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| | 3001 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, |
| | 3002 | }, |
| | 3003 | }, |
| | 3004 | .{ |
| | 3005 | .required_features = &.{.sse2}, |
| | 3006 | .scalar = .{ .any_int = .dword }, |
| | 3007 | .loop = .elementwise, |
| | 3008 | .mir_tag = .{ .p_d, .cmpeq }, |
| | 3009 | .patterns = &.{ |
| | 3010 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| | 3011 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| | 3012 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, |
| | 3013 | }, |
| | 3014 | }, |
| | 3015 | .{ |
| | 3016 | .required_features = &.{.sse4_1}, |
| | 3017 | .scalar = .{ .any_int = .qword }, |
| | 3018 | .loop = .elementwise, |
| | 3019 | .mir_tag = .{ .p_q, .cmpeq }, |
| | 3020 | .patterns = &.{ |
| | 3021 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| | 3022 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| | 3023 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, |
| | 3024 | }, |
| | 3025 | }, |
| | 3026 | .{ |
| | 3027 | .required_features = &.{.mmx}, |
| | 3028 | .scalar = .{ .any_int = .byte }, |
| | 3029 | .loop = .elementwise, |
| | 3030 | .mir_tag = .{ .p_b, .cmpeq }, |
| | 3031 | .patterns = &.{ |
| | 3032 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| | 3033 | .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| | 3034 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } }, |
| | 3035 | }, |
| | 3036 | }, |
| | 3037 | .{ |
| | 3038 | .required_features = &.{.mmx}, |
| | 3039 | .scalar = .{ .any_int = .word }, |
| | 3040 | .loop = .elementwise, |
| | 3041 | .mir_tag = .{ .p_w, .cmpeq }, |
| | 3042 | .patterns = &.{ |
| | 3043 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| | 3044 | .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| | 3045 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } }, |
| | 3046 | }, |
| | 3047 | }, |
| | 3048 | .{ |
| | 3049 | .required_features = &.{.mmx}, |
| | 3050 | .scalar = .{ .any_int = .dword }, |
| | 3051 | .loop = .elementwise, |
| | 3052 | .mir_tag = .{ .p_d, .cmpeq }, |
| | 3053 | .patterns = &.{ |
| | 3054 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| | 3055 | .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| | 3056 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } }, |
| | 3057 | }, |
| | 3058 | }, |
| | 3059 | .{ |
| | 3060 | .scalar = .bool, |
| | 3061 | .clobbers = .{ .eflags = true }, |
| | 3062 | .invert_result = true, |
| | 3063 | .loop = .elementwise, |
| | 3064 | .mir_tag = .{ ._, .xor }, |
| | 3065 | .patterns = &.{ |
| | 3066 | .{ .ops = &.{ .mem_limb, .{ .implicit = 0 }, .gpr_limb } }, |
| | 3067 | .{ .ops = &.{ .mem_limb, .gpr_limb, .{ .implicit = 0 } } }, |
| | 3068 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .mem_limb } }, |
| | 3069 | .{ .ops = &.{ .gpr_limb, .mem_limb, .{ .implicit = 0 } } }, |
| | 3070 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .gpr_limb } }, |
| | 3071 | }, |
| | 3072 | }, |
| | 3073 | }, .{ .invert_result = switch (cmp_op) { |
| | 3074 | .eq => false, |
| | 3075 | .neq => true, |
| | 3076 | else => unreachable, |
| | 3077 | } }), |
| | 3078 | .gte => unreachable, |
| | 3079 | .gt => unreachable, |
| | 3080 | } |
| | 3081 | if (ops[0].index != res[0].index) try ops[0].die(cg); |
| | 3082 | if (ops[1].index != res[0].index) try ops[1].die(cg); |
| | 3083 | try res[0].moveTo(inst, cg); |
| 2636 | }, | 3084 | }, |
| 2637 | .call => try self.airCall(inst, .auto), | 3085 | |
| 2638 | .call_always_tail => try self.airCall(inst, .always_tail), | 3086 | .cond_br => try cg.airCondBr(inst), |
| 2639 | .call_never_tail => try self.airCall(inst, .never_tail), | 3087 | .switch_br => try cg.airSwitchBr(inst), |
| 2640 | .call_never_inline => try self.airCall(inst, .never_inline), | 3088 | .loop_switch_br => try cg.airLoopSwitchBr(inst), |
| 2641 | | 3089 | .switch_dispatch => try cg.airSwitchDispatch(inst), |
| 2642 | .cond_br => try self.airCondBr(inst), | 3090 | .@"try", .try_cold => try cg.airTry(inst), |
| 2643 | .switch_br => try self.airSwitchBr(inst), | 3091 | .try_ptr, .try_ptr_cold => try cg.airTryPtr(inst), |
| 2644 | .loop_switch_br => try self.airLoopSwitchBr(inst), | 3092 | .dbg_stmt => if (use_old) try cg.airDbgStmt(inst) else { |
| 2645 | .switch_dispatch => try self.airSwitchDispatch(inst), | | |
| 2646 | .@"try", .try_cold => try self.airTry(inst), | | |
| 2647 | .try_ptr, .try_ptr_cold => try self.airTryPtr(inst), | | |
| 2648 | .dbg_stmt => if (use_old) try self.airDbgStmt(inst) else { | | |
| 2649 | const dbg_stmt = air_datas[@intFromEnum(inst)].dbg_stmt; | 3093 | const dbg_stmt = air_datas[@intFromEnum(inst)].dbg_stmt; |
| 2650 | _ = try self.addInst(.{ | 3094 | _ = try cg.addInst(.{ |
| 2651 | .tag = .pseudo, | 3095 | .tag = .pseudo, |
| 2652 | .ops = .pseudo_dbg_line_line_column, | 3096 | .ops = .pseudo_dbg_line_line_column, |
| 2653 | .data = .{ .line_column = .{ | 3097 | .data = .{ .line_column = .{ |
| ... | @@ -2656,50 +3100,50 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2656,50 +3100,50 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2656 | } }, | 3100 | } }, |
| 2657 | }); | 3101 | }); |
| 2658 | }, | 3102 | }, |
| 2659 | .dbg_empty_stmt => if (use_old) try self.airDbgEmptyStmt() else { | 3103 | .dbg_empty_stmt => if (use_old) try cg.airDbgEmptyStmt() else { |
| 2660 | if (self.mir_instructions.len > 0) { | 3104 | if (cg.mir_instructions.len > 0) { |
| 2661 | const prev_mir_op = &self.mir_instructions.items(.ops)[self.mir_instructions.len - 1]; | 3105 | const prev_mir_op = &cg.mir_instructions.items(.ops)[cg.mir_instructions.len - 1]; |
| 2662 | if (prev_mir_op.* == .pseudo_dbg_line_stmt_line_column) | 3106 | if (prev_mir_op.* == .pseudo_dbg_line_stmt_line_column) |
| 2663 | prev_mir_op.* = .pseudo_dbg_line_line_column; | 3107 | prev_mir_op.* = .pseudo_dbg_line_line_column; |
| 2664 | } | 3108 | } |
| 2665 | try self.asmOpOnly(.{ ._, .nop }); | 3109 | try cg.asmOpOnly(.{ ._, .nop }); |
| 2666 | }, | 3110 | }, |
| 2667 | .dbg_inline_block => if (use_old) try self.airDbgInlineBlock(inst) else { | 3111 | .dbg_inline_block => if (use_old) try cg.airDbgInlineBlock(inst) else { |
| 2668 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; | 3112 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2669 | const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload); | 3113 | const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload); |
| 2670 | const old_inline_func = self.inline_func; | 3114 | const old_inline_func = cg.inline_func; |
| 2671 | defer self.inline_func = old_inline_func; | 3115 | defer cg.inline_func = old_inline_func; |
| 2672 | self.inline_func = extra.data.func; | 3116 | cg.inline_func = extra.data.func; |
| 2673 | _ = try self.addInst(.{ | 3117 | _ = try cg.addInst(.{ |
| 2674 | .tag = .pseudo, | 3118 | .tag = .pseudo, |
| 2675 | .ops = .pseudo_dbg_enter_inline_func, | 3119 | .ops = .pseudo_dbg_enter_inline_func, |
| 2676 | .data = .{ .func = extra.data.func }, | 3120 | .data = .{ .func = extra.data.func }, |
| 2677 | }); | 3121 | }); |
| 2678 | try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); | 3122 | try cg.lowerBlock(inst, @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len])); |
| 2679 | _ = try self.addInst(.{ | 3123 | _ = try cg.addInst(.{ |
| 2680 | .tag = .pseudo, | 3124 | .tag = .pseudo, |
| 2681 | .ops = .pseudo_dbg_leave_inline_func, | 3125 | .ops = .pseudo_dbg_leave_inline_func, |
| 2682 | .data = .{ .func = old_inline_func }, | 3126 | .data = .{ .func = old_inline_func }, |
| 2683 | }); | 3127 | }); |
| 2684 | }, | 3128 | }, |
| 2685 | .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => if (use_old) try self.airDbgVar(inst) else { | 3129 | .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => if (use_old) try cg.airDbgVar(inst) else { |
| 2686 | const pl_op = air_datas[@intFromEnum(inst)].pl_op; | 3130 | const pl_op = air_datas[@intFromEnum(inst)].pl_op; |
| 2687 | var ops = try self.tempsFromOperands(inst, .{pl_op.operand}); | 3131 | var ops = try cg.tempsFromOperands(inst, .{pl_op.operand}); |
| 2688 | try self.genLocalDebugInfo(inst, ops[0].tracking(self).short); | 3132 | try cg.genLocalDebugInfo(inst, ops[0].tracking(cg).short); |
| 2689 | try ops[0].die(self); | 3133 | try ops[0].die(cg); |
| 2690 | }, | 3134 | }, |
| 2691 | .is_null_ptr => if (use_old) try self.airIsNullPtr(inst) else { | 3135 | .is_null_ptr => if (use_old) try cg.airIsNullPtr(inst) else { |
| 2692 | const un_op = air_datas[@intFromEnum(inst)].un_op; | 3136 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2693 | const opt_ty = self.typeOf(un_op).childType(zcu); | 3137 | const opt_ty = cg.typeOf(un_op).childType(zcu); |
| 2694 | const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); | 3138 | const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); |
| 2695 | const opt_child_ty = opt_ty.optionalChild(zcu); | 3139 | const opt_child_ty = opt_ty.optionalChild(zcu); |
| 2696 | const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); | 3140 | const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); |
| 2697 | var ops = try self.tempsFromOperands(inst, .{un_op}); | 3141 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 2698 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, self); | 3142 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); |
| 2699 | while (try ops[0].toLea(self)) {} | 3143 | while (try ops[0].toLea(cg)) {} |
| 2700 | try self.asmMemoryImmediate( | 3144 | try cg.asmMemoryImmediate( |
| 2701 | .{ ._, .cmp }, | 3145 | .{ ._, .cmp }, |
| 2702 | try ops[0].tracking(self).short.deref().mem(self, .{ .size = if (!opt_repr_is_pl) | 3146 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) |
| 2703 | .byte | 3147 | .byte |
| 2704 | else if (opt_child_ty.isSlice(zcu)) | 3148 | else if (opt_child_ty.isSlice(zcu)) |
| 2705 | .qword | 3149 | .qword |
| ... | @@ -2707,22 +3151,22 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2707,22 +3151,22 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2707 | .fromSize(opt_child_abi_size) }), | 3151 | .fromSize(opt_child_abi_size) }), |
| 2708 | .u(0), | 3152 | .u(0), |
| 2709 | ); | 3153 | ); |
| 2710 | var is_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e }); | 3154 | var is_null = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .eflags = .e }); |
| 2711 | try ops[0].die(self); | 3155 | try ops[0].die(cg); |
| 2712 | try is_null.moveTo(inst, self); | 3156 | try is_null.moveTo(inst, cg); |
| 2713 | }, | 3157 | }, |
| 2714 | .is_non_null_ptr => if (use_old) try self.airIsNonNullPtr(inst) else { | 3158 | .is_non_null_ptr => if (use_old) try cg.airIsNonNullPtr(inst) else { |
| 2715 | const un_op = air_datas[@intFromEnum(inst)].un_op; | 3159 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2716 | const opt_ty = self.typeOf(un_op).childType(zcu); | 3160 | const opt_ty = cg.typeOf(un_op).childType(zcu); |
| 2717 | const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); | 3161 | const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); |
| 2718 | const opt_child_ty = opt_ty.optionalChild(zcu); | 3162 | const opt_child_ty = opt_ty.optionalChild(zcu); |
| 2719 | const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); | 3163 | const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); |
| 2720 | var ops = try self.tempsFromOperands(inst, .{un_op}); | 3164 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 2721 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, self); | 3165 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); |
| 2722 | while (try ops[0].toLea(self)) {} | 3166 | while (try ops[0].toLea(cg)) {} |
| 2723 | try self.asmMemoryImmediate( | 3167 | try cg.asmMemoryImmediate( |
| 2724 | .{ ._, .cmp }, | 3168 | .{ ._, .cmp }, |
| 2725 | try ops[0].tracking(self).short.deref().mem(self, .{ .size = if (!opt_repr_is_pl) | 3169 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) |
| 2726 | .byte | 3170 | .byte |
| 2727 | else if (opt_child_ty.isSlice(zcu)) | 3171 | else if (opt_child_ty.isSlice(zcu)) |
| 2728 | .qword | 3172 | .qword |
| ... | @@ -2730,244 +3174,244 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2730,244 +3174,244 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2730 | .fromSize(opt_child_abi_size) }), | 3174 | .fromSize(opt_child_abi_size) }), |
| 2731 | .u(0), | 3175 | .u(0), |
| 2732 | ); | 3176 | ); |
| 2733 | var is_non_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne }); | 3177 | var is_non_null = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .eflags = .ne }); |
| 2734 | try ops[0].die(self); | 3178 | try ops[0].die(cg); |
| 2735 | try is_non_null.moveTo(inst, self); | 3179 | try is_non_null.moveTo(inst, cg); |
| 2736 | }, | 3180 | }, |
| 2737 | .is_err_ptr => if (use_old) try self.airIsErrPtr(inst) else { | 3181 | .is_err_ptr => if (use_old) try cg.airIsErrPtr(inst) else { |
| 2738 | const un_op = air_datas[@intFromEnum(inst)].un_op; | 3182 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2739 | const eu_ty = self.typeOf(un_op).childType(zcu); | 3183 | const eu_ty = cg.typeOf(un_op).childType(zcu); |
| 2740 | const eu_err_ty = eu_ty.errorUnionSet(zcu); | 3184 | const eu_err_ty = eu_ty.errorUnionSet(zcu); |
| 2741 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); | 3185 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); |
| 2742 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); | 3186 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); |
| 2743 | var ops = try self.tempsFromOperands(inst, .{un_op}); | 3187 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 2744 | try ops[0].toOffset(eu_err_off, self); | 3188 | try ops[0].toOffset(eu_err_off, cg); |
| 2745 | while (try ops[0].toLea(self)) {} | 3189 | while (try ops[0].toLea(cg)) {} |
| 2746 | try self.asmMemoryImmediate( | 3190 | try cg.asmMemoryImmediate( |
| 2747 | .{ ._, .cmp }, | 3191 | .{ ._, .cmp }, |
| 2748 | try ops[0].tracking(self).short.deref().mem(self, .{ .size = self.memSize(eu_err_ty) }), | 3192 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), |
| 2749 | .u(0), | 3193 | .u(0), |
| 2750 | ); | 3194 | ); |
| 2751 | var is_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne }); | 3195 | var is_err = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .eflags = .ne }); |
| 2752 | try ops[0].die(self); | 3196 | try ops[0].die(cg); |
| 2753 | try is_err.moveTo(inst, self); | 3197 | try is_err.moveTo(inst, cg); |
| 2754 | }, | 3198 | }, |
| 2755 | .is_non_err_ptr => if (use_old) try self.airIsNonErrPtr(inst) else { | 3199 | .is_non_err_ptr => if (use_old) try cg.airIsNonErrPtr(inst) else { |
| 2756 | const un_op = air_datas[@intFromEnum(inst)].un_op; | 3200 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2757 | const eu_ty = self.typeOf(un_op).childType(zcu); | 3201 | const eu_ty = cg.typeOf(un_op).childType(zcu); |
| 2758 | const eu_err_ty = eu_ty.errorUnionSet(zcu); | 3202 | const eu_err_ty = eu_ty.errorUnionSet(zcu); |
| 2759 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); | 3203 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); |
| 2760 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); | 3204 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); |
| 2761 | var ops = try self.tempsFromOperands(inst, .{un_op}); | 3205 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 2762 | try ops[0].toOffset(eu_err_off, self); | 3206 | try ops[0].toOffset(eu_err_off, cg); |
| 2763 | while (try ops[0].toLea(self)) {} | 3207 | while (try ops[0].toLea(cg)) {} |
| 2764 | try self.asmMemoryImmediate( | 3208 | try cg.asmMemoryImmediate( |
| 2765 | .{ ._, .cmp }, | 3209 | .{ ._, .cmp }, |
| 2766 | try ops[0].tracking(self).short.deref().mem(self, .{ .size = self.memSize(eu_err_ty) }), | 3210 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), |
| 2767 | .u(0), | 3211 | .u(0), |
| 2768 | ); | 3212 | ); |
| 2769 | var is_non_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e }); | 3213 | var is_non_err = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .eflags = .e }); |
| 2770 | try ops[0].die(self); | 3214 | try ops[0].die(cg); |
| 2771 | try is_non_err.moveTo(inst, self); | 3215 | try is_non_err.moveTo(inst, cg); |
| 2772 | }, | 3216 | }, |
| 2773 | .int_from_ptr => if (use_old) try self.airIntFromPtr(inst) else { | 3217 | .int_from_ptr => if (use_old) try cg.airIntFromPtr(inst) else { |
| 2774 | const un_op = air_datas[@intFromEnum(inst)].un_op; | 3218 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2775 | var ops = try self.tempsFromOperands(inst, .{un_op}); | 3219 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 2776 | try ops[0].toLimb(0, self); | 3220 | try ops[0].toLimb(0, cg); |
| 2777 | try ops[0].moveTo(inst, self); | 3221 | try ops[0].moveTo(inst, cg); |
| 2778 | }, | 3222 | }, |
| 2779 | .int_from_bool => if (use_old) try self.airIntFromBool(inst) else { | 3223 | .int_from_bool => if (use_old) try cg.airIntFromBool(inst) else { |
| 2780 | const un_op = air_datas[@intFromEnum(inst)].un_op; | 3224 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2781 | var ops = try self.tempsFromOperands(inst, .{un_op}); | 3225 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 2782 | try ops[0].moveTo(inst, self); | 3226 | try ops[0].moveTo(inst, cg); |
| 2783 | }, | 3227 | }, |
| 2784 | .ret => try self.airRet(inst, false), | 3228 | .ret => try cg.airRet(inst, false), |
| 2785 | .ret_safe => try self.airRet(inst, true), | 3229 | .ret_safe => try cg.airRet(inst, true), |
| 2786 | .ret_load => try self.airRetLoad(inst), | 3230 | .ret_load => try cg.airRetLoad(inst), |
| 2787 | .unreach => {}, | 3231 | .unreach => {}, |
| 2788 | .optional_payload_ptr => if (use_old) try self.airOptionalPayloadPtr(inst) else { | 3232 | .optional_payload_ptr => if (use_old) try cg.airOptionalPayloadPtr(inst) else { |
| 2789 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3233 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2790 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3234 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2791 | try ops[0].moveTo(inst, self); | 3235 | try ops[0].moveTo(inst, cg); |
| 2792 | }, | 3236 | }, |
| 2793 | .optional_payload_ptr_set => if (use_old) try self.airOptionalPayloadPtrSet(inst) else { | 3237 | .optional_payload_ptr_set => if (use_old) try cg.airOptionalPayloadPtrSet(inst) else { |
| 2794 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3238 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2795 | const opt_ty = self.typeOf(ty_op.operand).childType(zcu); | 3239 | const opt_ty = cg.typeOf(ty_op.operand).childType(zcu); |
| 2796 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3240 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2797 | if (!opt_ty.optionalReprIsPayload(zcu)) { | 3241 | if (!opt_ty.optionalReprIsPayload(zcu)) { |
| 2798 | const opt_child_ty = opt_ty.optionalChild(zcu); | 3242 | const opt_child_ty = opt_ty.optionalChild(zcu); |
| 2799 | const opt_child_abi_size: i32 = @intCast(opt_child_ty.abiSize(zcu)); | 3243 | const opt_child_abi_size: i32 = @intCast(opt_child_ty.abiSize(zcu)); |
| 2800 | try ops[0].toOffset(opt_child_abi_size, self); | 3244 | try ops[0].toOffset(opt_child_abi_size, cg); |
| 2801 | var has_value = try self.tempFromValue(Type.bool, .{ .immediate = 1 }); | 3245 | var has_value = try cg.tempFromValue(Type.bool, .{ .immediate = 1 }); |
| 2802 | try ops[0].store(&has_value, self); | 3246 | try ops[0].store(&has_value, cg); |
| 2803 | try has_value.die(self); | 3247 | try has_value.die(cg); |
| 2804 | try ops[0].toOffset(-opt_child_abi_size, self); | 3248 | try ops[0].toOffset(-opt_child_abi_size, cg); |
| 2805 | } | 3249 | } |
| 2806 | try ops[0].moveTo(inst, self); | 3250 | try ops[0].moveTo(inst, cg); |
| 2807 | }, | 3251 | }, |
| 2808 | .unwrap_errunion_payload_ptr => if (use_old) try self.airUnwrapErrUnionPayloadPtr(inst) else { | 3252 | .unwrap_errunion_payload_ptr => if (use_old) try cg.airUnwrapErrUnionPayloadPtr(inst) else { |
| 2809 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3253 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2810 | const eu_ty = self.typeOf(ty_op.operand).childType(zcu); | 3254 | const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); |
| 2811 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); | 3255 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); |
| 2812 | const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); | 3256 | const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); |
| 2813 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3257 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2814 | try ops[0].toOffset(eu_pl_off, self); | 3258 | try ops[0].toOffset(eu_pl_off, cg); |
| 2815 | try ops[0].moveTo(inst, self); | 3259 | try ops[0].moveTo(inst, cg); |
| 2816 | }, | 3260 | }, |
| 2817 | .unwrap_errunion_err_ptr => if (use_old) try self.airUnwrapErrUnionErrPtr(inst) else { | 3261 | .unwrap_errunion_err_ptr => if (use_old) try cg.airUnwrapErrUnionErrPtr(inst) else { |
| 2818 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3262 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2819 | const eu_ty = self.typeOf(ty_op.operand).childType(zcu); | 3263 | const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); |
| 2820 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); | 3264 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); |
| 2821 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); | 3265 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); |
| 2822 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3266 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2823 | try ops[0].toOffset(eu_err_off, self); | 3267 | try ops[0].toOffset(eu_err_off, cg); |
| 2824 | var err = try ops[0].load(eu_ty.errorUnionSet(zcu), self); | 3268 | var err = try ops[0].load(eu_ty.errorUnionSet(zcu), cg); |
| 2825 | try ops[0].die(self); | 3269 | try ops[0].die(cg); |
| 2826 | try err.moveTo(inst, self); | 3270 | try err.moveTo(inst, cg); |
| 2827 | }, | 3271 | }, |
| 2828 | .errunion_payload_ptr_set => if (use_old) try self.airErrUnionPayloadPtrSet(inst) else { | 3272 | .errunion_payload_ptr_set => if (use_old) try cg.airErrUnionPayloadPtrSet(inst) else { |
| 2829 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3273 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2830 | const eu_ty = self.typeOf(ty_op.operand).childType(zcu); | 3274 | const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); |
| 2831 | const eu_err_ty = eu_ty.errorUnionSet(zcu); | 3275 | const eu_err_ty = eu_ty.errorUnionSet(zcu); |
| 2832 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); | 3276 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); |
| 2833 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); | 3277 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); |
| 2834 | const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); | 3278 | const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); |
| 2835 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3279 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2836 | try ops[0].toOffset(eu_err_off, self); | 3280 | try ops[0].toOffset(eu_err_off, cg); |
| 2837 | var no_err = try self.tempFromValue(eu_err_ty, .{ .immediate = 0 }); | 3281 | var no_err = try cg.tempFromValue(eu_err_ty, .{ .immediate = 0 }); |
| 2838 | try ops[0].store(&no_err, self); | 3282 | try ops[0].store(&no_err, cg); |
| 2839 | try no_err.die(self); | 3283 | try no_err.die(cg); |
| 2840 | try ops[0].toOffset(eu_pl_off - eu_err_off, self); | 3284 | try ops[0].toOffset(eu_pl_off - eu_err_off, cg); |
| 2841 | try ops[0].moveTo(inst, self); | 3285 | try ops[0].moveTo(inst, cg); |
| 2842 | }, | 3286 | }, |
| 2843 | .struct_field_ptr => if (use_old) try self.airStructFieldPtr(inst) else { | 3287 | .struct_field_ptr => if (use_old) try cg.airStructFieldPtr(inst) else { |
| 2844 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; | 3288 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2845 | const extra = self.air.extraData(Air.StructField, ty_pl.payload).data; | 3289 | const extra = cg.air.extraData(Air.StructField, ty_pl.payload).data; |
| 2846 | var ops = try self.tempsFromOperands(inst, .{extra.struct_operand}); | 3290 | var ops = try cg.tempsFromOperands(inst, .{extra.struct_operand}); |
| 2847 | try ops[0].toOffset(self.fieldOffset( | 3291 | try ops[0].toOffset(cg.fieldOffset( |
| 2848 | self.typeOf(extra.struct_operand), | 3292 | cg.typeOf(extra.struct_operand), |
| 2849 | self.typeOfIndex(inst), | 3293 | cg.typeOfIndex(inst), |
| 2850 | extra.field_index, | 3294 | extra.field_index, |
| 2851 | ), self); | 3295 | ), cg); |
| 2852 | try ops[0].moveTo(inst, self); | 3296 | try ops[0].moveTo(inst, cg); |
| 2853 | }, | 3297 | }, |
| 2854 | .struct_field_ptr_index_0 => if (use_old) try self.airStructFieldPtrIndex(inst, 0) else { | 3298 | .struct_field_ptr_index_0 => if (use_old) try cg.airStructFieldPtrIndex(inst, 0) else { |
| 2855 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3299 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2856 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3300 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2857 | try ops[0].toOffset(self.fieldOffset( | 3301 | try ops[0].toOffset(cg.fieldOffset( |
| 2858 | self.typeOf(ty_op.operand), | 3302 | cg.typeOf(ty_op.operand), |
| 2859 | self.typeOfIndex(inst), | 3303 | cg.typeOfIndex(inst), |
| 2860 | 0, | 3304 | 0, |
| 2861 | ), self); | 3305 | ), cg); |
| 2862 | try ops[0].moveTo(inst, self); | 3306 | try ops[0].moveTo(inst, cg); |
| 2863 | }, | 3307 | }, |
| 2864 | .struct_field_ptr_index_1 => if (use_old) try self.airStructFieldPtrIndex(inst, 1) else { | 3308 | .struct_field_ptr_index_1 => if (use_old) try cg.airStructFieldPtrIndex(inst, 1) else { |
| 2865 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3309 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2866 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3310 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2867 | try ops[0].toOffset(self.fieldOffset( | 3311 | try ops[0].toOffset(cg.fieldOffset( |
| 2868 | self.typeOf(ty_op.operand), | 3312 | cg.typeOf(ty_op.operand), |
| 2869 | self.typeOfIndex(inst), | 3313 | cg.typeOfIndex(inst), |
| 2870 | 1, | 3314 | 1, |
| 2871 | ), self); | 3315 | ), cg); |
| 2872 | try ops[0].moveTo(inst, self); | 3316 | try ops[0].moveTo(inst, cg); |
| 2873 | }, | 3317 | }, |
| 2874 | .struct_field_ptr_index_2 => if (use_old) try self.airStructFieldPtrIndex(inst, 2) else { | 3318 | .struct_field_ptr_index_2 => if (use_old) try cg.airStructFieldPtrIndex(inst, 2) else { |
| 2875 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3319 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2876 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3320 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2877 | try ops[0].toOffset(self.fieldOffset( | 3321 | try ops[0].toOffset(cg.fieldOffset( |
| 2878 | self.typeOf(ty_op.operand), | 3322 | cg.typeOf(ty_op.operand), |
| 2879 | self.typeOfIndex(inst), | 3323 | cg.typeOfIndex(inst), |
| 2880 | 2, | 3324 | 2, |
| 2881 | ), self); | 3325 | ), cg); |
| 2882 | try ops[0].moveTo(inst, self); | 3326 | try ops[0].moveTo(inst, cg); |
| 2883 | }, | 3327 | }, |
| 2884 | .struct_field_ptr_index_3 => if (use_old) try self.airStructFieldPtrIndex(inst, 3) else { | 3328 | .struct_field_ptr_index_3 => if (use_old) try cg.airStructFieldPtrIndex(inst, 3) else { |
| 2885 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3329 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2886 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3330 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2887 | try ops[0].toOffset(self.fieldOffset( | 3331 | try ops[0].toOffset(cg.fieldOffset( |
| 2888 | self.typeOf(ty_op.operand), | 3332 | cg.typeOf(ty_op.operand), |
| 2889 | self.typeOfIndex(inst), | 3333 | cg.typeOfIndex(inst), |
| 2890 | 3, | 3334 | 3, |
| 2891 | ), self); | 3335 | ), cg); |
| 2892 | try ops[0].moveTo(inst, self); | 3336 | try ops[0].moveTo(inst, cg); |
| 2893 | }, | 3337 | }, |
| 2894 | .slice => if (use_old) try self.airSlice(inst) else { | 3338 | .slice => if (use_old) try cg.airSlice(inst) else { |
| 2895 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; | 3339 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2896 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; | 3340 | const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; |
| 2897 | var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); | 3341 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 2898 | try ops[0].toPair(&ops[1], self); | 3342 | try ops[0].toPair(&ops[1], cg); |
| 2899 | try ops[0].moveTo(inst, self); | 3343 | try ops[0].moveTo(inst, cg); |
| 2900 | }, | 3344 | }, |
| 2901 | .slice_len => if (use_old) try self.airSliceLen(inst) else { | 3345 | .slice_len => if (use_old) try cg.airSliceLen(inst) else { |
| 2902 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3346 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2903 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3347 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2904 | try ops[0].toLimb(1, self); | 3348 | try ops[0].toLimb(1, cg); |
| 2905 | try ops[0].moveTo(inst, self); | 3349 | try ops[0].moveTo(inst, cg); |
| 2906 | }, | 3350 | }, |
| 2907 | .slice_ptr => if (use_old) try self.airSlicePtr(inst) else { | 3351 | .slice_ptr => if (use_old) try cg.airSlicePtr(inst) else { |
| 2908 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3352 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2909 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3353 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2910 | try ops[0].toLimb(0, self); | 3354 | try ops[0].toLimb(0, cg); |
| 2911 | try ops[0].moveTo(inst, self); | 3355 | try ops[0].moveTo(inst, cg); |
| 2912 | }, | 3356 | }, |
| 2913 | .ptr_slice_len_ptr => if (use_old) try self.airPtrSliceLenPtr(inst) else { | 3357 | .ptr_slice_len_ptr => if (use_old) try cg.airPtrSliceLenPtr(inst) else { |
| 2914 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3358 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2915 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3359 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2916 | try ops[0].toOffset(8, self); | 3360 | try ops[0].toOffset(8, cg); |
| 2917 | try ops[0].moveTo(inst, self); | 3361 | try ops[0].moveTo(inst, cg); |
| 2918 | }, | 3362 | }, |
| 2919 | .ptr_slice_ptr_ptr => if (use_old) try self.airPtrSlicePtrPtr(inst) else { | 3363 | .ptr_slice_ptr_ptr => if (use_old) try cg.airPtrSlicePtrPtr(inst) else { |
| 2920 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3364 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2921 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3365 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2922 | try ops[0].toOffset(0, self); | 3366 | try ops[0].toOffset(0, cg); |
| 2923 | try ops[0].moveTo(inst, self); | 3367 | try ops[0].moveTo(inst, cg); |
| 2924 | }, | 3368 | }, |
| 2925 | .slice_elem_ptr, .ptr_elem_ptr => |tag| if (use_old) switch (tag) { | 3369 | .slice_elem_ptr, .ptr_elem_ptr => |tag| if (use_old) switch (tag) { |
| 2926 | else => unreachable, | 3370 | else => unreachable, |
| 2927 | .slice_elem_ptr => try self.airSliceElemPtr(inst), | 3371 | .slice_elem_ptr => try cg.airSliceElemPtr(inst), |
| 2928 | .ptr_elem_ptr => try self.airPtrElemPtr(inst), | 3372 | .ptr_elem_ptr => try cg.airPtrElemPtr(inst), |
| 2929 | } else { | 3373 | } else { |
| 2930 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; | 3374 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2931 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; | 3375 | const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; |
| 2932 | var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); | 3376 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 2933 | switch (tag) { | 3377 | switch (tag) { |
| 2934 | else => unreachable, | 3378 | else => unreachable, |
| 2935 | .slice_elem_ptr => try ops[0].toLimb(0, self), | 3379 | .slice_elem_ptr => try ops[0].toLimb(0, cg), |
| 2936 | .ptr_elem_ptr => {}, | 3380 | .ptr_elem_ptr => {}, |
| 2937 | } | 3381 | } |
| 2938 | const dst_ty = self.typeOfIndex(inst); | 3382 | const dst_ty = cg.typeOfIndex(inst); |
| 2939 | if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: { | 3383 | if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: { |
| 2940 | const elem_size = dst_ty.childType(zcu).abiSize(zcu); | 3384 | const elem_size = dst_ty.childType(zcu).abiSize(zcu); |
| 2941 | if (elem_size == 0) break :zero_offset; | 3385 | if (elem_size == 0) break :zero_offset; |
| 2942 | while (true) for (&ops) |*op| { | 3386 | while (true) for (&ops) |*op| { |
| 2943 | if (try op.toAnyReg(self)) break; | 3387 | if (try op.toAnyReg(cg)) break; |
| 2944 | } else break; | 3388 | } else break; |
| 2945 | const lhs_reg = ops[0].unwrap(self).temp.tracking(self).short.register.to64(); | 3389 | const lhs_reg = ops[0].unwrap(cg).temp.tracking(cg).short.register.to64(); |
| 2946 | const rhs_reg = ops[1].unwrap(self).temp.tracking(self).short.register.to64(); | 3390 | const rhs_reg = ops[1].unwrap(cg).temp.tracking(cg).short.register.to64(); |
| 2947 | if (!std.math.isPowerOfTwo(elem_size)) { | 3391 | if (!std.math.isPowerOfTwo(elem_size)) { |
| 2948 | try self.spillEflagsIfOccupied(); | 3392 | try cg.spillEflagsIfOccupied(); |
| 2949 | try self.asmRegisterRegisterImmediate( | 3393 | try cg.asmRegisterRegisterImmediate( |
| 2950 | .{ .i_, .mul }, | 3394 | .{ .i_, .mul }, |
| 2951 | rhs_reg, | 3395 | rhs_reg, |
| 2952 | rhs_reg, | 3396 | rhs_reg, |
| 2953 | .u(elem_size), | 3397 | .u(elem_size), |
| 2954 | ); | 3398 | ); |
| 2955 | try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ | 3399 | try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 2956 | .base = .{ .reg = lhs_reg }, | 3400 | .base = .{ .reg = lhs_reg }, |
| 2957 | .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, | 3401 | .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, |
| 2958 | }); | 3402 | }); |
| 2959 | } else if (elem_size > 8) { | 3403 | } else if (elem_size > 8) { |
| 2960 | try self.spillEflagsIfOccupied(); | 3404 | try cg.spillEflagsIfOccupied(); |
| 2961 | try self.asmRegisterImmediate( | 3405 | try cg.asmRegisterImmediate( |
| 2962 | .{ ._l, .sh }, | 3406 | .{ ._l, .sh }, |
| 2963 | rhs_reg, | 3407 | rhs_reg, |
| 2964 | .u(std.math.log2_int(u64, elem_size)), | 3408 | .u(std.math.log2_int(u64, elem_size)), |
| 2965 | ); | 3409 | ); |
| 2966 | try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ | 3410 | try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 2967 | .base = .{ .reg = lhs_reg }, | 3411 | .base = .{ .reg = lhs_reg }, |
| 2968 | .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, | 3412 | .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, |
| 2969 | }); | 3413 | }); |
| 2970 | } else try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ | 3414 | } else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 2971 | .base = .{ .reg = lhs_reg }, | 3415 | .base = .{ .reg = lhs_reg }, |
| 2972 | .mod = .{ .rm = .{ | 3416 | .mod = .{ .rm = .{ |
| 2973 | .size = .qword, | 3417 | .size = .qword, |
| ... | @@ -2976,57 +3420,57 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2976,57 +3420,57 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2976 | } }, | 3420 | } }, |
| 2977 | }); | 3421 | }); |
| 2978 | } | 3422 | } |
| 2979 | try ops[1].die(self); | 3423 | try ops[1].die(cg); |
| 2980 | try ops[0].moveTo(inst, self); | 3424 | try ops[0].moveTo(inst, cg); |
| 2981 | }, | 3425 | }, |
| 2982 | .array_to_slice => if (use_old) try self.airArrayToSlice(inst) else { | 3426 | .array_to_slice => if (use_old) try cg.airArrayToSlice(inst) else { |
| 2983 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3427 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2984 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3428 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2985 | var len = try self.tempFromValue(Type.usize, .{ | 3429 | var len = try cg.tempFromValue(Type.usize, .{ |
| 2986 | .immediate = self.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu), | 3430 | .immediate = cg.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu), |
| 2987 | }); | 3431 | }); |
| 2988 | try ops[0].toPair(&len, self); | 3432 | try ops[0].toPair(&len, cg); |
| 2989 | try ops[0].moveTo(inst, self); | 3433 | try ops[0].moveTo(inst, cg); |
| 2990 | }, | 3434 | }, |
| 2991 | .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}), | 3435 | .error_set_has_value => return cg.fail("TODO implement error_set_has_value", .{}), |
| 2992 | .field_parent_ptr => if (use_old) try self.airFieldParentPtr(inst) else { | 3436 | .field_parent_ptr => if (use_old) try cg.airFieldParentPtr(inst) else { |
| 2993 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; | 3437 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2994 | const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; | 3438 | const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; |
| 2995 | var ops = try self.tempsFromOperands(inst, .{extra.field_ptr}); | 3439 | var ops = try cg.tempsFromOperands(inst, .{extra.field_ptr}); |
| 2996 | try ops[0].toOffset(-self.fieldOffset( | 3440 | try ops[0].toOffset(-cg.fieldOffset( |
| 2997 | self.typeOfIndex(inst), | 3441 | cg.typeOfIndex(inst), |
| 2998 | self.typeOf(extra.field_ptr), | 3442 | cg.typeOf(extra.field_ptr), |
| 2999 | extra.field_index, | 3443 | extra.field_index, |
| 3000 | ), self); | 3444 | ), cg); |
| 3001 | try ops[0].moveTo(inst, self); | 3445 | try ops[0].moveTo(inst, cg); |
| 3002 | }, | 3446 | }, |
| 3003 | | 3447 | |
| 3004 | .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}), | 3448 | .is_named_enum_value => return cg.fail("TODO implement is_named_enum_value", .{}), |
| 3005 | | 3449 | |
| 3006 | .wasm_memory_size => unreachable, | 3450 | .wasm_memory_size => unreachable, |
| 3007 | .wasm_memory_grow => unreachable, | 3451 | .wasm_memory_grow => unreachable, |
| 3008 | | 3452 | |
| 3009 | .addrspace_cast => { | 3453 | .addrspace_cast => { |
| 3010 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 3454 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 3011 | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); | 3455 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3012 | try ops[0].moveTo(inst, self); | 3456 | try ops[0].moveTo(inst, cg); |
| 3013 | }, | 3457 | }, |
| 3014 | | 3458 | |
| 3015 | .vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}), | 3459 | .vector_store_elem => return cg.fail("TODO implement vector_store_elem", .{}), |
| 3016 | | 3460 | |
| 3017 | .c_va_arg => try self.airVaArg(inst), | 3461 | .c_va_arg => try cg.airVaArg(inst), |
| 3018 | .c_va_copy => try self.airVaCopy(inst), | 3462 | .c_va_copy => try cg.airVaCopy(inst), |
| 3019 | .c_va_end => try self.airVaEnd(inst), | 3463 | .c_va_end => try cg.airVaEnd(inst), |
| 3020 | .c_va_start => try self.airVaStart(inst), | 3464 | .c_va_start => try cg.airVaStart(inst), |
| 3021 | | 3465 | |
| 3022 | .work_item_id => unreachable, | 3466 | .work_item_id => unreachable, |
| 3023 | .work_group_size => unreachable, | 3467 | .work_group_size => unreachable, |
| 3024 | .work_group_id => unreachable, | 3468 | .work_group_id => unreachable, |
| 3025 | } | 3469 | } |
| 3026 | self.resetTemps(); | 3470 | cg.resetTemps(); |
| 3027 | self.checkInvariantsAfterAirInst(); | 3471 | cg.checkInvariantsAfterAirInst(); |
| 3028 | } | 3472 | } |
| 3029 | verbose_tracking_log.debug("{}", .{self.fmtTracking()}); | 3473 | verbose_tracking_log.debug("{}", .{cg.fmtTracking()}); |
| 3030 | } | 3474 | } |
| 3031 | | 3475 | |
| 3032 | fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void { | 3476 | fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void { |
| ... | @@ -3117,12 +3561,16 @@ fn freeReg(self: *CodeGen, reg: Register) !void { | ... | @@ -3117,12 +3561,16 @@ fn freeReg(self: *CodeGen, reg: Register) !void { |
| 3117 | fn freeValue(self: *CodeGen, value: MCValue) !void { | 3561 | fn freeValue(self: *CodeGen, value: MCValue) !void { |
| 3118 | switch (value) { | 3562 | switch (value) { |
| 3119 | .register => |reg| try self.freeReg(reg), | 3563 | .register => |reg| try self.freeReg(reg), |
| 3120 | .register_pair => |regs| for (regs) |reg| try self.freeReg(reg), | 3564 | inline .register_pair, |
| | 3565 | .register_triple, |
| | 3566 | .register_quadruple, |
| | 3567 | => |regs| for (regs) |reg| try self.freeReg(reg), |
| 3121 | .register_offset, .indirect => |reg_off| try self.freeReg(reg_off.reg), | 3568 | .register_offset, .indirect => |reg_off| try self.freeReg(reg_off.reg), |
| 3122 | .register_overflow => |reg_ov| { | 3569 | .register_overflow => |reg_ov| { |
| 3123 | try self.freeReg(reg_ov.reg); | 3570 | try self.freeReg(reg_ov.reg); |
| 3124 | self.eflags_inst = null; | 3571 | self.eflags_inst = null; |
| 3125 | }, | 3572 | }, |
| | 3573 | .register_mask => |reg_mask| try self.freeReg(reg_mask.reg), |
| 3126 | .eflags => self.eflags_inst = null, | 3574 | .eflags => self.eflags_inst = null, |
| 3127 | else => {}, // TODO process stack allocation death | 3575 | else => {}, // TODO process stack allocation death |
| 3128 | } | 3576 | } |
| ... | @@ -3323,11 +3771,11 @@ fn allocRegOrMemAdvanced(self: *CodeGen, ty: Type, inst: ?Air.Inst.Index, reg_ok | ... | @@ -3323,11 +3771,11 @@ fn allocRegOrMemAdvanced(self: *CodeGen, ty: Type, inst: ?Air.Inst.Index, reg_ok |
| 3323 | }, | 3771 | }, |
| 3324 | .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { | 3772 | .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { |
| 3325 | .float => switch (ty.childType(zcu).floatBits(self.target.*)) { | 3773 | .float => switch (ty.childType(zcu).floatBits(self.target.*)) { |
| 3326 | 16, 32, 64, 128 => if (self.hasFeature(.avx)) 32 else 16, | 3774 | 16, 32, 64, 128 => self.vectorSize(.float), |
| 3327 | 80 => break :need_mem, | 3775 | 80 => break :need_mem, |
| 3328 | else => unreachable, | 3776 | else => unreachable, |
| 3329 | }, | 3777 | }, |
| 3330 | else => if (self.hasFeature(.avx)) 32 else 16, | 3778 | else => self.vectorSize(.int), |
| 3331 | }, | 3779 | }, |
| 3332 | else => 8, | 3780 | else => 8, |
| 3333 | })) { | 3781 | })) { |
| ... | @@ -3374,6 +3822,28 @@ fn regSetForType(self: *CodeGen, ty: Type) RegisterManager.RegisterBitSet { | ... | @@ -3374,6 +3822,28 @@ fn regSetForType(self: *CodeGen, ty: Type) RegisterManager.RegisterBitSet { |
| 3374 | return regSetForRegClass(self.regClassForType(ty)); | 3822 | return regSetForRegClass(self.regClassForType(ty)); |
| 3375 | } | 3823 | } |
| 3376 | | 3824 | |
| | 3825 | fn vectorSize(cg: *CodeGen, kind: enum { int, float }) u6 { |
| | 3826 | return if (cg.hasFeature(switch (kind) { |
| | 3827 | .int => .avx2, |
| | 3828 | .float => .avx, |
| | 3829 | })) 32 else if (cg.hasFeature(.sse)) 16 else 8; |
| | 3830 | } |
| | 3831 | |
| | 3832 | fn limbType(cg: *CodeGen, ty: Type) Type { |
| | 3833 | const pt = cg.pt; |
| | 3834 | const zcu = pt.zcu; |
| | 3835 | const vector_size = cg.vectorSize(if (ty.isRuntimeFloat()) .float else .int); |
| | 3836 | const scalar_ty, const scalar_size = scalar: { |
| | 3837 | const scalar_ty = ty.scalarType(zcu); |
| | 3838 | const scalar_size = scalar_ty.abiSize(zcu); |
| | 3839 | if (scalar_size <= vector_size) break :scalar .{ scalar_ty, scalar_size }; |
| | 3840 | }; |
| | 3841 | pt.vectorType(.{ |
| | 3842 | .len = @divExact(vector_size, scalar_size), |
| | 3843 | .child = scalar_ty.toIntern(), |
| | 3844 | }); |
| | 3845 | } |
| | 3846 | |
| 3377 | const State = struct { | 3847 | const State = struct { |
| 3378 | registers: RegisterManager.TrackedRegisters, | 3848 | registers: RegisterManager.TrackedRegisters, |
| 3379 | reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking, | 3849 | reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking, |
| ... | @@ -3639,7 +4109,7 @@ fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -3639,7 +4109,7 @@ fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3639 | } | 4109 | } |
| 3640 | } else { | 4110 | } else { |
| 3641 | assert(src_bits == 64 and dst_bits == 32); | 4111 | assert(src_bits == 64 and dst_bits == 32); |
| 3642 | if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory( | 4112 | if (self.hasFeature(.avx)) if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 3643 | .{ .v_ss, .cvtsd2 }, | 4113 | .{ .v_ss, .cvtsd2 }, |
| 3644 | dst_reg, | 4114 | dst_reg, |
| 3645 | dst_reg, | 4115 | dst_reg, |
| ... | @@ -3652,7 +4122,7 @@ fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -3652,7 +4122,7 @@ fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3652 | src_mcv.getReg().? | 4122 | src_mcv.getReg().? |
| 3653 | else | 4123 | else |
| 3654 | try self.copyToTmpRegister(src_ty, src_mcv)).to128(), | 4124 | try self.copyToTmpRegister(src_ty, src_mcv)).to128(), |
| 3655 | ) else if (src_mcv.isMemory()) try self.asmRegisterMemory( | 4125 | ) else if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 3656 | .{ ._ss, .cvtsd2 }, | 4126 | .{ ._ss, .cvtsd2 }, |
| 3657 | dst_reg, | 4127 | dst_reg, |
| 3658 | try src_mcv.mem(self, .{ .size = .qword }), | 4128 | try src_mcv.mem(self, .{ .size = .qword }), |
| ... | @@ -3751,7 +4221,7 @@ fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -3751,7 +4221,7 @@ fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3751 | } else { | 4221 | } else { |
| 3752 | assert(src_bits == 32 and dst_bits == 64); | 4222 | assert(src_bits == 32 and dst_bits == 64); |
| 3753 | if (self.hasFeature(.avx)) switch (vec_len) { | 4223 | if (self.hasFeature(.avx)) switch (vec_len) { |
| 3754 | 1 => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory( | 4224 | 1 => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 3755 | .{ .v_sd, .cvtss2 }, | 4225 | .{ .v_sd, .cvtss2 }, |
| 3756 | dst_alias, | 4226 | dst_alias, |
| 3757 | dst_alias, | 4227 | dst_alias, |
| ... | @@ -3765,7 +4235,7 @@ fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -3765,7 +4235,7 @@ fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3765 | else | 4235 | else |
| 3766 | try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), | 4236 | try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), |
| 3767 | ), | 4237 | ), |
| 3768 | 2...4 => if (src_mcv.isMemory()) try self.asmRegisterMemory( | 4238 | 2...4 => if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 3769 | .{ .v_pd, .cvtps2 }, | 4239 | .{ .v_pd, .cvtps2 }, |
| 3770 | dst_alias, | 4240 | dst_alias, |
| 3771 | try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), | 4241 | try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), |
| ... | @@ -3778,7 +4248,7 @@ fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -3778,7 +4248,7 @@ fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3778 | try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), | 4248 | try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), |
| 3779 | ), | 4249 | ), |
| 3780 | else => break :result null, | 4250 | else => break :result null, |
| 3781 | } else if (src_mcv.isMemory()) try self.asmRegisterMemory( | 4251 | } else if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 3782 | switch (vec_len) { | 4252 | switch (vec_len) { |
| 3783 | 1 => .{ ._sd, .cvtss2 }, | 4253 | 1 => .{ ._sd, .cvtss2 }, |
| 3784 | 2 => .{ ._pd, .cvtps2 }, | 4254 | 2 => .{ ._pd, .cvtps2 }, |
| ... | @@ -3827,7 +4297,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -3827,7 +4297,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3827 | if (dst_ty.isVector(zcu)) { | 4297 | if (dst_ty.isVector(zcu)) { |
| 3828 | const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu)); | 4298 | const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu)); |
| 3829 | const max_abi_size = @max(dst_abi_size, src_abi_size); | 4299 | const max_abi_size = @max(dst_abi_size, src_abi_size); |
| 3830 | if (max_abi_size > @as(u32, if (self.hasFeature(.avx2)) 32 else 16)) break :result null; | 4300 | if (max_abi_size > self.vectorSize(.int)) break :result null; |
| 3831 | const has_avx = self.hasFeature(.avx); | 4301 | const has_avx = self.hasFeature(.avx); |
| 3832 | | 4302 | |
| 3833 | const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu); | 4303 | const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu); |
| ... | @@ -3919,7 +4389,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -3919,7 +4389,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3919 | const dst_reg = dst_mcv.getReg().?; | 4389 | const dst_reg = dst_mcv.getReg().?; |
| 3920 | const dst_alias = registerAlias(dst_reg, dst_abi_size); | 4390 | const dst_alias = registerAlias(dst_reg, dst_abi_size); |
| 3921 | | 4391 | |
| 3922 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 4392 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 3923 | mir_tag, | 4393 | mir_tag, |
| 3924 | dst_alias, | 4394 | dst_alias, |
| 3925 | try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), | 4395 | try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), |
| ... | @@ -4017,7 +4487,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -4017,7 +4487,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4017 | const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable; | 4487 | const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable; |
| 4018 | const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable; | 4488 | const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable; |
| 4019 | | 4489 | |
| 4020 | const high_mcv: MCValue = if (dst_mcv.isMemory()) | 4490 | const high_mcv: MCValue = if (dst_mcv.isBase()) |
| 4021 | dst_mcv.address().offset((src_limbs_len - 1) * 8).deref() | 4491 | dst_mcv.address().offset((src_limbs_len - 1) * 8).deref() |
| 4022 | else | 4492 | else |
| 4023 | .{ .register = dst_mcv.register_pair[1] }; | 4493 | .{ .register = dst_mcv.register_pair[1] }; |
| ... | @@ -4392,7 +4862,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -4392,7 +4862,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4392 | else => null, | 4862 | else => null, |
| 4393 | }; | 4863 | }; |
| 4394 | defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); | 4864 | defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 4395 | if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory( | 4865 | if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 4396 | .{ ._, .mov }, | 4866 | .{ ._, .mov }, |
| 4397 | tmp_reg, | 4867 | tmp_reg, |
| 4398 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), | 4868 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | @@ -4416,7 +4886,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -4416,7 +4886,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4416 | else => null, | 4886 | else => null, |
| 4417 | }; | 4887 | }; |
| 4418 | defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); | 4888 | defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 4419 | if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory( | 4889 | if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 4420 | .{ ._, .xor }, | 4890 | .{ ._, .xor }, |
| 4421 | tmp_reg, | 4891 | tmp_reg, |
| 4422 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), | 4892 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | @@ -4518,7 +4988,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -4518,7 +4988,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4518 | | 4988 | |
| 4519 | for (tmp_regs, dst_regs) |tmp_reg, dst_reg| | 4989 | for (tmp_regs, dst_regs) |tmp_reg, dst_reg| |
| 4520 | try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_reg); | 4990 | try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_reg); |
| 4521 | if (mat_rhs_mcv.isMemory()) { | 4991 | if (mat_rhs_mcv.isBase()) { |
| 4522 | try self.asmRegisterMemory( | 4992 | try self.asmRegisterMemory( |
| 4523 | .{ ._, .add }, | 4993 | .{ ._, .add }, |
| 4524 | tmp_regs[0], | 4994 | tmp_regs[0], |
| ... | @@ -4793,7 +5263,7 @@ fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -4793,7 +5263,7 @@ fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4793 | else => null, | 5263 | else => null, |
| 4794 | }; | 5264 | }; |
| 4795 | defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); | 5265 | defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 4796 | if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory( | 5266 | if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 4797 | .{ ._, .mov }, | 5267 | .{ ._, .mov }, |
| 4798 | tmp_reg, | 5268 | tmp_reg, |
| 4799 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), | 5269 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | @@ -4817,7 +5287,7 @@ fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -4817,7 +5287,7 @@ fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4817 | else => null, | 5287 | else => null, |
| 4818 | }; | 5288 | }; |
| 4819 | defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); | 5289 | defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 4820 | if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory( | 5290 | if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 4821 | .{ ._, .xor }, | 5291 | .{ ._, .xor }, |
| 4822 | tmp_reg, | 5292 | tmp_reg, |
| 4823 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), | 5293 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | @@ -5348,7 +5818,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -5348,7 +5818,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 5348 | }; | 5818 | }; |
| 5349 | defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); | 5819 | defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 5350 | | 5820 | |
| 5351 | if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory( | 5821 | if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 5352 | .{ ._, .mov }, | 5822 | .{ ._, .mov }, |
| 5353 | .rax, | 5823 | .rax, |
| 5354 | try mat_lhs_mcv.mem(self, .{ .size = .qword }), | 5824 | try mat_lhs_mcv.mem(self, .{ .size = .qword }), |
| ... | @@ -5357,7 +5827,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -5357,7 +5827,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 5357 | .rax, | 5827 | .rax, |
| 5358 | mat_lhs_mcv.register_pair[0], | 5828 | mat_lhs_mcv.register_pair[0], |
| 5359 | ); | 5829 | ); |
| 5360 | if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory( | 5830 | if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 5361 | .{ ._, .mov }, | 5831 | .{ ._, .mov }, |
| 5362 | tmp_regs[0], | 5832 | tmp_regs[0], |
| 5363 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), | 5833 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | @@ -5370,14 +5840,14 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -5370,14 +5840,14 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 5370 | try self.asmSetccRegister(.nz, tmp_regs[1].to8()); | 5840 | try self.asmSetccRegister(.nz, tmp_regs[1].to8()); |
| 5371 | try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax); | 5841 | try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax); |
| 5372 | try self.asmSetccRegister(.o, tmp_regs[2].to8()); | 5842 | try self.asmSetccRegister(.o, tmp_regs[2].to8()); |
| 5373 | if (mat_rhs_mcv.isMemory()) | 5843 | if (mat_rhs_mcv.isBase()) |
| 5374 | try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword })) | 5844 | try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword })) |
| 5375 | else | 5845 | else |
| 5376 | try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]); | 5846 | try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]); |
| 5377 | try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]); | 5847 | try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]); |
| 5378 | try self.asmSetccRegister(.c, tmp_regs[3].to8()); | 5848 | try self.asmSetccRegister(.c, tmp_regs[3].to8()); |
| 5379 | try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[2].to8(), tmp_regs[3].to8()); | 5849 | try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[2].to8(), tmp_regs[3].to8()); |
| 5380 | if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory( | 5850 | if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 5381 | .{ ._, .mov }, | 5851 | .{ ._, .mov }, |
| 5382 | tmp_regs[0], | 5852 | tmp_regs[0], |
| 5383 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), | 5853 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | @@ -5394,7 +5864,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -5394,7 +5864,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 5394 | tmp_regs[3].to8(), | 5864 | tmp_regs[3].to8(), |
| 5395 | ); | 5865 | ); |
| 5396 | try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8()); | 5866 | try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8()); |
| 5397 | if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory( | 5867 | if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 5398 | .{ .i_, .mul }, | 5868 | .{ .i_, .mul }, |
| 5399 | tmp_regs[0], | 5869 | tmp_regs[0], |
| 5400 | try mat_rhs_mcv.mem(self, .{ .size = .qword }), | 5870 | try mat_rhs_mcv.mem(self, .{ .size = .qword }), |
| ... | @@ -6512,7 +6982,7 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -6512,7 +6982,7 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6512 | | 6982 | |
| 6513 | const index_ty = self.typeOf(bin_op.rhs); | 6983 | const index_ty = self.typeOf(bin_op.rhs); |
| 6514 | const index_mcv = try self.resolveInst(bin_op.rhs); | 6984 | const index_mcv = try self.resolveInst(bin_op.rhs); |
| 6515 | const index_lock: ?RegisterLock = switch (index_mcv) { | 6985 | const index_lock = switch (index_mcv) { |
| 6516 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | 6986 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 6517 | else => null, | 6987 | else => null, |
| 6518 | }; | 6988 | }; |
| ... | @@ -6520,48 +6990,102 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -6520,48 +6990,102 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6520 | | 6990 | |
| 6521 | try self.spillEflagsIfOccupied(); | 6991 | try self.spillEflagsIfOccupied(); |
| 6522 | if (array_ty.isVector(zcu) and elem_ty.bitSize(zcu) == 1) { | 6992 | if (array_ty.isVector(zcu) and elem_ty.bitSize(zcu) == 1) { |
| 6523 | const index_reg = switch (index_mcv) { | 6993 | const array_mat_mcv: MCValue = switch (array_mcv) { |
| 6524 | .register => |reg| reg, | 6994 | else => array_mcv, |
| 6525 | else => try self.copyToTmpRegister(index_ty, index_mcv), | 6995 | .register_mask => .{ .register = try self.copyToTmpRegister(array_ty, array_mcv) }, |
| 6526 | }; | 6996 | }; |
| 6527 | switch (array_mcv) { | 6997 | const array_mat_lock = switch (array_mat_mcv) { |
| | 6998 | .register => |reg| self.register_manager.lockReg(reg), |
| | 6999 | else => null, |
| | 7000 | }; |
| | 7001 | defer if (array_mat_lock) |lock| self.register_manager.unlockReg(lock); |
| | 7002 | |
| | 7003 | switch (array_mat_mcv) { |
| 6528 | .register => |array_reg| switch (array_reg.class()) { | 7004 | .register => |array_reg| switch (array_reg.class()) { |
| 6529 | .general_purpose => try self.asmRegisterRegister( | 7005 | .general_purpose => switch (index_mcv) { |
| 6530 | .{ ._, .bt }, | 7006 | .immediate => |index_imm| try self.asmRegisterImmediate( |
| 6531 | array_reg.to64(), | 7007 | .{ ._, .bt }, |
| 6532 | index_reg.to64(), | 7008 | array_reg.to64(), |
| 6533 | ), | 7009 | .u(index_imm), |
| | 7010 | ), |
| | 7011 | else => try self.asmRegisterRegister( |
| | 7012 | .{ ._, .bt }, |
| | 7013 | array_reg.to64(), |
| | 7014 | switch (index_mcv) { |
| | 7015 | .register => |index_reg| index_reg, |
| | 7016 | else => try self.copyToTmpRegister(index_ty, index_mcv), |
| | 7017 | }.to64(), |
| | 7018 | ), |
| | 7019 | }, |
| 6534 | .sse => { | 7020 | .sse => { |
| 6535 | const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu)); | 7021 | const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu)); |
| 6536 | try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{}); | 7022 | try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mat_mcv, .{}); |
| 6537 | try self.asmMemoryRegister( | 7023 | switch (index_mcv) { |
| 6538 | .{ ._, .bt }, | 7024 | .immediate => |index_imm| try self.asmMemoryImmediate( |
| 6539 | .{ | 7025 | .{ ._, .bt }, |
| 6540 | .base = .{ .frame = frame_index }, | 7026 | .{ |
| 6541 | .mod = .{ .rm = .{ .size = .qword } }, | 7027 | .base = .{ .frame = frame_index }, |
| 6542 | }, | 7028 | .mod = .{ .rm = .{ .size = .qword } }, |
| 6543 | index_reg.to64(), | 7029 | }, |
| 6544 | ); | 7030 | .u(index_imm), |
| | 7031 | ), |
| | 7032 | else => try self.asmMemoryRegister( |
| | 7033 | .{ ._, .bt }, |
| | 7034 | .{ |
| | 7035 | .base = .{ .frame = frame_index }, |
| | 7036 | .mod = .{ .rm = .{ .size = .qword } }, |
| | 7037 | }, |
| | 7038 | switch (index_mcv) { |
| | 7039 | .register => |index_reg| index_reg, |
| | 7040 | else => try self.copyToTmpRegister(index_ty, index_mcv), |
| | 7041 | }.to64(), |
| | 7042 | ), |
| | 7043 | } |
| 6545 | }, | 7044 | }, |
| 6546 | else => unreachable, | 7045 | else => unreachable, |
| 6547 | }, | 7046 | }, |
| 6548 | .load_frame => try self.asmMemoryRegister( | 7047 | .load_frame => switch (index_mcv) { |
| 6549 | .{ ._, .bt }, | 7048 | .immediate => |index_imm| try self.asmMemoryImmediate( |
| 6550 | try array_mcv.mem(self, .{ .size = .qword }), | 7049 | .{ ._, .bt }, |
| 6551 | index_reg.to64(), | 7050 | try array_mat_mcv.mem(self, .{ .size = .qword }), |
| 6552 | ), | 7051 | .u(index_imm), |
| 6553 | .memory, .load_symbol, .load_direct, .load_got, .load_tlv => try self.asmMemoryRegister( | 7052 | ), |
| 6554 | .{ ._, .bt }, | 7053 | else => try self.asmMemoryRegister( |
| 6555 | .{ | 7054 | .{ ._, .bt }, |
| 6556 | .base = .{ | 7055 | try array_mat_mcv.mem(self, .{ .size = .qword }), |
| 6557 | .reg = try self.copyToTmpRegister(Type.usize, array_mcv.address()), | 7056 | switch (index_mcv) { |
| | 7057 | .register => |index_reg| index_reg, |
| | 7058 | else => try self.copyToTmpRegister(index_ty, index_mcv), |
| | 7059 | }.to64(), |
| | 7060 | ), |
| | 7061 | }, |
| | 7062 | .memory, .load_symbol, .load_direct, .load_got, .load_tlv => switch (index_mcv) { |
| | 7063 | .immediate => |index_imm| try self.asmMemoryImmediate( |
| | 7064 | .{ ._, .bt }, |
| | 7065 | .{ |
| | 7066 | .base = .{ |
| | 7067 | .reg = try self.copyToTmpRegister(Type.usize, array_mat_mcv.address()), |
| | 7068 | }, |
| | 7069 | .mod = .{ .rm = .{ .size = .qword } }, |
| 6558 | }, | 7070 | }, |
| 6559 | .mod = .{ .rm = .{ .size = .qword } }, | 7071 | .u(index_imm), |
| 6560 | }, | 7072 | ), |
| 6561 | index_reg.to64(), | 7073 | else => try self.asmMemoryRegister( |
| 6562 | ), | 7074 | .{ ._, .bt }, |
| | 7075 | .{ |
| | 7076 | .base = .{ |
| | 7077 | .reg = try self.copyToTmpRegister(Type.usize, array_mat_mcv.address()), |
| | 7078 | }, |
| | 7079 | .mod = .{ .rm = .{ .size = .qword } }, |
| | 7080 | }, |
| | 7081 | switch (index_mcv) { |
| | 7082 | .register => |index_reg| index_reg, |
| | 7083 | else => try self.copyToTmpRegister(index_ty, index_mcv), |
| | 7084 | }.to64(), |
| | 7085 | ), |
| | 7086 | }, |
| 6563 | else => return self.fail("TODO airArrayElemVal for {s} of {}", .{ | 7087 | else => return self.fail("TODO airArrayElemVal for {s} of {}", .{ |
| 6564 | @tagName(array_mcv), array_ty.fmt(pt), | 7088 | @tagName(array_mat_mcv), array_ty.fmt(pt), |
| 6565 | }), | 7089 | }), |
| 6566 | } | 7090 | } |
| 6567 | | 7091 | |
| ... | @@ -6856,6 +7380,15 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -6856,6 +7380,15 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6856 | const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); | 7380 | const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); |
| 6857 | const has_lzcnt = self.hasFeature(.lzcnt); | 7381 | const has_lzcnt = self.hasFeature(.lzcnt); |
| 6858 | if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) { | 7382 | if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) { |
| | 7383 | const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) { |
| | 7384 | .load_frame => |src_frame_addr| src_frame_addr, |
| | 7385 | else => { |
| | 7386 | const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu)); |
| | 7387 | try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{}); |
| | 7388 | break :src_frame_addr .{ .index = src_frame_addr }; |
| | 7389 | }, |
| | 7390 | }; |
| | 7391 | |
| 6859 | const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; | 7392 | const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; |
| 6860 | const extra_bits = abi_size * 8 - src_bits; | 7393 | const extra_bits = abi_size * 8 - src_bits; |
| 6861 | | 7394 | |
| ... | @@ -6881,22 +7414,22 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -6881,22 +7414,22 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6881 | try self.asmRegister(.{ ._, .dec }, index_reg.to32()); | 7414 | try self.asmRegister(.{ ._, .dec }, index_reg.to32()); |
| 6882 | } | 7415 | } |
| 6883 | try self.asmMemoryImmediate(.{ ._, .cmp }, .{ | 7416 | try self.asmMemoryImmediate(.{ ._, .cmp }, .{ |
| 6884 | .base = .{ .frame = src_mcv.load_frame.index }, | 7417 | .base = .{ .frame = src_frame_addr.index }, |
| 6885 | .mod = .{ .rm = .{ | 7418 | .mod = .{ .rm = .{ |
| 6886 | .size = .qword, | 7419 | .size = .qword, |
| 6887 | .index = index_reg.to64(), | 7420 | .index = index_reg.to64(), |
| 6888 | .scale = .@"8", | 7421 | .scale = .@"8", |
| 6889 | .disp = src_mcv.load_frame.off, | 7422 | .disp = src_frame_addr.off, |
| 6890 | } }, | 7423 | } }, |
| 6891 | }, .u(0)); | 7424 | }, .u(0)); |
| 6892 | _ = try self.asmJccReloc(.e, loop); | 7425 | _ = try self.asmJccReloc(.e, loop); |
| 6893 | try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{ | 7426 | try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{ |
| 6894 | .base = .{ .frame = src_mcv.load_frame.index }, | 7427 | .base = .{ .frame = src_frame_addr.index }, |
| 6895 | .mod = .{ .rm = .{ | 7428 | .mod = .{ .rm = .{ |
| 6896 | .size = .qword, | 7429 | .size = .qword, |
| 6897 | .index = index_reg.to64(), | 7430 | .index = index_reg.to64(), |
| 6898 | .scale = .@"8", | 7431 | .scale = .@"8", |
| 6899 | .disp = src_mcv.load_frame.off, | 7432 | .disp = src_frame_addr.off, |
| 6900 | } }, | 7433 | } }, |
| 6901 | }); | 7434 | }); |
| 6902 | self.performReloc(zero); | 7435 | self.performReloc(zero); |
| ... | @@ -6935,7 +7468,7 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -6935,7 +7468,7 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6935 | .{ ._, .lzcnt }, | 7468 | .{ ._, .lzcnt }, |
| 6936 | Type.u64, | 7469 | Type.u64, |
| 6937 | dst_mcv, | 7470 | dst_mcv, |
| 6938 | if (mat_src_mcv.isMemory()) | 7471 | if (mat_src_mcv.isBase()) |
| 6939 | mat_src_mcv | 7472 | mat_src_mcv |
| 6940 | else | 7473 | else |
| 6941 | .{ .register = mat_src_mcv.register_pair[0] }, | 7474 | .{ .register = mat_src_mcv.register_pair[0] }, |
| ... | @@ -6945,7 +7478,7 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -6945,7 +7478,7 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6945 | .{ ._, .lzcnt }, | 7478 | .{ ._, .lzcnt }, |
| 6946 | Type.u64, | 7479 | Type.u64, |
| 6947 | tmp_mcv, | 7480 | tmp_mcv, |
| 6948 | if (mat_src_mcv.isMemory()) | 7481 | if (mat_src_mcv.isBase()) |
| 6949 | mat_src_mcv.address().offset(8).deref() | 7482 | mat_src_mcv.address().offset(8).deref() |
| 6950 | else | 7483 | else |
| 6951 | .{ .register = mat_src_mcv.register_pair[1] }, | 7484 | .{ .register = mat_src_mcv.register_pair[1] }, |
| ... | @@ -7053,6 +7586,15 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -7053,6 +7586,15 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 7053 | const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); | 7586 | const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); |
| 7054 | const has_bmi = self.hasFeature(.bmi); | 7587 | const has_bmi = self.hasFeature(.bmi); |
| 7055 | if (src_bits > @as(u32, if (has_bmi) 128 else 64)) { | 7588 | if (src_bits > @as(u32, if (has_bmi) 128 else 64)) { |
| | 7589 | const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) { |
| | 7590 | .load_frame => |src_frame_addr| src_frame_addr, |
| | 7591 | else => { |
| | 7592 | const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu)); |
| | 7593 | try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{}); |
| | 7594 | break :src_frame_addr .{ .index = src_frame_addr }; |
| | 7595 | }, |
| | 7596 | }; |
| | 7597 | |
| 7056 | const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; | 7598 | const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; |
| 7057 | const extra_bits = abi_size * 8 - src_bits; | 7599 | const extra_bits = abi_size * 8 - src_bits; |
| 7058 | | 7600 | |
| ... | @@ -7079,22 +7621,22 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -7079,22 +7621,22 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 7079 | try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(limbs_len)); | 7621 | try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(limbs_len)); |
| 7080 | const zero = try self.asmJccReloc(.nb, undefined); | 7622 | const zero = try self.asmJccReloc(.nb, undefined); |
| 7081 | try self.asmMemoryImmediate(.{ ._, .cmp }, .{ | 7623 | try self.asmMemoryImmediate(.{ ._, .cmp }, .{ |
| 7082 | .base = .{ .frame = src_mcv.load_frame.index }, | 7624 | .base = .{ .frame = src_frame_addr.index }, |
| 7083 | .mod = .{ .rm = .{ | 7625 | .mod = .{ .rm = .{ |
| 7084 | .size = .qword, | 7626 | .size = .qword, |
| 7085 | .index = index_reg.to64(), | 7627 | .index = index_reg.to64(), |
| 7086 | .scale = .@"8", | 7628 | .scale = .@"8", |
| 7087 | .disp = src_mcv.load_frame.off, | 7629 | .disp = src_frame_addr.off, |
| 7088 | } }, | 7630 | } }, |
| 7089 | }, .u(0)); | 7631 | }, .u(0)); |
| 7090 | _ = try self.asmJccReloc(.e, loop); | 7632 | _ = try self.asmJccReloc(.e, loop); |
| 7091 | try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{ | 7633 | try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{ |
| 7092 | .base = .{ .frame = src_mcv.load_frame.index }, | 7634 | .base = .{ .frame = src_frame_addr.index }, |
| 7093 | .mod = .{ .rm = .{ | 7635 | .mod = .{ .rm = .{ |
| 7094 | .size = .qword, | 7636 | .size = .qword, |
| 7095 | .index = index_reg.to64(), | 7637 | .index = index_reg.to64(), |
| 7096 | .scale = .@"8", | 7638 | .scale = .@"8", |
| 7097 | .disp = src_mcv.load_frame.off, | 7639 | .disp = src_frame_addr.off, |
| 7098 | } }, | 7640 | } }, |
| 7099 | }); | 7641 | }); |
| 7100 | self.performReloc(zero); | 7642 | self.performReloc(zero); |
| ... | @@ -7131,11 +7673,11 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -7131,11 +7673,11 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 7131 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | 7673 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 7132 | defer self.register_manager.unlockReg(tmp_lock); | 7674 | defer self.register_manager.unlockReg(tmp_lock); |
| 7133 | | 7675 | |
| 7134 | const lo_mat_src_mcv: MCValue = if (mat_src_mcv.isMemory()) | 7676 | const lo_mat_src_mcv: MCValue = if (mat_src_mcv.isBase()) |
| 7135 | mat_src_mcv | 7677 | mat_src_mcv |
| 7136 | else | 7678 | else |
| 7137 | .{ .register = mat_src_mcv.register_pair[0] }; | 7679 | .{ .register = mat_src_mcv.register_pair[0] }; |
| 7138 | const hi_mat_src_mcv: MCValue = if (mat_src_mcv.isMemory()) | 7680 | const hi_mat_src_mcv: MCValue = if (mat_src_mcv.isBase()) |
| 7139 | mat_src_mcv.address().offset(8).deref() | 7681 | mat_src_mcv.address().offset(8).deref() |
| 7140 | else | 7682 | else |
| 7141 | .{ .register = mat_src_mcv.register_pair[1] }; | 7683 | .{ .register = mat_src_mcv.register_pair[1] }; |
| ... | @@ -7224,13 +7766,13 @@ fn airPopCount(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -7224,13 +7766,13 @@ fn airPopCount(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 7224 | const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs); | 7766 | const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs); |
| 7225 | defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); | 7767 | defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); |
| 7226 | | 7768 | |
| 7227 | try self.genPopCount(tmp_regs[0], Type.usize, if (mat_src_mcv.isMemory()) | 7769 | try self.genPopCount(tmp_regs[0], Type.usize, if (mat_src_mcv.isBase()) |
| 7228 | mat_src_mcv | 7770 | mat_src_mcv |
| 7229 | else | 7771 | else |
| 7230 | .{ .register = mat_src_mcv.register_pair[0] }, false); | 7772 | .{ .register = mat_src_mcv.register_pair[0] }, false); |
| 7231 | const src_info = src_ty.intInfo(zcu); | 7773 | const src_info = src_ty.intInfo(zcu); |
| 7232 | const hi_ty = try pt.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1); | 7774 | const hi_ty = try pt.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1); |
| 7233 | try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isMemory()) | 7775 | try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isBase()) |
| 7234 | mat_src_mcv.address().offset(8).deref() | 7776 | mat_src_mcv.address().offset(8).deref() |
| 7235 | else | 7777 | else |
| 7236 | .{ .register = mat_src_mcv.register_pair[1] }, false); | 7778 | .{ .register = mat_src_mcv.register_pair[1] }, false); |
| ... | @@ -7388,7 +7930,7 @@ fn genByteSwap( | ... | @@ -7388,7 +7930,7 @@ fn genByteSwap( |
| 7388 | defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); | 7930 | defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); |
| 7389 | | 7931 | |
| 7390 | for (dst_regs, 0..) |dst_reg, limb_index| { | 7932 | for (dst_regs, 0..) |dst_reg, limb_index| { |
| 7391 | if (src_mcv.isMemory()) { | 7933 | if (src_mcv.isBase()) { |
| 7392 | try self.asmRegisterMemory( | 7934 | try self.asmRegisterMemory( |
| 7393 | .{ ._, if (has_movbe) .movbe else .mov }, | 7935 | .{ ._, if (has_movbe) .movbe else .mov }, |
| 7394 | dst_reg.to64(), | 7936 | dst_reg.to64(), |
| ... | @@ -7706,7 +8248,7 @@ fn floatSign(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Ty | ... | @@ -7706,7 +8248,7 @@ fn floatSign(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Ty |
| 7706 | .abs => try vec_ty.maxInt(pt, vec_ty), | 8248 | .abs => try vec_ty.maxInt(pt, vec_ty), |
| 7707 | else => unreachable, | 8249 | else => unreachable, |
| 7708 | }); | 8250 | }); |
| 7709 | const sign_mem: Memory = if (sign_mcv.isMemory()) | 8251 | const sign_mem: Memory = if (sign_mcv.isBase()) |
| 7710 | try sign_mcv.mem(self, .{ .size = .fromSize(abi_size) }) | 8252 | try sign_mcv.mem(self, .{ .size = .fromSize(abi_size) }) |
| 7711 | else | 8253 | else |
| 7712 | .{ | 8254 | .{ |
| ... | @@ -7888,7 +8430,7 @@ fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: | ... | @@ -7888,7 +8430,7 @@ fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: |
| 7888 | const abi_size: u32 = @intCast(ty.abiSize(pt.zcu)); | 8430 | const abi_size: u32 = @intCast(ty.abiSize(pt.zcu)); |
| 7889 | const dst_alias = registerAlias(dst_reg, abi_size); | 8431 | const dst_alias = registerAlias(dst_reg, abi_size); |
| 7890 | switch (mir_tag[0]) { | 8432 | switch (mir_tag[0]) { |
| 7891 | .v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( | 8433 | .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 7892 | mir_tag, | 8434 | mir_tag, |
| 7893 | dst_alias, | 8435 | dst_alias, |
| 7894 | dst_alias, | 8436 | dst_alias, |
| ... | @@ -7904,7 +8446,7 @@ fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: | ... | @@ -7904,7 +8446,7 @@ fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: |
| 7904 | try self.copyToTmpRegister(ty, src_mcv), abi_size), | 8446 | try self.copyToTmpRegister(ty, src_mcv), abi_size), |
| 7905 | .u(@as(u5, @bitCast(mode))), | 8447 | .u(@as(u5, @bitCast(mode))), |
| 7906 | ), | 8448 | ), |
| 7907 | else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( | 8449 | else => if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 7908 | mir_tag, | 8450 | mir_tag, |
| 7909 | dst_alias, | 8451 | dst_alias, |
| 7910 | try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), | 8452 | try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), |
| ... | @@ -8105,7 +8647,7 @@ fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -8105,7 +8647,7 @@ fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 8105 | else | 8647 | else |
| 8106 | try self.register_manager.allocReg(inst, self.regSetForType(ty)); | 8648 | try self.register_manager.allocReg(inst, self.regSetForType(ty)); |
| 8107 | const dst_alias = registerAlias(dst_reg, abi_size); | 8649 | const dst_alias = registerAlias(dst_reg, abi_size); |
| 8108 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 8650 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 8109 | mir_tag, | 8651 | mir_tag, |
| 8110 | dst_alias, | 8652 | dst_alias, |
| 8111 | try src_mcv.mem(self, .{ .size = self.memSize(ty) }), | 8653 | try src_mcv.mem(self, .{ .size = self.memSize(ty) }), |
| ... | @@ -8212,7 +8754,7 @@ fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -8212,7 +8754,7 @@ fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 8212 | }, | 8754 | }, |
| 8213 | 2...8 => { | 8755 | 2...8 => { |
| 8214 | const wide_reg = registerAlias(dst_reg, abi_size * 2); | 8756 | const wide_reg = registerAlias(dst_reg, abi_size * 2); |
| 8215 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 8757 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 8216 | .{ .v_ps, .cvtph2 }, | 8758 | .{ .v_ps, .cvtph2 }, |
| 8217 | wide_reg, | 8759 | wide_reg, |
| 8218 | try src_mcv.mem(self, .{ .size = .fromSize( | 8760 | try src_mcv.mem(self, .{ .size = .fromSize( |
| ... | @@ -8257,7 +8799,7 @@ fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -8257,7 +8799,7 @@ fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 8257 | else => unreachable, | 8799 | else => unreachable, |
| 8258 | }) orelse return self.fail("TODO implement airSqrt for {}", .{ty.fmt(pt)}); | 8800 | }) orelse return self.fail("TODO implement airSqrt for {}", .{ty.fmt(pt)}); |
| 8259 | switch (mir_tag[0]) { | 8801 | switch (mir_tag[0]) { |
| 8260 | .v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory( | 8802 | .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 8261 | mir_tag, | 8803 | mir_tag, |
| 8262 | dst_reg, | 8804 | dst_reg, |
| 8263 | dst_reg, | 8805 | dst_reg, |
| ... | @@ -8271,7 +8813,7 @@ fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -8271,7 +8813,7 @@ fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 8271 | else | 8813 | else |
| 8272 | try self.copyToTmpRegister(ty, src_mcv), abi_size), | 8814 | try self.copyToTmpRegister(ty, src_mcv), abi_size), |
| 8273 | ), | 8815 | ), |
| 8274 | else => if (src_mcv.isMemory()) try self.asmRegisterMemory( | 8816 | else => if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 8275 | mir_tag, | 8817 | mir_tag, |
| 8276 | dst_reg, | 8818 | dst_reg, |
| 8277 | try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), | 8819 | try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), |
| ... | @@ -8339,7 +8881,7 @@ fn reuseOperandAdvanced( | ... | @@ -8339,7 +8881,7 @@ fn reuseOperandAdvanced( |
| 8339 | return false; | 8881 | return false; |
| 8340 | | 8882 | |
| 8341 | switch (mcv) { | 8883 | switch (mcv) { |
| 8342 | .register, .register_pair, .register_overflow => for (mcv.getRegs()) |reg| { | 8884 | .register, .register_pair, .register_overflow, .register_mask => for (mcv.getRegs()) |reg| { |
| 8343 | // If it's in the registers table, need to associate the register(s) with the | 8885 | // If it's in the registers table, need to associate the register(s) with the |
| 8344 | // new instruction. | 8886 | // new instruction. |
| 8345 | if (maybe_tracked_inst) |tracked_inst| { | 8887 | if (maybe_tracked_inst) |tracked_inst| { |
| ... | @@ -8486,7 +9028,10 @@ fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerE | ... | @@ -8486,7 +9028,10 @@ fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerE |
| 8486 | .undef, | 9028 | .undef, |
| 8487 | .eflags, | 9029 | .eflags, |
| 8488 | .register_pair, | 9030 | .register_pair, |
| | 9031 | .register_triple, |
| | 9032 | .register_quadruple, |
| 8489 | .register_overflow, | 9033 | .register_overflow, |
| | 9034 | .register_mask, |
| 8490 | .elementwise_regs_then_frame, | 9035 | .elementwise_regs_then_frame, |
| 8491 | .reserved_frame, | 9036 | .reserved_frame, |
| 8492 | => unreachable, // not a valid pointer | 9037 | => unreachable, // not a valid pointer |
| ... | @@ -8694,7 +9239,10 @@ fn store( | ... | @@ -8694,7 +9239,10 @@ fn store( |
| 8694 | .undef, | 9239 | .undef, |
| 8695 | .eflags, | 9240 | .eflags, |
| 8696 | .register_pair, | 9241 | .register_pair, |
| | 9242 | .register_triple, |
| | 9243 | .register_quadruple, |
| 8697 | .register_overflow, | 9244 | .register_overflow, |
| | 9245 | .register_mask, |
| 8698 | .elementwise_regs_then_frame, | 9246 | .elementwise_regs_then_frame, |
| 8699 | .reserved_frame, | 9247 | .reserved_frame, |
| 8700 | => unreachable, // not a valid pointer | 9248 | => unreachable, // not a valid pointer |
| ... | @@ -8986,7 +9534,7 @@ fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -8986,7 +9534,7 @@ fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 8986 | try self.genCopy(field_ty, dst_mcv, off_mcv, .{}); | 9534 | try self.genCopy(field_ty, dst_mcv, off_mcv, .{}); |
| 8987 | break :dst dst_mcv; | 9535 | break :dst dst_mcv; |
| 8988 | }; | 9536 | }; |
| 8989 | if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) { | 9537 | if (field_abi_size * 8 > field_bit_size and dst_mcv.isBase()) { |
| 8990 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | 9538 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 8991 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | 9539 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 8992 | defer self.register_manager.unlockReg(tmp_lock); | 9540 | defer self.register_manager.unlockReg(tmp_lock); |
| ... | @@ -9194,6 +9742,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: | ... | @@ -9194,6 +9742,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: |
| 9194 | .register_offset, | 9742 | .register_offset, |
| 9195 | .eflags, | 9743 | .eflags, |
| 9196 | .register_overflow, | 9744 | .register_overflow, |
| | 9745 | .register_mask, |
| 9197 | .lea_symbol, | 9746 | .lea_symbol, |
| 9198 | .lea_direct, | 9747 | .lea_direct, |
| 9199 | .lea_got, | 9748 | .lea_got, |
| ... | @@ -9204,7 +9753,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: | ... | @@ -9204,7 +9753,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: |
| 9204 | .air_ref, | 9753 | .air_ref, |
| 9205 | => unreachable, // unmodifiable destination | 9754 | => unreachable, // unmodifiable destination |
| 9206 | .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)), | 9755 | .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)), |
| 9207 | .register_pair => unreachable, // unimplemented | 9756 | .register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented |
| 9208 | .memory, .load_symbol, .load_got, .load_direct, .load_tlv => { | 9757 | .memory, .load_symbol, .load_got, .load_direct, .load_tlv => { |
| 9209 | const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | 9758 | const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 9210 | const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); | 9759 | const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); |
| ... | @@ -9974,27 +10523,27 @@ fn genMulDivBinOp( | ... | @@ -9974,27 +10523,27 @@ fn genMulDivBinOp( |
| 9974 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | 10523 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 9975 | defer self.register_manager.unlockReg(tmp_lock); | 10524 | defer self.register_manager.unlockReg(tmp_lock); |
| 9976 | | 10525 | |
| 9977 | if (mat_lhs_mcv.isMemory()) | 10526 | if (mat_lhs_mcv.isBase()) |
| 9978 | try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .{ .size = .qword })) | 10527 | try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .{ .size = .qword })) |
| 9979 | else | 10528 | else |
| 9980 | try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]); | 10529 | try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]); |
| 9981 | if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory( | 10530 | if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 9982 | .{ ._, .mov }, | 10531 | .{ ._, .mov }, |
| 9983 | tmp_reg, | 10532 | tmp_reg, |
| 9984 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), | 10533 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| 9985 | ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]); | 10534 | ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]); |
| 9986 | try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax); | 10535 | try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax); |
| 9987 | if (mat_rhs_mcv.isMemory()) | 10536 | if (mat_rhs_mcv.isBase()) |
| 9988 | try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword })) | 10537 | try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword })) |
| 9989 | else | 10538 | else |
| 9990 | try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]); | 10539 | try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]); |
| 9991 | try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg); | 10540 | try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg); |
| 9992 | if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory( | 10541 | if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 9993 | .{ ._, .mov }, | 10542 | .{ ._, .mov }, |
| 9994 | tmp_reg, | 10543 | tmp_reg, |
| 9995 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), | 10544 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| 9996 | ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]); | 10545 | ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]); |
| 9997 | if (mat_rhs_mcv.isMemory()) | 10546 | if (mat_rhs_mcv.isBase()) |
| 9998 | try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .{ .size = .qword })) | 10547 | try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .{ .size = .qword })) |
| 9999 | else | 10548 | else |
| 10000 | try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]); | 10549 | try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]); |
| ... | @@ -10414,7 +10963,7 @@ fn genBinOp( | ... | @@ -10414,7 +10963,7 @@ fn genBinOp( |
| 10414 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | 10963 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 10415 | defer self.register_manager.unlockReg(tmp_lock); | 10964 | defer self.register_manager.unlockReg(tmp_lock); |
| 10416 | | 10965 | |
| 10417 | if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( | 10966 | if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 10418 | .{ .vp_w, .insr }, | 10967 | .{ .vp_w, .insr }, |
| 10419 | dst_reg, | 10968 | dst_reg, |
| 10420 | dst_reg, | 10969 | dst_reg, |
| ... | @@ -10439,7 +10988,7 @@ fn genBinOp( | ... | @@ -10439,7 +10988,7 @@ fn genBinOp( |
| 10439 | 64 => .{ .v_sd, .add }, | 10988 | 64 => .{ .v_sd, .add }, |
| 10440 | else => unreachable, | 10989 | else => unreachable, |
| 10441 | }; | 10990 | }; |
| 10442 | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory( | 10991 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 10443 | mir_tag, | 10992 | mir_tag, |
| 10444 | dst_reg, | 10993 | dst_reg, |
| 10445 | dst_reg, | 10994 | dst_reg, |
| ... | @@ -10459,7 +11008,7 @@ fn genBinOp( | ... | @@ -10459,7 +11008,7 @@ fn genBinOp( |
| 10459 | 64 => .{ ._sd, .add }, | 11008 | 64 => .{ ._sd, .add }, |
| 10460 | else => unreachable, | 11009 | else => unreachable, |
| 10461 | }; | 11010 | }; |
| 10462 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 11011 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 10463 | mir_tag, | 11012 | mir_tag, |
| 10464 | dst_reg, | 11013 | dst_reg, |
| 10465 | try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }), | 11014 | try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }), |
| ... | @@ -10514,7 +11063,7 @@ fn genBinOp( | ... | @@ -10514,7 +11063,7 @@ fn genBinOp( |
| 10514 | }; | 11063 | }; |
| 10515 | if (sse_op and ((lhs_ty.scalarType(zcu).isRuntimeFloat() and | 11064 | if (sse_op and ((lhs_ty.scalarType(zcu).isRuntimeFloat() and |
| 10516 | lhs_ty.scalarType(zcu).floatBits(self.target.*) == 80) or | 11065 | lhs_ty.scalarType(zcu).floatBits(self.target.*) == 80) or |
| 10517 | lhs_ty.abiSize(zcu) > @as(u6, if (self.hasFeature(.avx)) 32 else 16))) | 11066 | lhs_ty.abiSize(zcu) > self.vectorSize(.float))) |
| 10518 | return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(air_tag), lhs_ty.fmt(pt) }); | 11067 | return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(air_tag), lhs_ty.fmt(pt) }); |
| 10519 | | 11068 | |
| 10520 | const maybe_mask_reg = switch (air_tag) { | 11069 | const maybe_mask_reg = switch (air_tag) { |
| ... | @@ -10731,7 +11280,7 @@ fn genBinOp( | ... | @@ -10731,7 +11280,7 @@ fn genBinOp( |
| 10731 | }; | 11280 | }; |
| 10732 | | 11281 | |
| 10733 | try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]); | 11282 | try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]); |
| 10734 | if (src_mcv.isMemory()) { | 11283 | if (src_mcv.isBase()) { |
| 10735 | try self.asmRegisterMemory( | 11284 | try self.asmRegisterMemory( |
| 10736 | .{ ._, .cmp }, | 11285 | .{ ._, .cmp }, |
| 10737 | dst_regs[0], | 11286 | dst_regs[0], |
| ... | @@ -10828,8 +11377,11 @@ fn genBinOp( | ... | @@ -10828,8 +11377,11 @@ fn genBinOp( |
| 10828 | .immediate, | 11377 | .immediate, |
| 10829 | .eflags, | 11378 | .eflags, |
| 10830 | .register_pair, | 11379 | .register_pair, |
| | 11380 | .register_triple, |
| | 11381 | .register_quadruple, |
| 10831 | .register_offset, | 11382 | .register_offset, |
| 10832 | .register_overflow, | 11383 | .register_overflow, |
| | 11384 | .register_mask, |
| 10833 | .load_symbol, | 11385 | .load_symbol, |
| 10834 | .lea_symbol, | 11386 | .lea_symbol, |
| 10835 | .load_direct, | 11387 | .load_direct, |
| ... | @@ -10909,7 +11461,7 @@ fn genBinOp( | ... | @@ -10909,7 +11461,7 @@ fn genBinOp( |
| 10909 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | 11461 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 10910 | defer self.register_manager.unlockReg(tmp_lock); | 11462 | defer self.register_manager.unlockReg(tmp_lock); |
| 10911 | | 11463 | |
| 10912 | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( | 11464 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 10913 | .{ .vp_w, .insr }, | 11465 | .{ .vp_w, .insr }, |
| 10914 | dst_reg, | 11466 | dst_reg, |
| 10915 | dst_reg, | 11467 | dst_reg, |
| ... | @@ -11355,7 +11907,7 @@ fn genBinOp( | ... | @@ -11355,7 +11907,7 @@ fn genBinOp( |
| 11355 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | 11907 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 11356 | defer self.register_manager.unlockReg(tmp_lock); | 11908 | defer self.register_manager.unlockReg(tmp_lock); |
| 11357 | | 11909 | |
| 11358 | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( | 11910 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 11359 | .{ .vp_w, .insr }, | 11911 | .{ .vp_w, .insr }, |
| 11360 | dst_reg, | 11912 | dst_reg, |
| 11361 | dst_reg, | 11913 | dst_reg, |
| ... | @@ -11402,7 +11954,7 @@ fn genBinOp( | ... | @@ -11402,7 +11954,7 @@ fn genBinOp( |
| 11402 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | 11954 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 11403 | defer self.register_manager.unlockReg(tmp_lock); | 11955 | defer self.register_manager.unlockReg(tmp_lock); |
| 11404 | | 11956 | |
| 11405 | if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( | 11957 | if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 11406 | .{ .vp_d, .insr }, | 11958 | .{ .vp_d, .insr }, |
| 11407 | dst_reg, | 11959 | dst_reg, |
| 11408 | try src_mcv.mem(self, .{ .size = .dword }), | 11960 | try src_mcv.mem(self, .{ .size = .dword }), |
| ... | @@ -11454,7 +12006,7 @@ fn genBinOp( | ... | @@ -11454,7 +12006,7 @@ fn genBinOp( |
| 11454 | defer self.register_manager.unlockReg(tmp_lock); | 12006 | defer self.register_manager.unlockReg(tmp_lock); |
| 11455 | | 12007 | |
| 11456 | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); | 12008 | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); |
| 11457 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 12009 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 11458 | .{ .v_ps, .cvtph2 }, | 12010 | .{ .v_ps, .cvtph2 }, |
| 11459 | tmp_reg, | 12011 | tmp_reg, |
| 11460 | try src_mcv.mem(self, .{ .size = .qword }), | 12012 | try src_mcv.mem(self, .{ .size = .qword }), |
| ... | @@ -11497,7 +12049,7 @@ fn genBinOp( | ... | @@ -11497,7 +12049,7 @@ fn genBinOp( |
| 11497 | defer self.register_manager.unlockReg(tmp_lock); | 12049 | defer self.register_manager.unlockReg(tmp_lock); |
| 11498 | | 12050 | |
| 11499 | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), dst_reg); | 12051 | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), dst_reg); |
| 11500 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 12052 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 11501 | .{ .v_ps, .cvtph2 }, | 12053 | .{ .v_ps, .cvtph2 }, |
| 11502 | tmp_reg, | 12054 | tmp_reg, |
| 11503 | try src_mcv.mem(self, .{ .size = .xword }), | 12055 | try src_mcv.mem(self, .{ .size = .xword }), |
| ... | @@ -11659,7 +12211,7 @@ fn genBinOp( | ... | @@ -11659,7 +12211,7 @@ fn genBinOp( |
| 11659 | else => if (self.hasFeature(.avx)) { | 12211 | else => if (self.hasFeature(.avx)) { |
| 11660 | const lhs_reg = | 12212 | const lhs_reg = |
| 11661 | if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); | 12213 | if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); |
| 11662 | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory( | 12214 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 11663 | mir_tag, | 12215 | mir_tag, |
| 11664 | dst_reg, | 12216 | dst_reg, |
| 11665 | lhs_reg, | 12217 | lhs_reg, |
| ... | @@ -11678,7 +12230,7 @@ fn genBinOp( | ... | @@ -11678,7 +12230,7 @@ fn genBinOp( |
| 11678 | ); | 12230 | ); |
| 11679 | } else { | 12231 | } else { |
| 11680 | assert(copied_to_dst); | 12232 | assert(copied_to_dst); |
| 11681 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 12233 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 11682 | mir_tag, | 12234 | mir_tag, |
| 11683 | dst_reg, | 12235 | dst_reg, |
| 11684 | try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { | 12236 | try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { |
| ... | @@ -11705,7 +12257,7 @@ fn genBinOp( | ... | @@ -11705,7 +12257,7 @@ fn genBinOp( |
| 11705 | if (self.hasFeature(.avx)) { | 12257 | if (self.hasFeature(.avx)) { |
| 11706 | const lhs_reg = | 12258 | const lhs_reg = |
| 11707 | if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); | 12259 | if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); |
| 11708 | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( | 12260 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 11709 | mir_tag, | 12261 | mir_tag, |
| 11710 | dst_reg, | 12262 | dst_reg, |
| 11711 | lhs_reg, | 12263 | lhs_reg, |
| ... | @@ -11726,7 +12278,7 @@ fn genBinOp( | ... | @@ -11726,7 +12278,7 @@ fn genBinOp( |
| 11726 | ); | 12278 | ); |
| 11727 | } else { | 12279 | } else { |
| 11728 | assert(copied_to_dst); | 12280 | assert(copied_to_dst); |
| 11729 | if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( | 12281 | if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 11730 | mir_tag, | 12282 | mir_tag, |
| 11731 | dst_reg, | 12283 | dst_reg, |
| 11732 | try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { | 12284 | try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { |
| ... | @@ -11937,7 +12489,7 @@ fn genBinOp( | ... | @@ -11937,7 +12489,7 @@ fn genBinOp( |
| 11937 | => { | 12489 | => { |
| 11938 | const unsigned_ty = try lhs_ty.toUnsigned(pt); | 12490 | const unsigned_ty = try lhs_ty.toUnsigned(pt); |
| 11939 | const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty)); | 12491 | const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty)); |
| 11940 | const not_mem: Memory = if (not_mcv.isMemory()) | 12492 | const not_mem: Memory = if (not_mcv.isBase()) |
| 11941 | try not_mcv.mem(self, .{ .size = .fromSize(abi_size) }) | 12493 | try not_mcv.mem(self, .{ .size = .fromSize(abi_size) }) |
| 11942 | else | 12494 | else |
| 11943 | .{ .base = .{ | 12495 | .{ .base = .{ |
| ... | @@ -12017,6 +12569,7 @@ fn genBinOpMir( | ... | @@ -12017,6 +12569,7 @@ fn genBinOpMir( |
| 12017 | .immediate, | 12569 | .immediate, |
| 12018 | .eflags, | 12570 | .eflags, |
| 12019 | .register_overflow, | 12571 | .register_overflow, |
| | 12572 | .register_mask, |
| 12020 | .lea_direct, | 12573 | .lea_direct, |
| 12021 | .lea_got, | 12574 | .lea_got, |
| 12022 | .lea_tlv, | 12575 | .lea_tlv, |
| ... | @@ -12026,9 +12579,9 @@ fn genBinOpMir( | ... | @@ -12026,9 +12579,9 @@ fn genBinOpMir( |
| 12026 | .reserved_frame, | 12579 | .reserved_frame, |
| 12027 | .air_ref, | 12580 | .air_ref, |
| 12028 | => unreachable, // unmodifiable destination | 12581 | => unreachable, // unmodifiable destination |
| 12029 | .register, .register_pair, .register_offset => { | 12582 | .register, .register_pair, .register_triple, .register_quadruple, .register_offset => { |
| 12030 | switch (dst_mcv) { | 12583 | switch (dst_mcv) { |
| 12031 | .register, .register_pair => {}, | 12584 | .register, .register_pair, .register_triple, .register_quadruple => {}, |
| 12032 | .register_offset => |ro| assert(ro.off == 0), | 12585 | .register_offset => |ro| assert(ro.off == 0), |
| 12033 | else => unreachable, | 12586 | else => unreachable, |
| 12034 | } | 12587 | } |
| ... | @@ -12057,10 +12610,15 @@ fn genBinOpMir( | ... | @@ -12057,10 +12610,15 @@ fn genBinOpMir( |
| 12057 | .dead, | 12610 | .dead, |
| 12058 | .undef, | 12611 | .undef, |
| 12059 | .register_overflow, | 12612 | .register_overflow, |
| | 12613 | .register_mask, |
| 12060 | .elementwise_regs_then_frame, | 12614 | .elementwise_regs_then_frame, |
| 12061 | .reserved_frame, | 12615 | .reserved_frame, |
| 12062 | => unreachable, | 12616 | => unreachable, |
| 12063 | .register, .register_pair => try self.asmRegisterRegister( | 12617 | .register, |
| | 12618 | .register_pair, |
| | 12619 | .register_triple, |
| | 12620 | .register_quadruple, |
| | 12621 | => try self.asmRegisterRegister( |
| 12064 | mir_limb_tag, | 12622 | mir_limb_tag, |
| 12065 | dst_alias, | 12623 | dst_alias, |
| 12066 | registerAlias(src_mcv.getRegs()[dst_reg_i], limb_abi_size), | 12624 | registerAlias(src_mcv.getRegs()[dst_reg_i], limb_abi_size), |
| ... | @@ -12216,6 +12774,7 @@ fn genBinOpMir( | ... | @@ -12216,6 +12774,7 @@ fn genBinOpMir( |
| 12216 | .dead, | 12774 | .dead, |
| 12217 | .undef, | 12775 | .undef, |
| 12218 | .register_overflow, | 12776 | .register_overflow, |
| | 12777 | .register_mask, |
| 12219 | .elementwise_regs_then_frame, | 12778 | .elementwise_regs_then_frame, |
| 12220 | .reserved_frame, | 12779 | .reserved_frame, |
| 12221 | .air_ref, | 12780 | .air_ref, |
| ... | @@ -12224,6 +12783,8 @@ fn genBinOpMir( | ... | @@ -12224,6 +12783,8 @@ fn genBinOpMir( |
| 12224 | .eflags, | 12783 | .eflags, |
| 12225 | .register, | 12784 | .register, |
| 12226 | .register_pair, | 12785 | .register_pair, |
| | 12786 | .register_triple, |
| | 12787 | .register_quadruple, |
| 12227 | .register_offset, | 12788 | .register_offset, |
| 12228 | .indirect, | 12789 | .indirect, |
| 12229 | .lea_direct, | 12790 | .lea_direct, |
| ... | @@ -12311,6 +12872,7 @@ fn genBinOpMir( | ... | @@ -12311,6 +12872,7 @@ fn genBinOpMir( |
| 12311 | .dead, | 12872 | .dead, |
| 12312 | .undef, | 12873 | .undef, |
| 12313 | .register_overflow, | 12874 | .register_overflow, |
| | 12875 | .register_mask, |
| 12314 | .elementwise_regs_then_frame, | 12876 | .elementwise_regs_then_frame, |
| 12315 | .reserved_frame, | 12877 | .reserved_frame, |
| 12316 | .air_ref, | 12878 | .air_ref, |
| ... | @@ -12364,6 +12926,8 @@ fn genBinOpMir( | ... | @@ -12364,6 +12926,8 @@ fn genBinOpMir( |
| 12364 | }, | 12926 | }, |
| 12365 | .register, | 12927 | .register, |
| 12366 | .register_pair, | 12928 | .register_pair, |
| | 12929 | .register_triple, |
| | 12930 | .register_quadruple, |
| 12367 | .register_offset, | 12931 | .register_offset, |
| 12368 | .eflags, | 12932 | .eflags, |
| 12369 | .memory, | 12933 | .memory, |
| ... | @@ -12382,7 +12946,7 @@ fn genBinOpMir( | ... | @@ -12382,7 +12946,7 @@ fn genBinOpMir( |
| 12382 | const src_limb_mcv: MCValue = if (src_info) |info| .{ | 12946 | const src_limb_mcv: MCValue = if (src_info) |info| .{ |
| 12383 | .indirect = .{ .reg = info.addr_reg, .off = off }, | 12947 | .indirect = .{ .reg = info.addr_reg, .off = off }, |
| 12384 | } else switch (resolved_src_mcv) { | 12948 | } else switch (resolved_src_mcv) { |
| 12385 | .register, .register_pair => .{ | 12949 | .register, .register_pair, .register_triple, .register_quadruple => .{ |
| 12386 | .register = resolved_src_mcv.getRegs()[limb_i], | 12950 | .register = resolved_src_mcv.getRegs()[limb_i], |
| 12387 | }, | 12951 | }, |
| 12388 | .eflags, | 12952 | .eflags, |
| ... | @@ -12438,6 +13002,7 @@ fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv | ... | @@ -12438,6 +13002,7 @@ fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv |
| 12438 | .eflags, | 13002 | .eflags, |
| 12439 | .register_offset, | 13003 | .register_offset, |
| 12440 | .register_overflow, | 13004 | .register_overflow, |
| | 13005 | .register_mask, |
| 12441 | .lea_symbol, | 13006 | .lea_symbol, |
| 12442 | .lea_direct, | 13007 | .lea_direct, |
| 12443 | .lea_got, | 13008 | .lea_got, |
| ... | @@ -12462,7 +13027,10 @@ fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv | ... | @@ -12462,7 +13027,10 @@ fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv |
| 12462 | .dead, | 13027 | .dead, |
| 12463 | .undef, | 13028 | .undef, |
| 12464 | .register_pair, | 13029 | .register_pair, |
| | 13030 | .register_triple, |
| | 13031 | .register_quadruple, |
| 12465 | .register_overflow, | 13032 | .register_overflow, |
| | 13033 | .register_mask, |
| 12466 | .elementwise_regs_then_frame, | 13034 | .elementwise_regs_then_frame, |
| 12467 | .reserved_frame, | 13035 | .reserved_frame, |
| 12468 | .air_ref, | 13036 | .air_ref, |
| ... | @@ -12539,7 +13107,7 @@ fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv | ... | @@ -12539,7 +13107,7 @@ fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv |
| 12539 | ), | 13107 | ), |
| 12540 | } | 13108 | } |
| 12541 | }, | 13109 | }, |
| 12542 | .register_pair => unreachable, // unimplemented | 13110 | .register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented |
| 12543 | .memory, .indirect, .load_symbol, .load_direct, .load_got, .load_tlv, .load_frame => { | 13111 | .memory, .indirect, .load_symbol, .load_direct, .load_got, .load_tlv, .load_frame => { |
| 12544 | const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv); | 13112 | const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv); |
| 12545 | const tmp_mcv = MCValue{ .register = tmp_reg }; | 13113 | const tmp_mcv = MCValue{ .register = tmp_reg }; |
| ... | @@ -12892,7 +13460,7 @@ fn genCall(self: *CodeGen, info: union(enum) { | ... | @@ -12892,7 +13460,7 @@ fn genCall(self: *CodeGen, info: union(enum) { |
| 12892 | const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); | 13460 | const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); |
| 12893 | defer self.register_manager.unlockReg(index_lock); | 13461 | defer self.register_manager.unlockReg(index_lock); |
| 12894 | | 13462 | |
| 12895 | const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .{ .size = .dword }) else .{ | 13463 | const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{ |
| 12896 | .base = .{ .reg = try self.copyToTmpRegister( | 13464 | .base = .{ .reg = try self.copyToTmpRegister( |
| 12897 | Type.usize, | 13465 | Type.usize, |
| 12898 | switch (src_arg) { | 13466 | switch (src_arg) { |
| ... | @@ -12984,7 +13552,7 @@ fn genCall(self: *CodeGen, info: union(enum) { | ... | @@ -12984,7 +13552,7 @@ fn genCall(self: *CodeGen, info: union(enum) { |
| 12984 | .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, | 13552 | .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, |
| 12985 | }, .{}), | 13553 | }, .{}), |
| 12986 | .elementwise_regs_then_frame => |regs_frame_addr| { | 13554 | .elementwise_regs_then_frame => |regs_frame_addr| { |
| 12987 | const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .{ .size = .dword }) else .{ | 13555 | const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{ |
| 12988 | .base = .{ .reg = try self.copyToTmpRegister( | 13556 | .base = .{ .reg = try self.copyToTmpRegister( |
| 12989 | Type.usize, | 13557 | Type.usize, |
| 12990 | switch (src_arg) { | 13558 | switch (src_arg) { |
| ... | @@ -13100,6 +13668,8 @@ fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { | ... | @@ -13100,6 +13668,8 @@ fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { |
| 13100 | .none => {}, | 13668 | .none => {}, |
| 13101 | .register, | 13669 | .register, |
| 13102 | .register_pair, | 13670 | .register_pair, |
| | 13671 | .register_triple, |
| | 13672 | .register_quadruple, |
| 13103 | => try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }), | 13673 | => try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }), |
| 13104 | .indirect => |reg_off| { | 13674 | .indirect => |reg_off| { |
| 13105 | try self.register_manager.getReg(reg_off.reg, null); | 13675 | try self.register_manager.getReg(reg_off.reg, null); |
| ... | @@ -13226,7 +13796,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v | ... | @@ -13226,7 +13796,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13226 | const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg); | 13796 | const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg); |
| 13227 | defer self.register_manager.unlockReg(temp_lhs_lock); | 13797 | defer self.register_manager.unlockReg(temp_lhs_lock); |
| 13228 | | 13798 | |
| 13229 | if (lhs_mcv.isMemory()) try self.asmRegisterMemory( | 13799 | if (lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 13230 | .{ ._, .mov }, | 13800 | .{ ._, .mov }, |
| 13231 | temp_lhs_reg.to8(), | 13801 | temp_lhs_reg.to8(), |
| 13232 | try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }), | 13802 | try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }), |
| ... | @@ -13240,7 +13810,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v | ... | @@ -13240,7 +13810,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13240 | } | 13810 | } |
| 13241 | | 13811 | |
| 13242 | const payload_compare = payload_compare: { | 13812 | const payload_compare = payload_compare: { |
| 13243 | if (rhs_mcv.isMemory()) { | 13813 | if (rhs_mcv.isBase()) { |
| 13244 | const rhs_mem = | 13814 | const rhs_mem = |
| 13245 | try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }); | 13815 | try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }); |
| 13246 | try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8()); | 13816 | try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8()); |
| ... | @@ -13291,13 +13861,13 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v | ... | @@ -13291,13 +13861,13 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13291 | } else .may_flip; | 13861 | } else .may_flip; |
| 13292 | | 13862 | |
| 13293 | const flipped = switch (may_flip) { | 13863 | const flipped = switch (may_flip) { |
| 13294 | .may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isMemory(), | 13864 | .may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isBase(), |
| 13295 | .must_flip => true, | 13865 | .must_flip => true, |
| 13296 | .must_not_flip => false, | 13866 | .must_not_flip => false, |
| 13297 | }; | 13867 | }; |
| 13298 | const unmat_dst_mcv = if (flipped) rhs_mcv else lhs_mcv; | 13868 | const unmat_dst_mcv = if (flipped) rhs_mcv else lhs_mcv; |
| 13299 | const dst_mcv = if (unmat_dst_mcv.isRegister() or | 13869 | const dst_mcv = if (unmat_dst_mcv.isRegister() or |
| 13300 | (abi_size <= 8 and unmat_dst_mcv.isMemory())) unmat_dst_mcv else dst: { | 13870 | (abi_size <= 8 and unmat_dst_mcv.isBase())) unmat_dst_mcv else dst: { |
| 13301 | const dst_mcv = try self.allocTempRegOrMem(ty, true); | 13871 | const dst_mcv = try self.allocTempRegOrMem(ty, true); |
| 13302 | try self.genCopy(ty, dst_mcv, unmat_dst_mcv, .{}); | 13872 | try self.genCopy(ty, dst_mcv, unmat_dst_mcv, .{}); |
| 13303 | break :dst dst_mcv; | 13873 | break :dst dst_mcv; |
| ... | @@ -13335,6 +13905,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v | ... | @@ -13335,6 +13905,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13335 | .register, | 13905 | .register, |
| 13336 | .register_offset, | 13906 | .register_offset, |
| 13337 | .register_overflow, | 13907 | .register_overflow, |
| | 13908 | .register_mask, |
| 13338 | .indirect, | 13909 | .indirect, |
| 13339 | .lea_direct, | 13910 | .lea_direct, |
| 13340 | .lea_got, | 13911 | .lea_got, |
| ... | @@ -13345,7 +13916,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v | ... | @@ -13345,7 +13916,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13345 | .reserved_frame, | 13916 | .reserved_frame, |
| 13346 | .air_ref, | 13917 | .air_ref, |
| 13347 | => unreachable, | 13918 | => unreachable, |
| 13348 | .register_pair, .load_frame => null, | 13919 | .register_pair, .register_triple, .register_quadruple, .load_frame => null, |
| 13349 | .memory, .load_symbol, .load_got, .load_direct, .load_tlv => dst: { | 13920 | .memory, .load_symbol, .load_got, .load_direct, .load_tlv => dst: { |
| 13350 | switch (resolved_dst_mcv) { | 13921 | switch (resolved_dst_mcv) { |
| 13351 | .memory => |addr| if (std.math.cast( | 13922 | .memory => |addr| if (std.math.cast( |
| ... | @@ -13396,6 +13967,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v | ... | @@ -13396,6 +13967,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13396 | .register, | 13967 | .register, |
| 13397 | .register_offset, | 13968 | .register_offset, |
| 13398 | .register_overflow, | 13969 | .register_overflow, |
| | 13970 | .register_mask, |
| 13399 | .indirect, | 13971 | .indirect, |
| 13400 | .lea_symbol, | 13972 | .lea_symbol, |
| 13401 | .lea_direct, | 13973 | .lea_direct, |
| ... | @@ -13406,7 +13978,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v | ... | @@ -13406,7 +13978,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13406 | .reserved_frame, | 13978 | .reserved_frame, |
| 13407 | .air_ref, | 13979 | .air_ref, |
| 13408 | => unreachable, | 13980 | => unreachable, |
| 13409 | .register_pair, .load_frame => null, | 13981 | .register_pair, .register_triple, .register_quadruple, .load_frame => null, |
| 13410 | .memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: { | 13982 | .memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: { |
| 13411 | switch (resolved_src_mcv) { | 13983 | switch (resolved_src_mcv) { |
| 13412 | .memory => |addr| if (std.math.cast( | 13984 | .memory => |addr| if (std.math.cast( |
| ... | @@ -13457,7 +14029,10 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v | ... | @@ -13457,7 +14029,10 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13457 | try self.genSetReg(tmp_reg, Type.usize, if (dst_info) |info| .{ | 14029 | try self.genSetReg(tmp_reg, Type.usize, if (dst_info) |info| .{ |
| 13458 | .indirect = .{ .reg = info.addr_reg, .off = off }, | 14030 | .indirect = .{ .reg = info.addr_reg, .off = off }, |
| 13459 | } else switch (resolved_dst_mcv) { | 14031 | } else switch (resolved_dst_mcv) { |
| 13460 | .register_pair => |dst_regs| .{ .register = dst_regs[limb_i] }, | 14032 | inline .register_pair, |
| | 14033 | .register_triple, |
| | 14034 | .register_quadruple, |
| | 14035 | => |dst_regs| .{ .register = dst_regs[limb_i] }, |
| 13461 | .memory => |dst_addr| .{ | 14036 | .memory => |dst_addr| .{ |
| 13462 | .memory = @bitCast(@as(i64, @bitCast(dst_addr)) + off), | 14037 | .memory = @bitCast(@as(i64, @bitCast(dst_addr)) + off), |
| 13463 | }, | 14038 | }, |
| ... | @@ -13479,9 +14054,10 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v | ... | @@ -13479,9 +14054,10 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13479 | if (src_info) |info| .{ | 14054 | if (src_info) |info| .{ |
| 13480 | .indirect = .{ .reg = info.addr_reg, .off = off }, | 14055 | .indirect = .{ .reg = info.addr_reg, .off = off }, |
| 13481 | } else switch (resolved_src_mcv) { | 14056 | } else switch (resolved_src_mcv) { |
| 13482 | .register_pair => |src_regs| .{ | 14057 | inline .register_pair, |
| 13483 | .register = src_regs[limb_i], | 14058 | .register_triple, |
| 13484 | }, | 14059 | .register_quadruple, |
| | 14060 | => |src_regs| .{ .register = src_regs[limb_i] }, |
| 13485 | .memory => |src_addr| .{ | 14061 | .memory => |src_addr| .{ |
| 13486 | .memory = @bitCast(@as(i64, @bitCast(src_addr)) + off), | 14062 | .memory = @bitCast(@as(i64, @bitCast(src_addr)) + off), |
| 13487 | }, | 14063 | }, |
| ... | @@ -13539,7 +14115,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v | ... | @@ -13539,7 +14115,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13539 | const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg); | 14115 | const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg); |
| 13540 | defer self.register_manager.unlockReg(tmp2_lock); | 14116 | defer self.register_manager.unlockReg(tmp2_lock); |
| 13541 | | 14117 | |
| 13542 | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( | 14118 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 13543 | .{ .vp_w, .insr }, | 14119 | .{ .vp_w, .insr }, |
| 13544 | tmp1_reg, | 14120 | tmp1_reg, |
| 13545 | dst_reg.to128(), | 14121 | dst_reg.to128(), |
| ... | @@ -13840,8 +14416,11 @@ fn isNull(self: *CodeGen, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) | ... | @@ -13840,8 +14416,11 @@ fn isNull(self: *CodeGen, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) |
| 13840 | .undef, | 14416 | .undef, |
| 13841 | .immediate, | 14417 | .immediate, |
| 13842 | .eflags, | 14418 | .eflags, |
| | 14419 | .register_triple, |
| | 14420 | .register_quadruple, |
| 13843 | .register_offset, | 14421 | .register_offset, |
| 13844 | .register_overflow, | 14422 | .register_overflow, |
| | 14423 | .register_mask, |
| 13845 | .lea_direct, | 14424 | .lea_direct, |
| 13846 | .lea_got, | 14425 | .lea_got, |
| 13847 | .lea_tlv, | 14426 | .lea_tlv, |
| ... | @@ -15481,6 +16060,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C | ... | @@ -15481,6 +16060,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C |
| 15481 | .immediate, | 16060 | .immediate, |
| 15482 | .eflags, | 16061 | .eflags, |
| 15483 | .register_overflow, | 16062 | .register_overflow, |
| | 16063 | .register_mask, |
| 15484 | .lea_direct, | 16064 | .lea_direct, |
| 15485 | .lea_got, | 16065 | .lea_got, |
| 15486 | .lea_tlv, | 16066 | .lea_tlv, |
| ... | @@ -15510,7 +16090,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C | ... | @@ -15510,7 +16090,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C |
| 15510 | .off = -dst_reg_off.off, | 16090 | .off = -dst_reg_off.off, |
| 15511 | } }, | 16091 | } }, |
| 15512 | }, opts), | 16092 | }, opts), |
| 15513 | .register_pair => |dst_regs| { | 16093 | inline .register_pair, .register_triple, .register_quadruple => |dst_regs| { |
| 15514 | const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = switch (src_mcv) { | 16094 | const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = switch (src_mcv) { |
| 15515 | .register_pair, .memory, .indirect, .load_frame => null, | 16095 | .register_pair, .memory, .indirect, .load_frame => null, |
| 15516 | .load_symbol, .load_direct, .load_got, .load_tlv => src: { | 16096 | .load_symbol, .load_direct, .load_got, .load_tlv => src: { |
| ... | @@ -15535,9 +16115,12 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C | ... | @@ -15535,9 +16115,12 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C |
| 15535 | defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock); | 16115 | defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock); |
| 15536 | | 16116 | |
| 15537 | var part_disp: i32 = 0; | 16117 | var part_disp: i32 = 0; |
| 15538 | for (dst_regs, try self.splitType(ty), 0..) |dst_reg, dst_ty, part_i| { | 16118 | for (dst_regs, try self.splitType(dst_regs.len, ty), 0..) |dst_reg, dst_ty, part_i| { |
| 15539 | try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) { | 16119 | try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) { |
| 15540 | .register_pair => |src_regs| .{ .register = src_regs[part_i] }, | 16120 | inline .register_pair, |
| | 16121 | .register_triple, |
| | 16122 | .register_quadruple, |
| | 16123 | => |src_regs| .{ .register = src_regs[part_i] }, |
| 15541 | .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(), | 16124 | .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(), |
| 15542 | .load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{ | 16125 | .load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{ |
| 15543 | .reg = src_info.?.addr_reg, | 16126 | .reg = src_info.?.addr_reg, |
| ... | @@ -15733,7 +16316,10 @@ fn genSetReg( | ... | @@ -15733,7 +16316,10 @@ fn genSetReg( |
| 15733 | }, | 16316 | }, |
| 15734 | .ip => unreachable, | 16317 | .ip => unreachable, |
| 15735 | }, | 16318 | }, |
| 15736 | .register_pair => |src_regs| try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts), | 16319 | inline .register_pair, |
| | 16320 | .register_triple, |
| | 16321 | .register_quadruple, |
| | 16322 | => |src_regs| try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts), |
| 15737 | .register_offset, | 16323 | .register_offset, |
| 15738 | .indirect, | 16324 | .indirect, |
| 15739 | .load_frame, | 16325 | .load_frame, |
| ... | @@ -15770,6 +16356,47 @@ fn genSetReg( | ... | @@ -15770,6 +16356,47 @@ fn genSetReg( |
| 15770 | }, | 16356 | }, |
| 15771 | else => unreachable, | 16357 | else => unreachable, |
| 15772 | }), | 16358 | }), |
| | 16359 | .register_mask => |src_reg_mask| { |
| | 16360 | assert(src_reg_mask.reg.class() == .sse); |
| | 16361 | const has_avx = self.hasFeature(.avx); |
| | 16362 | const bits_reg = switch (dst_reg.class()) { |
| | 16363 | .general_purpose => dst_reg, |
| | 16364 | else => try self.register_manager.allocReg(null, abi.RegisterClass.gp), |
| | 16365 | }; |
| | 16366 | const bits_lock = self.register_manager.lockReg(bits_reg); |
| | 16367 | defer if (bits_lock) |lock| self.register_manager.unlockReg(lock); |
| | 16368 | |
| | 16369 | const pack_reg = switch (src_reg_mask.scalar) { |
| | 16370 | else => src_reg_mask.reg, |
| | 16371 | .word => try self.register_manager.allocReg(null, abi.RegisterClass.sse), |
| | 16372 | }; |
| | 16373 | const pack_lock = self.register_manager.lockReg(pack_reg); |
| | 16374 | defer if (pack_lock) |lock| self.register_manager.unlockReg(lock); |
| | 16375 | |
| | 16376 | var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.scalar.bitSize(), 8)); |
| | 16377 | switch (src_reg_mask.scalar) { |
| | 16378 | else => {}, |
| | 16379 | .word => { |
| | 16380 | const src_alias = registerAlias(src_reg_mask.reg, mask_size); |
| | 16381 | const pack_alias = registerAlias(pack_reg, mask_size); |
| | 16382 | if (has_avx) { |
| | 16383 | try self.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, src_alias, src_alias); |
| | 16384 | } else { |
| | 16385 | try self.asmRegisterRegister(.{ ._, .movdqa }, pack_alias, src_alias); |
| | 16386 | try self.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias); |
| | 16387 | } |
| | 16388 | mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable; |
| | 16389 | }, |
| | 16390 | } |
| | 16391 | try self.asmRegisterRegister(.{ switch (src_reg_mask.scalar) { |
| | 16392 | .byte, .word => if (has_avx) .vp_b else .p_b, |
| | 16393 | .dword => if (has_avx) .v_ps else ._ps, |
| | 16394 | .qword => if (has_avx) .v_pd else ._pd, |
| | 16395 | else => unreachable, |
| | 16396 | }, .movmsk }, bits_reg.to32(), registerAlias(pack_reg, mask_size)); |
| | 16397 | if (src_reg_mask.inverted) try self.asmRegister(.{ ._, .not }, registerAlias(bits_reg, abi_size)); |
| | 16398 | try self.genSetReg(dst_reg, ty, .{ .register = bits_reg }, .{}); |
| | 16399 | }, |
| 15773 | .memory, .load_symbol, .load_direct, .load_got, .load_tlv => { | 16400 | .memory, .load_symbol, .load_direct, .load_got, .load_tlv => { |
| 15774 | switch (src_mcv) { | 16401 | switch (src_mcv) { |
| 15775 | .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| | 16402 | .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| |
| ... | @@ -15998,9 +16625,9 @@ fn genSetMem( | ... | @@ -15998,9 +16625,9 @@ fn genSetMem( |
| 15998 | src_alias, | 16625 | src_alias, |
| 15999 | ); | 16626 | ); |
| 16000 | }, | 16627 | }, |
| 16001 | .register_pair => |src_regs| { | 16628 | inline .register_pair, .register_triple, .register_quadruple => |src_regs| { |
| 16002 | var part_disp: i32 = disp; | 16629 | var part_disp: i32 = disp; |
| 16003 | for (try self.splitType(ty), src_regs) |src_ty, src_reg| { | 16630 | for (try self.splitType(src_regs.len, ty), src_regs) |src_ty, src_reg| { |
| 16004 | try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg }, opts); | 16631 | try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg }, opts); |
| 16005 | part_disp += @intCast(src_ty.abiSize(zcu)); | 16632 | part_disp += @intCast(src_ty.abiSize(zcu)); |
| 16006 | } | 16633 | } |
| ... | @@ -16065,6 +16692,13 @@ fn genSetMem( | ... | @@ -16065,6 +16692,13 @@ fn genSetMem( |
| 16065 | | 16692 | |
| 16066 | try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); | 16693 | try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); |
| 16067 | }, | 16694 | }, |
| | 16695 | .register_mask => { |
| | 16696 | const src_reg = try self.copyToTmpRegister(ty, src_mcv); |
| | 16697 | const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); |
| | 16698 | defer self.register_manager.unlockReg(src_lock); |
| | 16699 | |
| | 16700 | try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); |
| | 16701 | }, |
| 16068 | .memory, | 16702 | .memory, |
| 16069 | .indirect, | 16703 | .indirect, |
| 16070 | .load_direct, | 16704 | .load_direct, |
| ... | @@ -16283,7 +16917,7 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -16283,7 +16917,7 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 16283 | const dst_mcv = try self.allocRegOrMem(inst, true); | 16917 | const dst_mcv = try self.allocRegOrMem(inst, true); |
| 16284 | try self.genCopy(switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) { | 16918 | try self.genCopy(switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) { |
| 16285 | .lt => dst_ty, | 16919 | .lt => dst_ty, |
| 16286 | .eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty, | 16920 | .eq => if (!dst_mcv.isBase() or src_mcv.isBase()) dst_ty else src_ty, |
| 16287 | .gt => src_ty, | 16921 | .gt => src_ty, |
| 16288 | }, dst_mcv, src_mcv, .{}); | 16922 | }, dst_mcv, src_mcv, .{}); |
| 16289 | break :dst dst_mcv; | 16923 | break :dst dst_mcv; |
| ... | @@ -16729,7 +17363,7 @@ fn atomicOp( | ... | @@ -16729,7 +17363,7 @@ fn atomicOp( |
| 16729 | }); | 17363 | }); |
| 16730 | try self.genSetReg(sse_reg, val_ty, .{ .register = .rax }, .{}); | 17364 | try self.genSetReg(sse_reg, val_ty, .{ .register = .rax }, .{}); |
| 16731 | switch (mir_tag[0]) { | 17365 | switch (mir_tag[0]) { |
| 16732 | .v_ss, .v_sd => if (val_mcv.isMemory()) try self.asmRegisterRegisterMemory( | 17366 | .v_ss, .v_sd => if (val_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 16733 | mir_tag, | 17367 | mir_tag, |
| 16734 | sse_reg.to128(), | 17368 | sse_reg.to128(), |
| 16735 | sse_reg.to128(), | 17369 | sse_reg.to128(), |
| ... | @@ -16743,7 +17377,7 @@ fn atomicOp( | ... | @@ -16743,7 +17377,7 @@ fn atomicOp( |
| 16743 | else | 17377 | else |
| 16744 | try self.copyToTmpRegister(val_ty, val_mcv)).to128(), | 17378 | try self.copyToTmpRegister(val_ty, val_mcv)).to128(), |
| 16745 | ), | 17379 | ), |
| 16746 | ._ss, ._sd => if (val_mcv.isMemory()) try self.asmRegisterMemory( | 17380 | ._ss, ._sd => if (val_mcv.isBase()) try self.asmRegisterMemory( |
| 16747 | mir_tag, | 17381 | mir_tag, |
| 16748 | sse_reg.to128(), | 17382 | sse_reg.to128(), |
| 16749 | try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }), | 17383 | try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }), |
| ... | @@ -17443,7 +18077,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -17443,7 +18077,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17443 | defer self.register_manager.unlockReg(dst_lock); | 18077 | defer self.register_manager.unlockReg(dst_lock); |
| 17444 | | 18078 | |
| 17445 | const src_mcv = try self.resolveInst(ty_op.operand); | 18079 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17446 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 18080 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17447 | mir_tag, | 18081 | mir_tag, |
| 17448 | registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))), | 18082 | registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))), |
| 17449 | try src_mcv.mem(self, .{ .size = self.memSize(scalar_ty) }), | 18083 | try src_mcv.mem(self, .{ .size = self.memSize(scalar_ty) }), |
| ... | @@ -17519,7 +18153,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -17519,7 +18153,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17519 | const src_mcv = try self.resolveInst(ty_op.operand); | 18153 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17520 | if (self.hasFeature(.avx)) { | 18154 | if (self.hasFeature(.avx)) { |
| 17521 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); | 18155 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); |
| 17522 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 18156 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17523 | .{ .v_ss, .broadcast }, | 18157 | .{ .v_ss, .broadcast }, |
| 17524 | dst_reg.to128(), | 18158 | dst_reg.to128(), |
| 17525 | try src_mcv.mem(self, .{ .size = .dword }), | 18159 | try src_mcv.mem(self, .{ .size = .dword }), |
| ... | @@ -17556,7 +18190,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -17556,7 +18190,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17556 | 5...8 => if (self.hasFeature(.avx)) { | 18190 | 5...8 => if (self.hasFeature(.avx)) { |
| 17557 | const src_mcv = try self.resolveInst(ty_op.operand); | 18191 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17558 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); | 18192 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); |
| 17559 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 18193 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17560 | .{ .v_ss, .broadcast }, | 18194 | .{ .v_ss, .broadcast }, |
| 17561 | dst_reg.to256(), | 18195 | dst_reg.to256(), |
| 17562 | try src_mcv.mem(self, .{ .size = .dword }), | 18196 | try src_mcv.mem(self, .{ .size = .dword }), |
| ... | @@ -17602,7 +18236,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -17602,7 +18236,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17602 | const src_mcv = try self.resolveInst(ty_op.operand); | 18236 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17603 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); | 18237 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); |
| 17604 | if (self.hasFeature(.sse3)) { | 18238 | if (self.hasFeature(.sse3)) { |
| 17605 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 18239 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17606 | if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup }, | 18240 | if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup }, |
| 17607 | dst_reg.to128(), | 18241 | dst_reg.to128(), |
| 17608 | try src_mcv.mem(self, .{ .size = .qword }), | 18242 | try src_mcv.mem(self, .{ .size = .qword }), |
| ... | @@ -17627,7 +18261,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -17627,7 +18261,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17627 | 3...4 => if (self.hasFeature(.avx)) { | 18261 | 3...4 => if (self.hasFeature(.avx)) { |
| 17628 | const src_mcv = try self.resolveInst(ty_op.operand); | 18262 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17629 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); | 18263 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); |
| 17630 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 18264 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17631 | .{ .v_sd, .broadcast }, | 18265 | .{ .v_sd, .broadcast }, |
| 17632 | dst_reg.to256(), | 18266 | dst_reg.to256(), |
| 17633 | try src_mcv.mem(self, .{ .size = .qword }), | 18267 | try src_mcv.mem(self, .{ .size = .qword }), |
| ... | @@ -17670,7 +18304,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -17670,7 +18304,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17670 | 2 => if (self.hasFeature(.avx)) { | 18304 | 2 => if (self.hasFeature(.avx)) { |
| 17671 | const src_mcv = try self.resolveInst(ty_op.operand); | 18305 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17672 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); | 18306 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); |
| 17673 | if (src_mcv.isMemory()) try self.asmRegisterMemory( | 18307 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17674 | .{ .v_f128, .broadcast }, | 18308 | .{ .v_f128, .broadcast }, |
| 17675 | dst_reg.to256(), | 18309 | dst_reg.to256(), |
| 17676 | try src_mcv.mem(self, .{ .size = .xword }), | 18310 | try src_mcv.mem(self, .{ .size = .xword }), |
| ... | @@ -17779,7 +18413,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -17779,7 +18413,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17779 | else => unreachable, | 18413 | else => unreachable, |
| 17780 | }, .broadcast }, | 18414 | }, .broadcast }, |
| 17781 | mask_alias, | 18415 | mask_alias, |
| 17782 | if (pred_mcv.isMemory()) try pred_mcv.mem(self, .{ .size = .byte }) else .{ | 18416 | if (pred_mcv.isBase()) try pred_mcv.mem(self, .{ .size = .byte }) else .{ |
| 17783 | .base = .{ .reg = (try self.copyToTmpRegister( | 18417 | .base = .{ .reg = (try self.copyToTmpRegister( |
| 17784 | Type.usize, | 18418 | Type.usize, |
| 17785 | pred_mcv.address(), | 18419 | pred_mcv.address(), |
| ... | @@ -17973,7 +18607,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -17973,7 +18607,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17973 | try self.genSetReg(dst_reg, ty, rhs_mcv, .{}); | 18607 | try self.genSetReg(dst_reg, ty, rhs_mcv, .{}); |
| 17974 | break :rhs dst_alias; | 18608 | break :rhs dst_alias; |
| 17975 | }; | 18609 | }; |
| 17976 | if (lhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister( | 18610 | if (lhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( |
| 17977 | mir_tag, | 18611 | mir_tag, |
| 17978 | dst_alias, | 18612 | dst_alias, |
| 17979 | rhs_alias, | 18613 | rhs_alias, |
| ... | @@ -17989,7 +18623,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -17989,7 +18623,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17989 | try self.copyToTmpRegister(ty, lhs_mcv), abi_size), | 18623 | try self.copyToTmpRegister(ty, lhs_mcv), abi_size), |
| 17990 | mask_alias, | 18624 | mask_alias, |
| 17991 | ); | 18625 | ); |
| 17992 | } else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister( | 18626 | } else if (has_blend) if (lhs_mcv.isBase()) try self.asmRegisterMemoryRegister( |
| 17993 | mir_tag, | 18627 | mir_tag, |
| 17994 | dst_alias, | 18628 | dst_alias, |
| 17995 | try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }), | 18629 | try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }), |
| ... | @@ -18014,7 +18648,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18014,7 +18648,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18014 | }, | 18648 | }, |
| 18015 | }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(pt)}); | 18649 | }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(pt)}); |
| 18016 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); | 18650 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); |
| 18017 | if (rhs_mcv.isMemory()) try self.asmRegisterMemory( | 18651 | if (rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 18018 | .{ mir_fixes, .andn }, | 18652 | .{ mir_fixes, .andn }, |
| 18019 | mask_alias, | 18653 | mask_alias, |
| 18020 | try rhs_mcv.mem(self, .{ .size = .fromSize(abi_size) }), | 18654 | try rhs_mcv.mem(self, .{ .size = .fromSize(abi_size) }), |
| ... | @@ -18093,10 +18727,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18093,10 +18727,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18093 | | 18727 | |
| 18094 | for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) |variant| unpck: { | 18728 | for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) |variant| unpck: { |
| 18095 | if (elem_abi_size > 8) break :unpck; | 18729 | if (elem_abi_size > 8) break :unpck; |
| 18096 | if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4) | 18730 | if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :unpck; |
| 18097 | has_avx | | |
| 18098 | else | | |
| 18099 | self.hasFeature(.avx2)) 32 else 16)) break :unpck; | | |
| 18100 | | 18731 | |
| 18101 | var sources: [2]?u1 = @splat(null); | 18732 | var sources: [2]?u1 = @splat(null); |
| 18102 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { | 18733 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { |
| ... | @@ -18154,7 +18785,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18154,7 +18785,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18154 | }, | 18785 | }, |
| 18155 | else => unreachable, | 18786 | else => unreachable, |
| 18156 | } }; | 18787 | } }; |
| 18157 | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemory( | 18788 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 18158 | mir_tag, | 18789 | mir_tag, |
| 18159 | dst_alias, | 18790 | dst_alias, |
| 18160 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), | 18791 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), |
| ... | @@ -18167,7 +18798,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18167,7 +18798,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18167 | rhs_mcv.getReg().? | 18798 | rhs_mcv.getReg().? |
| 18168 | else | 18799 | else |
| 18169 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), | 18800 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), |
| 18170 | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory( | 18801 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 18171 | mir_tag, | 18802 | mir_tag, |
| 18172 | dst_alias, | 18803 | dst_alias, |
| 18173 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), | 18804 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), |
| ... | @@ -18184,7 +18815,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18184,7 +18815,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18184 | | 18815 | |
| 18185 | pshufd: { | 18816 | pshufd: { |
| 18186 | if (elem_abi_size != 4) break :pshufd; | 18817 | if (elem_abi_size != 4) break :pshufd; |
| 18187 | if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd; | 18818 | if (max_abi_size > self.vectorSize(.float)) break :pshufd; |
| 18188 | | 18819 | |
| 18189 | var control: u8 = 0b00_00_00_00; | 18820 | var control: u8 = 0b00_00_00_00; |
| 18190 | var sources: [1]?u1 = @splat(null); | 18821 | var sources: [1]?u1 = @splat(null); |
| ... | @@ -18216,7 +18847,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18216,7 +18847,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18216 | try self.register_manager.allocReg(inst, abi.RegisterClass.sse); | 18847 | try self.register_manager.allocReg(inst, abi.RegisterClass.sse); |
| 18217 | const dst_alias = registerAlias(dst_reg, max_abi_size); | 18848 | const dst_alias = registerAlias(dst_reg, max_abi_size); |
| 18218 | | 18849 | |
| 18219 | if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( | 18850 | if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 18220 | .{ if (has_avx) .vp_d else .p_d, .shuf }, | 18851 | .{ if (has_avx) .vp_d else .p_d, .shuf }, |
| 18221 | dst_alias, | 18852 | dst_alias, |
| 18222 | try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), | 18853 | try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), |
| ... | @@ -18235,7 +18866,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18235,7 +18866,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18235 | | 18866 | |
| 18236 | shufps: { | 18867 | shufps: { |
| 18237 | if (elem_abi_size != 4) break :shufps; | 18868 | if (elem_abi_size != 4) break :shufps; |
| 18238 | if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps; | 18869 | if (max_abi_size > self.vectorSize(.float)) break :shufps; |
| 18239 | | 18870 | |
| 18240 | var control: u8 = 0b00_00_00_00; | 18871 | var control: u8 = 0b00_00_00_00; |
| 18241 | var sources: [2]?u1 = @splat(null); | 18872 | var sources: [2]?u1 = @splat(null); |
| ... | @@ -18272,7 +18903,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18272,7 +18903,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18272 | const dst_reg = dst_mcv.getReg().?; | 18903 | const dst_reg = dst_mcv.getReg().?; |
| 18273 | const dst_alias = registerAlias(dst_reg, max_abi_size); | 18904 | const dst_alias = registerAlias(dst_reg, max_abi_size); |
| 18274 | | 18905 | |
| 18275 | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( | 18906 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 18276 | .{ .v_ps, .shuf }, | 18907 | .{ .v_ps, .shuf }, |
| 18277 | dst_alias, | 18908 | dst_alias, |
| 18278 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), | 18909 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), |
| ... | @@ -18287,7 +18918,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18287,7 +18918,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18287 | else | 18918 | else |
| 18288 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), | 18919 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), |
| 18289 | .u(control), | 18920 | .u(control), |
| 18290 | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( | 18921 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 18291 | .{ ._ps, .shuf }, | 18922 | .{ ._ps, .shuf }, |
| 18292 | dst_alias, | 18923 | dst_alias, |
| 18293 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), | 18924 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), |
| ... | @@ -18306,7 +18937,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18306,7 +18937,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18306 | | 18937 | |
| 18307 | shufpd: { | 18938 | shufpd: { |
| 18308 | if (elem_abi_size != 8) break :shufpd; | 18939 | if (elem_abi_size != 8) break :shufpd; |
| 18309 | if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd; | 18940 | if (max_abi_size > self.vectorSize(.float)) break :shufpd; |
| 18310 | | 18941 | |
| 18311 | var control: u4 = 0b0_0_0_0; | 18942 | var control: u4 = 0b0_0_0_0; |
| 18312 | var sources: [2]?u1 = @splat(null); | 18943 | var sources: [2]?u1 = @splat(null); |
| ... | @@ -18339,7 +18970,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18339,7 +18970,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18339 | const dst_reg = dst_mcv.getReg().?; | 18970 | const dst_reg = dst_mcv.getReg().?; |
| 18340 | const dst_alias = registerAlias(dst_reg, max_abi_size); | 18971 | const dst_alias = registerAlias(dst_reg, max_abi_size); |
| 18341 | | 18972 | |
| 18342 | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( | 18973 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 18343 | .{ .v_pd, .shuf }, | 18974 | .{ .v_pd, .shuf }, |
| 18344 | dst_alias, | 18975 | dst_alias, |
| 18345 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), | 18976 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), |
| ... | @@ -18354,7 +18985,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18354,7 +18985,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18354 | else | 18985 | else |
| 18355 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), | 18986 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), |
| 18356 | .u(control), | 18987 | .u(control), |
| 18357 | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( | 18988 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 18358 | .{ ._pd, .shuf }, | 18989 | .{ ._pd, .shuf }, |
| 18359 | dst_alias, | 18990 | dst_alias, |
| 18360 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), | 18991 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), |
| ... | @@ -18373,7 +19004,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18373,7 +19004,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18373 | | 19004 | |
| 18374 | blend: { | 19005 | blend: { |
| 18375 | if (elem_abi_size < 2) break :blend; | 19006 | if (elem_abi_size < 2) break :blend; |
| 18376 | if (dst_abi_size > @as(u32, if (has_avx) 32 else 16)) break :blend; | 19007 | if (dst_abi_size > self.vectorSize(.float)) break :blend; |
| 18377 | if (!self.hasFeature(.sse4_1)) break :blend; | 19008 | if (!self.hasFeature(.sse4_1)) break :blend; |
| 18378 | | 19009 | |
| 18379 | var control: u8 = 0b0_0_0_0_0_0_0_0; | 19010 | var control: u8 = 0b0_0_0_0_0_0_0_0; |
| ... | @@ -18409,7 +19040,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18409,7 +19040,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18409 | | 19040 | |
| 18410 | const rhs_mcv = try self.resolveInst(extra.b); | 19041 | const rhs_mcv = try self.resolveInst(extra.b); |
| 18411 | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); | 19042 | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); |
| 18412 | if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( | 19043 | if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 18413 | .{ .vp_d, .blend }, | 19044 | .{ .vp_d, .blend }, |
| 18414 | registerAlias(dst_reg, dst_abi_size), | 19045 | registerAlias(dst_reg, dst_abi_size), |
| 18415 | registerAlias(lhs_reg, dst_abi_size), | 19046 | registerAlias(lhs_reg, dst_abi_size), |
| ... | @@ -18461,7 +19092,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18461,7 +19092,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18461 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); | 19092 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); |
| 18462 | const dst_reg = dst_mcv.getReg().?; | 19093 | const dst_reg = dst_mcv.getReg().?; |
| 18463 | | 19094 | |
| 18464 | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( | 19095 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 18465 | .{ .vp_w, .blend }, | 19096 | .{ .vp_w, .blend }, |
| 18466 | registerAlias(dst_reg, dst_abi_size), | 19097 | registerAlias(dst_reg, dst_abi_size), |
| 18467 | registerAlias(if (lhs_mcv.isRegister()) | 19098 | registerAlias(if (lhs_mcv.isRegister()) |
| ... | @@ -18482,7 +19113,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18482,7 +19113,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18482 | else | 19113 | else |
| 18483 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), | 19114 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| 18484 | .u(expanded_control), | 19115 | .u(expanded_control), |
| 18485 | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( | 19116 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 18486 | .{ .p_w, .blend }, | 19117 | .{ .p_w, .blend }, |
| 18487 | registerAlias(dst_reg, dst_abi_size), | 19118 | registerAlias(dst_reg, dst_abi_size), |
| 18488 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), | 19119 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), |
| ... | @@ -18518,7 +19149,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18518,7 +19149,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18518 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); | 19149 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); |
| 18519 | const dst_reg = dst_mcv.getReg().?; | 19150 | const dst_reg = dst_mcv.getReg().?; |
| 18520 | | 19151 | |
| 18521 | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( | 19152 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 18522 | switch (elem_abi_size) { | 19153 | switch (elem_abi_size) { |
| 18523 | 4 => .{ .v_ps, .blend }, | 19154 | 4 => .{ .v_ps, .blend }, |
| 18524 | 8, 16 => .{ .v_pd, .blend }, | 19155 | 8, 16 => .{ .v_pd, .blend }, |
| ... | @@ -18547,7 +19178,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18547,7 +19178,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18547 | else | 19178 | else |
| 18548 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), | 19179 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| 18549 | .u(expanded_control), | 19180 | .u(expanded_control), |
| 18550 | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( | 19181 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 18551 | switch (elem_abi_size) { | 19182 | switch (elem_abi_size) { |
| 18552 | 4 => .{ ._ps, .blend }, | 19183 | 4 => .{ ._ps, .blend }, |
| 18553 | 8, 16 => .{ ._pd, .blend }, | 19184 | 8, 16 => .{ ._pd, .blend }, |
| ... | @@ -18573,10 +19204,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18573,10 +19204,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18573 | } | 19204 | } |
| 18574 | | 19205 | |
| 18575 | blendv: { | 19206 | blendv: { |
| 18576 | if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4) | 19207 | if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :blendv; |
| 18577 | has_avx | | |
| 18578 | else | | |
| 18579 | self.hasFeature(.avx2)) 32 else 16)) break :blendv; | | |
| 18580 | | 19208 | |
| 18581 | const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8); | 19209 | const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8); |
| 18582 | const select_mask_ty = try pt.vectorType(.{ | 19210 | const select_mask_ty = try pt.vectorType(.{ |
| ... | @@ -18637,7 +19265,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18637,7 +19265,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18637 | const dst_reg = dst_mcv.getReg().?; | 19265 | const dst_reg = dst_mcv.getReg().?; |
| 18638 | const dst_alias = registerAlias(dst_reg, dst_abi_size); | 19266 | const dst_alias = registerAlias(dst_reg, dst_abi_size); |
| 18639 | | 19267 | |
| 18640 | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister( | 19268 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( |
| 18641 | mir_tag, | 19269 | mir_tag, |
| 18642 | dst_alias, | 19270 | dst_alias, |
| 18643 | if (lhs_mcv.isRegister()) | 19271 | if (lhs_mcv.isRegister()) |
| ... | @@ -18658,7 +19286,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18658,7 +19286,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18658 | else | 19286 | else |
| 18659 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), | 19287 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| 18660 | select_mask_alias, | 19288 | select_mask_alias, |
| 18661 | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister( | 19289 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryRegister( |
| 18662 | mir_tag, | 19290 | mir_tag, |
| 18663 | dst_alias, | 19291 | dst_alias, |
| 18664 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), | 19292 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), |
| ... | @@ -18701,7 +19329,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18701,7 +19329,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18701 | else | 19329 | else |
| 18702 | .p_; | 19330 | .p_; |
| 18703 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); | 19331 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); |
| 18704 | if (lhs_mcv.isMemory()) try self.asmRegisterMemory( | 19332 | if (lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 18705 | .{ mir_fixes, .andn }, | 19333 | .{ mir_fixes, .andn }, |
| 18706 | mask_alias, | 19334 | mask_alias, |
| 18707 | try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), | 19335 | try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), |
| ... | @@ -18851,22 +19479,61 @@ fn airReduce(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18851,22 +19479,61 @@ fn airReduce(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18851 | if (operand_ty.isVector(zcu) and operand_ty.childType(zcu).toIntern() == .bool_type) { | 19479 | if (operand_ty.isVector(zcu) and operand_ty.childType(zcu).toIntern() == .bool_type) { |
| 18852 | try self.spillEflagsIfOccupied(); | 19480 | try self.spillEflagsIfOccupied(); |
| 18853 | | 19481 | |
| 18854 | const operand_mcv = try self.resolveInst(reduce.operand); | | |
| 18855 | const mask_len = (std.math.cast(u6, operand_ty.vectorLen(zcu)) orelse | | |
| 18856 | return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)})); | | |
| 18857 | const mask = (@as(u64, 1) << mask_len) - 1; | | |
| 18858 | const abi_size: u32 = @intCast(operand_ty.abiSize(zcu)); | 19482 | const abi_size: u32 = @intCast(operand_ty.abiSize(zcu)); |
| | 19483 | const operand_mcv = try self.resolveInst(reduce.operand); |
| | 19484 | const mask_len = operand_ty.vectorLen(zcu); |
| | 19485 | const mask_len_minus_one = (std.math.cast(u6, mask_len - 1) orelse { |
| | 19486 | const acc_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| | 19487 | const acc_lock = self.register_manager.lockRegAssumeUnused(acc_reg); |
| | 19488 | defer self.register_manager.unlockReg(acc_lock); |
| | 19489 | var limb_offset: i31 = 0; |
| | 19490 | while (limb_offset < abi_size) : (limb_offset += 8) { |
| | 19491 | try self.asmRegisterMemory( |
| | 19492 | .{ ._, if (limb_offset == 0) .mov else switch (reduce.operation) { |
| | 19493 | .Or => .@"or", |
| | 19494 | .And => .@"and", |
| | 19495 | else => return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}), |
| | 19496 | } }, |
| | 19497 | acc_reg.to64(), |
| | 19498 | try operand_mcv.mem(self, .{ |
| | 19499 | .size = .qword, |
| | 19500 | .disp = limb_offset, |
| | 19501 | }), |
| | 19502 | ); |
| | 19503 | } |
| | 19504 | switch (reduce.operation) { |
| | 19505 | .Or => { |
| | 19506 | try self.asmRegisterRegister(.{ ._, .@"test" }, acc_reg.to64(), acc_reg.to64()); |
| | 19507 | break :result .{ .eflags = .nz }; |
| | 19508 | }, |
| | 19509 | .And => { |
| | 19510 | try self.asmRegisterImmediate(.{ ._, .cmp }, acc_reg.to64(), .s(-1)); |
| | 19511 | break :result .{ .eflags = .z }; |
| | 19512 | }, |
| | 19513 | else => unreachable, |
| | 19514 | } |
| | 19515 | }); |
| | 19516 | const mask = @as(u64, std.math.maxInt(u64)) >> ~mask_len_minus_one; |
| 18859 | switch (reduce.operation) { | 19517 | switch (reduce.operation) { |
| 18860 | .Or => { | 19518 | .Or => { |
| 18861 | if (operand_mcv.isMemory()) try self.asmMemoryImmediate( | 19519 | if (operand_mcv.isBase()) try self.asmMemoryImmediate( |
| 18862 | .{ ._, .@"test" }, | 19520 | .{ ._, .@"test" }, |
| 18863 | try operand_mcv.mem(self, .{ .size = .fromSize(abi_size) }), | 19521 | try operand_mcv.mem(self, .{ .size = .fromSize(abi_size) }), |
| 18864 | .u(mask), | 19522 | if (mask_len < abi_size * 8) |
| 18865 | ) else { | 19523 | .u(mask) |
| 18866 | const operand_reg = registerAlias(if (operand_mcv.isRegister()) | | |
| 18867 | operand_mcv.getReg().? | | |
| 18868 | else | 19524 | else |
| 18869 | try self.copyToTmpRegister(operand_ty, operand_mcv), abi_size); | 19525 | .s(-1), |
| | 19526 | ) else { |
| | 19527 | const operand_reg = registerAlias(operand_reg: { |
| | 19528 | if (operand_mcv.isRegister()) { |
| | 19529 | const operand_reg = operand_mcv.getReg().?; |
| | 19530 | if (operand_reg.class() == .general_purpose) break :operand_reg operand_reg; |
| | 19531 | } |
| | 19532 | break :operand_reg try self.copyToTmpRegister(operand_ty, operand_mcv); |
| | 19533 | }, abi_size); |
| | 19534 | const operand_lock = self.register_manager.lockReg(operand_reg); |
| | 19535 | defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); |
| | 19536 | |
| 18870 | if (mask_len < abi_size * 8) try self.asmRegisterImmediate( | 19537 | if (mask_len < abi_size * 8) try self.asmRegisterImmediate( |
| 18871 | .{ ._, .@"test" }, | 19538 | .{ ._, .@"test" }, |
| 18872 | operand_reg, | 19539 | operand_reg, |
| ... | @@ -18880,7 +19547,10 @@ fn airReduce(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -18880,7 +19547,10 @@ fn airReduce(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18880 | break :result .{ .eflags = .nz }; | 19547 | break :result .{ .eflags = .nz }; |
| 18881 | }, | 19548 | }, |
| 18882 | .And => { | 19549 | .And => { |
| 18883 | const tmp_reg = try self.copyToTmpRegister(operand_ty, operand_mcv); | 19550 | const tmp_reg = registerAlias( |
| | 19551 | try self.copyToTmpRegister(operand_ty, operand_mcv), |
| | 19552 | abi_size, |
| | 19553 | ); |
| 18884 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | 19554 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 18885 | defer self.register_manager.unlockReg(tmp_lock); | 19555 | defer self.register_manager.unlockReg(tmp_lock); |
| 18886 | | 19556 | |
| ... | @@ -19184,7 +19854,7 @@ fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -19184,7 +19854,7 @@ fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 19184 | if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) { | 19854 | if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) { |
| 19185 | order[op_index] = 1; | 19855 | order[op_index] = 1; |
| 19186 | unused.unset(0); | 19856 | unused.unset(0); |
| 19187 | } else if (unused.isSet(2) and mcv.isMemory()) { | 19857 | } else if (unused.isSet(2) and mcv.isBase()) { |
| 19188 | order[op_index] = 3; | 19858 | order[op_index] = 3; |
| 19189 | unused.unset(2); | 19859 | unused.unset(2); |
| 19190 | } | 19860 | } |
| ... | @@ -19531,7 +20201,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -19531,7 +20201,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 19531 | const dst_lock = self.register_manager.lockReg(dst_reg); | 20201 | const dst_lock = self.register_manager.lockReg(dst_reg); |
| 19532 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); | 20202 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| 19533 | | 20203 | |
| 19534 | if (self.hasFeature(.avx)) if (promote_mcv.isMemory()) try self.asmRegisterRegisterMemory( | 20204 | if (self.hasFeature(.avx)) if (promote_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 19535 | .{ .v_ss, .cvtsd2 }, | 20205 | .{ .v_ss, .cvtsd2 }, |
| 19536 | dst_reg, | 20206 | dst_reg, |
| 19537 | dst_reg, | 20207 | dst_reg, |
| ... | @@ -19544,7 +20214,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -19544,7 +20214,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 19544 | promote_mcv.getReg().? | 20214 | promote_mcv.getReg().? |
| 19545 | else | 20215 | else |
| 19546 | try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(), | 20216 | try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(), |
| 19547 | ) else if (promote_mcv.isMemory()) try self.asmRegisterMemory( | 20217 | ) else if (promote_mcv.isBase()) try self.asmRegisterMemory( |
| 19548 | .{ ._ss, .cvtsd2 }, | 20218 | .{ ._ss, .cvtsd2 }, |
| 19549 | dst_reg, | 20219 | dst_reg, |
| 19550 | try promote_mcv.mem(self, .{ .size = .qword }), | 20220 | try promote_mcv.mem(self, .{ .size = .qword }), |
| ... | @@ -19751,7 +20421,7 @@ fn resolveCallingConventionValues( | ... | @@ -19751,7 +20421,7 @@ fn resolveCallingConventionValues( |
| 19751 | // TODO: is this even possible for C calling convention? | 20421 | // TODO: is this even possible for C calling convention? |
| 19752 | result.return_value = .init(.none); | 20422 | result.return_value = .init(.none); |
| 19753 | } else { | 20423 | } else { |
| 19754 | var ret_tracking: [2]InstTracking = undefined; | 20424 | var ret_tracking: [4]InstTracking = undefined; |
| 19755 | var ret_tracking_i: usize = 0; | 20425 | var ret_tracking_i: usize = 0; |
| 19756 | | 20426 | |
| 19757 | const classes = switch (resolved_cc) { | 20427 | const classes = switch (resolved_cc) { |
| ... | @@ -19771,14 +20441,17 @@ fn resolveCallingConventionValues( | ... | @@ -19771,14 +20441,17 @@ fn resolveCallingConventionValues( |
| 19771 | ret_tracking_i += 1; | 20441 | ret_tracking_i += 1; |
| 19772 | }, | 20442 | }, |
| 19773 | .sse, .float, .float_combine, .win_i128 => { | 20443 | .sse, .float, .float_combine, .win_i128 => { |
| 19774 | const ret_sse_reg = registerAlias( | 20444 | const ret_sse_regs = abi.getCAbiSseReturnRegs(resolved_cc); |
| 19775 | abi.getCAbiSseReturnRegs(resolved_cc)[ret_sse_reg_i], | 20445 | const abi_size: u32 = @intCast(ret_ty.abiSize(zcu)); |
| 19776 | @intCast(ret_ty.abiSize(zcu)), | 20446 | const reg_size = @min(abi_size, self.vectorSize(.float)); |
| 19777 | ); | 20447 | var byte_offset: u32 = 0; |
| 19778 | ret_sse_reg_i += 1; | 20448 | while (byte_offset < abi_size) : (byte_offset += reg_size) { |
| 19779 | | 20449 | const ret_sse_reg = registerAlias(ret_sse_regs[ret_sse_reg_i], reg_size); |
| 19780 | ret_tracking[ret_tracking_i] = .init(.{ .register = ret_sse_reg }); | 20450 | ret_sse_reg_i += 1; |
| 19781 | ret_tracking_i += 1; | 20451 | |
| | 20452 | ret_tracking[ret_tracking_i] = .init(.{ .register = ret_sse_reg }); |
| | 20453 | ret_tracking_i += 1; |
| | 20454 | } |
| 19782 | }, | 20455 | }, |
| 19783 | .sseup => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .sse), | 20456 | .sseup => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .sse), |
| 19784 | .x87 => { | 20457 | .x87 => { |
| ... | @@ -19811,6 +20484,17 @@ fn resolveCallingConventionValues( | ... | @@ -19811,6 +20484,17 @@ fn resolveCallingConventionValues( |
| 19811 | ret_tracking[0].short.register, | 20484 | ret_tracking[0].short.register, |
| 19812 | ret_tracking[1].short.register, | 20485 | ret_tracking[1].short.register, |
| 19813 | } }), | 20486 | } }), |
| | 20487 | 3 => .init(.{ .register_triple = .{ |
| | 20488 | ret_tracking[0].short.register, |
| | 20489 | ret_tracking[1].short.register, |
| | 20490 | ret_tracking[2].short.register, |
| | 20491 | } }), |
| | 20492 | 4 => .init(.{ .register_quadruple = .{ |
| | 20493 | ret_tracking[0].short.register, |
| | 20494 | ret_tracking[1].short.register, |
| | 20495 | ret_tracking[2].short.register, |
| | 20496 | ret_tracking[3].short.register, |
| | 20497 | } }), |
| 19814 | }; | 20498 | }; |
| 19815 | } | 20499 | } |
| 19816 | | 20500 | |
| ... | @@ -19826,7 +20510,7 @@ fn resolveCallingConventionValues( | ... | @@ -19826,7 +20510,7 @@ fn resolveCallingConventionValues( |
| 19826 | else => unreachable, | 20510 | else => unreachable, |
| 19827 | } | 20511 | } |
| 19828 | | 20512 | |
| 19829 | var arg_mcv: [2]MCValue = undefined; | 20513 | var arg_mcv: [4]MCValue = undefined; |
| 19830 | var arg_mcv_i: usize = 0; | 20514 | var arg_mcv_i: usize = 0; |
| 19831 | | 20515 | |
| 19832 | const classes = switch (resolved_cc) { | 20516 | const classes = switch (resolved_cc) { |
| ... | @@ -19834,15 +20518,13 @@ fn resolveCallingConventionValues( | ... | @@ -19834,15 +20518,13 @@ fn resolveCallingConventionValues( |
| 19834 | .x86_64_win => &.{abi.classifyWindows(ty, zcu)}, | 20518 | .x86_64_win => &.{abi.classifyWindows(ty, zcu)}, |
| 19835 | else => unreachable, | 20519 | else => unreachable, |
| 19836 | }; | 20520 | }; |
| 19837 | for (classes) |class| switch (class) { | 20521 | classes: for (classes) |class| switch (class) { |
| 19838 | .integer => { | 20522 | .integer => { |
| 19839 | const param_int_regs = abi.getCAbiIntParamRegs(resolved_cc); | 20523 | const param_int_regs = abi.getCAbiIntParamRegs(resolved_cc); |
| 19840 | if (param_int_reg_i >= param_int_regs.len) break; | 20524 | if (param_int_reg_i >= param_int_regs.len) break; |
| 19841 | | 20525 | |
| 19842 | const param_int_reg = registerAlias( | 20526 | const param_int_reg = |
| 19843 | abi.getCAbiIntParamRegs(resolved_cc)[param_int_reg_i], | 20527 | registerAlias(param_int_regs[param_int_reg_i], @intCast(@min(ty.abiSize(zcu), 8))); |
| 19844 | @intCast(@min(ty.abiSize(zcu), 8)), | | |
| 19845 | ); | | |
| 19846 | param_int_reg_i += 1; | 20528 | param_int_reg_i += 1; |
| 19847 | | 20529 | |
| 19848 | arg_mcv[arg_mcv_i] = .{ .register = param_int_reg }; | 20530 | arg_mcv[arg_mcv_i] = .{ .register = param_int_reg }; |
| ... | @@ -19850,16 +20532,18 @@ fn resolveCallingConventionValues( | ... | @@ -19850,16 +20532,18 @@ fn resolveCallingConventionValues( |
| 19850 | }, | 20532 | }, |
| 19851 | .sse, .float, .float_combine => { | 20533 | .sse, .float, .float_combine => { |
| 19852 | const param_sse_regs = abi.getCAbiSseParamRegs(resolved_cc); | 20534 | const param_sse_regs = abi.getCAbiSseParamRegs(resolved_cc); |
| 19853 | if (param_sse_reg_i >= param_sse_regs.len) break; | 20535 | const abi_size: u32 = @intCast(ty.abiSize(zcu)); |
| | 20536 | const reg_size = @min(abi_size, self.vectorSize(.float)); |
| | 20537 | var byte_offset: u32 = 0; |
| | 20538 | while (byte_offset < abi_size) : (byte_offset += reg_size) { |
| | 20539 | if (param_sse_reg_i >= param_sse_regs.len) break :classes; |
| 19854 | | 20540 | |
| 19855 | const param_sse_reg = registerAlias( | 20541 | const param_sse_reg = registerAlias(param_sse_regs[param_sse_reg_i], reg_size); |
| 19856 | abi.getCAbiSseParamRegs(resolved_cc)[param_sse_reg_i], | 20542 | param_sse_reg_i += 1; |
| 19857 | @intCast(ty.abiSize(zcu)), | | |
| 19858 | ); | | |
| 19859 | param_sse_reg_i += 1; | | |
| 19860 | | 20543 | |
| 19861 | arg_mcv[arg_mcv_i] = .{ .register = param_sse_reg }; | 20544 | arg_mcv[arg_mcv_i] = .{ .register = param_sse_reg }; |
| 19862 | arg_mcv_i += 1; | 20545 | arg_mcv_i += 1; |
| | 20546 | } |
| 19863 | }, | 20547 | }, |
| 19864 | .sseup => assert(arg_mcv[arg_mcv_i - 1].register.class() == .sse), | 20548 | .sseup => assert(arg_mcv[arg_mcv_i - 1].register.class() == .sse), |
| 19865 | .x87, .x87up, .complex_x87, .memory, .win_i128 => switch (resolved_cc) { | 20549 | .x87, .x87up, .complex_x87, .memory, .win_i128 => switch (resolved_cc) { |
| ... | @@ -19908,7 +20592,21 @@ fn resolveCallingConventionValues( | ... | @@ -19908,7 +20592,21 @@ fn resolveCallingConventionValues( |
| 19908 | arg.* = switch (arg_mcv_i) { | 20592 | arg.* = switch (arg_mcv_i) { |
| 19909 | else => unreachable, | 20593 | else => unreachable, |
| 19910 | 1 => arg_mcv[0], | 20594 | 1 => arg_mcv[0], |
| 19911 | 2 => .{ .register_pair = .{ arg_mcv[0].register, arg_mcv[1].register } }, | 20595 | 2 => .{ .register_pair = .{ |
| | 20596 | arg_mcv[0].register, |
| | 20597 | arg_mcv[1].register, |
| | 20598 | } }, |
| | 20599 | 3 => .{ .register_triple = .{ |
| | 20600 | arg_mcv[0].register, |
| | 20601 | arg_mcv[1].register, |
| | 20602 | arg_mcv[2].register, |
| | 20603 | } }, |
| | 20604 | 4 => .{ .register_quadruple = .{ |
| | 20605 | arg_mcv[0].register, |
| | 20606 | arg_mcv[1].register, |
| | 20607 | arg_mcv[2].register, |
| | 20608 | arg_mcv[3].register, |
| | 20609 | } }, |
| 19912 | }; | 20610 | }; |
| 19913 | continue; | 20611 | continue; |
| 19914 | } | 20612 | } |
| ... | @@ -20052,32 +20750,40 @@ fn memSize(self: *CodeGen, ty: Type) Memory.Size { | ... | @@ -20052,32 +20750,40 @@ fn memSize(self: *CodeGen, ty: Type) Memory.Size { |
| 20052 | }; | 20750 | }; |
| 20053 | } | 20751 | } |
| 20054 | | 20752 | |
| 20055 | fn splitType(self: *CodeGen, ty: Type) ![2]Type { | 20753 | fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Type { |
| 20056 | const pt = self.pt; | 20754 | const pt = self.pt; |
| 20057 | const zcu = pt.zcu; | 20755 | const zcu = pt.zcu; |
| | 20756 | var parts: [parts_len]Type = undefined; |
| | 20757 | if (ty.isVector(zcu)) if (std.math.divExact(u32, ty.vectorLen(zcu), parts_len)) |vec_len| return .{ |
| | 20758 | try pt.vectorType(.{ .len = vec_len, .child = ty.scalarType(zcu).toIntern() }), |
| | 20759 | } ** parts_len else |err| switch (err) { |
| | 20760 | error.DivisionByZero => unreachable, |
| | 20761 | error.UnexpectedRemainder => {}, |
| | 20762 | }; |
| 20058 | const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none); | 20763 | const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none); |
| 20059 | var parts: [2]Type = undefined; | 20764 | if (classes.len == parts_len) for (&parts, classes, 0..) |*part, class, part_i| { |
| 20060 | if (classes.len == 2) for (&parts, classes, 0..) |*part, class, part_i| { | | |
| 20061 | part.* = switch (class) { | 20765 | part.* = switch (class) { |
| 20062 | .integer => switch (part_i) { | 20766 | .integer => if (part_i < parts_len - 1) |
| 20063 | 0 => Type.u64, | 20767 | Type.u64 |
| 20064 | 1 => part: { | 20768 | else part: { |
| 20065 | const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?; | 20769 | const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?; |
| 20066 | const elem_ty = try pt.intType(.unsigned, @intCast(elem_size * 8)); | 20770 | const elem_ty = try pt.intType(.unsigned, @intCast(elem_size * 8)); |
| 20067 | break :part switch (@divExact(ty.abiSize(zcu) - 8, elem_size)) { | 20771 | break :part switch (@divExact(ty.abiSize(zcu) - part_i * 8, elem_size)) { |
| 20068 | 1 => elem_ty, | 20772 | 1 => elem_ty, |
| 20069 | else => |len| try pt.arrayType(.{ .len = len, .child = elem_ty.toIntern() }), | 20773 | else => |array_len| try pt.arrayType(.{ .len = array_len, .child = elem_ty.toIntern() }), |
| 20070 | }; | 20774 | }; |
| 20071 | }, | | |
| 20072 | else => unreachable, | | |
| 20073 | }, | 20775 | }, |
| 20074 | .float => Type.f32, | 20776 | .float => Type.f32, |
| 20075 | .float_combine => try pt.arrayType(.{ .len = 2, .child = .f32_type }), | 20777 | .float_combine => try pt.arrayType(.{ .len = 2, .child = .f32_type }), |
| 20076 | .sse => Type.f64, | 20778 | .sse => Type.f64, |
| 20077 | else => break, | 20779 | else => break, |
| 20078 | }; | 20780 | }; |
| 20079 | } else if (parts[0].abiSize(zcu) + parts[1].abiSize(zcu) == ty.abiSize(zcu)) return parts; | 20781 | } else { |
| 20080 | return self.fail("TODO implement splitType for {}", .{ty.fmt(pt)}); | 20782 | var part_sizes: u64 = 0; |
| | 20783 | for (parts) |part| part_sizes += part.abiSize(zcu); |
| | 20784 | if (part_sizes == ty.abiSize(zcu)) return parts; |
| | 20785 | }; |
| | 20786 | return self.fail("TODO implement splitType({d}, {})", .{ parts_len, ty.fmt(pt) }); |
| 20081 | } | 20787 | } |
| 20082 | | 20788 | |
| 20083 | /// Truncates the value in the register in place. | 20789 | /// Truncates the value in the register in place. |
| ... | @@ -20297,6 +21003,7 @@ const Temp = struct { | ... | @@ -20297,6 +21003,7 @@ const Temp = struct { |
| 20297 | .immediate, | 21003 | .immediate, |
| 20298 | .eflags, | 21004 | .eflags, |
| 20299 | .register_offset, | 21005 | .register_offset, |
| | 21006 | .register_mask, |
| 20300 | .memory, | 21007 | .memory, |
| 20301 | .load_symbol, | 21008 | .load_symbol, |
| 20302 | .lea_symbol, | 21009 | .lea_symbol, |
| ... | @@ -20314,6 +21021,8 @@ const Temp = struct { | ... | @@ -20314,6 +21021,8 @@ const Temp = struct { |
| 20314 | => false, | 21021 | => false, |
| 20315 | .register, | 21022 | .register, |
| 20316 | .register_pair, | 21023 | .register_pair, |
| | 21024 | .register_triple, |
| | 21025 | .register_quadruple, |
| 20317 | .register_overflow, | 21026 | .register_overflow, |
| 20318 | => true, | 21027 | => true, |
| 20319 | .load_frame => |frame_addr| !frame_addr.index.isNamed(), | 21028 | .load_frame => |frame_addr| !frame_addr.index.isNamed(), |
| ... | @@ -20330,7 +21039,7 @@ const Temp = struct { | ... | @@ -20330,7 +21039,7 @@ const Temp = struct { |
| 20330 | cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize; | 21039 | cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize; |
| 20331 | cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); | 21040 | cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); |
| 20332 | switch (temp.tracking(cg).short) { | 21041 | switch (temp.tracking(cg).short) { |
| 20333 | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), | 21042 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20334 | .register => |reg| { | 21043 | .register => |reg| { |
| 20335 | const new_reg = | 21044 | const new_reg = |
| 20336 | try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); | 21045 | try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); |
| ... | @@ -20434,7 +21143,7 @@ const Temp = struct { | ... | @@ -20434,7 +21143,7 @@ const Temp = struct { |
| 20434 | const new_temp_index = cg.next_temp_index; | 21143 | const new_temp_index = cg.next_temp_index; |
| 20435 | cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize; | 21144 | cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize; |
| 20436 | switch (temp.tracking(cg).short) { | 21145 | switch (temp.tracking(cg).short) { |
| 20437 | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), | 21146 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20438 | .immediate => |imm| { | 21147 | .immediate => |imm| { |
| 20439 | assert(limb_index == 0); | 21148 | assert(limb_index == 0); |
| 20440 | new_temp_index.tracking(cg).* = .init(.{ .immediate = imm }); | 21149 | new_temp_index.tracking(cg).* = .init(.{ .immediate = imm }); |
| ... | @@ -20635,6 +21344,17 @@ const Temp = struct { | ... | @@ -20635,6 +21344,17 @@ const Temp = struct { |
| 20635 | first_temp.* = result_temp; | 21344 | first_temp.* = result_temp; |
| 20636 | } | 21345 | } |
| 20637 | | 21346 | |
| | 21347 | fn asMask(temp: Temp, kind: MaskKind, inverted: bool, scalar: Memory.Size, cg: *CodeGen) void { |
| | 21348 | assert(scalar != .none); |
| | 21349 | const mcv = &temp.unwrap(cg).temp.tracking(cg).short; |
| | 21350 | mcv.* = .{ .register_mask = .{ |
| | 21351 | .reg = mcv.register, |
| | 21352 | .kind = kind, |
| | 21353 | .inverted = inverted, |
| | 21354 | .scalar = scalar, |
| | 21355 | } }; |
| | 21356 | } |
| | 21357 | |
| 20638 | fn toLea(temp: *Temp, cg: *CodeGen) !bool { | 21358 | fn toLea(temp: *Temp, cg: *CodeGen) !bool { |
| 20639 | switch (temp.tracking(cg).short) { | 21359 | switch (temp.tracking(cg).short) { |
| 20640 | .none, | 21360 | .none, |
| ... | @@ -20643,7 +21363,10 @@ const Temp = struct { | ... | @@ -20643,7 +21363,10 @@ const Temp = struct { |
| 20643 | .undef, | 21363 | .undef, |
| 20644 | .eflags, | 21364 | .eflags, |
| 20645 | .register_pair, | 21365 | .register_pair, |
| | 21366 | .register_triple, |
| | 21367 | .register_quadruple, |
| 20646 | .register_overflow, | 21368 | .register_overflow, |
| | 21369 | .register_mask, |
| 20647 | .elementwise_regs_then_frame, | 21370 | .elementwise_regs_then_frame, |
| 20648 | .reserved_frame, | 21371 | .reserved_frame, |
| 20649 | .air_ref, | 21372 | .air_ref, |
| ... | @@ -20677,10 +21400,7 @@ const Temp = struct { | ... | @@ -20677,10 +21400,7 @@ const Temp = struct { |
| 20677 | | 21400 | |
| 20678 | fn toBase(temp: *Temp, cg: *CodeGen) !bool { | 21401 | fn toBase(temp: *Temp, cg: *CodeGen) !bool { |
| 20679 | const temp_tracking = temp.tracking(cg); | 21402 | const temp_tracking = temp.tracking(cg); |
| 20680 | switch (temp_tracking.short) { | 21403 | if (temp_tracking.short.isBase()) return false; |
| 20681 | else => {}, | | |
| 20682 | .indirect, .load_frame => return false, | | |
| 20683 | } | | |
| 20684 | const new_temp_index = cg.next_temp_index; | 21404 | const new_temp_index = cg.next_temp_index; |
| 20685 | cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg); | 21405 | cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg); |
| 20686 | const new_reg = | 21406 | const new_reg = |
| ... | @@ -20697,7 +21417,7 @@ const Temp = struct { | ... | @@ -20697,7 +21417,7 @@ const Temp = struct { |
| 20697 | const val_abi_size: u32 = @intCast(val_ty.abiSize(cg.pt.zcu)); | 21417 | const val_abi_size: u32 = @intCast(val_ty.abiSize(cg.pt.zcu)); |
| 20698 | const val = try cg.tempAlloc(val_ty); | 21418 | const val = try cg.tempAlloc(val_ty); |
| 20699 | switch (val.tracking(cg).short) { | 21419 | switch (val.tracking(cg).short) { |
| 20700 | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), | 21420 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20701 | .register => |val_reg| { | 21421 | .register => |val_reg| { |
| 20702 | while (try ptr.toLea(cg)) {} | 21422 | while (try ptr.toLea(cg)) {} |
| 20703 | switch (val_reg.class()) { | 21423 | switch (val_reg.class()) { |
| ... | @@ -20706,7 +21426,7 @@ const Temp = struct { | ... | @@ -20706,7 +21426,7 @@ const Temp = struct { |
| 20706 | registerAlias(val_reg, val_abi_size), | 21426 | registerAlias(val_reg, val_abi_size), |
| 20707 | try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }), | 21427 | try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }), |
| 20708 | ), | 21428 | ), |
| 20709 | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), | 21429 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20710 | } | 21430 | } |
| 20711 | }, | 21431 | }, |
| 20712 | .load_frame => |val_frame_addr| { | 21432 | .load_frame => |val_frame_addr| { |
| ... | @@ -20724,7 +21444,7 @@ const Temp = struct { | ... | @@ -20724,7 +21444,7 @@ const Temp = struct { |
| 20724 | const val_ty = val.typeOf(cg); | 21444 | const val_ty = val.typeOf(cg); |
| 20725 | const val_abi_size: u32 = @intCast(val_ty.abiSize(cg.pt.zcu)); | 21445 | const val_abi_size: u32 = @intCast(val_ty.abiSize(cg.pt.zcu)); |
| 20726 | val: switch (val.tracking(cg).short) { | 21446 | val: switch (val.tracking(cg).short) { |
| 20727 | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), | 21447 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20728 | .immediate => |imm| if (std.math.cast(i32, imm)) |s| { | 21448 | .immediate => |imm| if (std.math.cast(i32, imm)) |s| { |
| 20729 | while (try ptr.toLea(cg)) {} | 21449 | while (try ptr.toLea(cg)) {} |
| 20730 | try cg.asmMemoryImmediate( | 21450 | try cg.asmMemoryImmediate( |
| ... | @@ -20742,7 +21462,7 @@ const Temp = struct { | ... | @@ -20742,7 +21462,7 @@ const Temp = struct { |
| 20742 | try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }), | 21462 | try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }), |
| 20743 | registerAlias(val_reg, val_abi_size), | 21463 | registerAlias(val_reg, val_abi_size), |
| 20744 | ), | 21464 | ), |
| 20745 | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), | 21465 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20746 | } | 21466 | } |
| 20747 | }, | 21467 | }, |
| 20748 | } | 21468 | } |
| ... | @@ -20876,6 +21596,7 @@ fn reuseTemp( | ... | @@ -20876,6 +21596,7 @@ fn reuseTemp( |
| 20876 | .register_pair, | 21596 | .register_pair, |
| 20877 | .register_offset, | 21597 | .register_offset, |
| 20878 | .register_overflow, | 21598 | .register_overflow, |
| | 21599 | .register_mask, |
| 20879 | .indirect, | 21600 | .indirect, |
| 20880 | => for (tracking.short.getRegs()) |tracked_reg| { | 21601 | => for (tracking.short.getRegs()) |tracked_reg| { |
| 20881 | if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) |tracked_index| { | 21602 | if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) |tracked_index| { |
| ... | @@ -20913,6 +21634,26 @@ fn tempAllocReg(cg: *CodeGen, ty: Type, rc: RegisterManager.RegisterBitSet) !Tem | ... | @@ -20913,6 +21634,26 @@ fn tempAllocReg(cg: *CodeGen, ty: Type, rc: RegisterManager.RegisterBitSet) !Tem |
| 20913 | return .{ .index = temp_index.toIndex() }; | 21634 | return .{ .index = temp_index.toIndex() }; |
| 20914 | } | 21635 | } |
| 20915 | | 21636 | |
| | 21637 | fn tempAllocRegPair(cg: *CodeGen, ty: Type, rc: RegisterManager.RegisterBitSet) !Temp { |
| | 21638 | const temp_index = cg.next_temp_index; |
| | 21639 | temp_index.tracking(cg).* = .init( |
| | 21640 | .{ .register_pair = try cg.register_manager.allocRegs(2, temp_index.toIndex(), rc) }, |
| | 21641 | ); |
| | 21642 | cg.temp_type[@intFromEnum(temp_index)] = ty; |
| | 21643 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); |
| | 21644 | return .{ .index = temp_index.toIndex() }; |
| | 21645 | } |
| | 21646 | |
| | 21647 | fn tempAllocMem(cg: *CodeGen, ty: Type) !Temp { |
| | 21648 | const temp_index = cg.next_temp_index; |
| | 21649 | temp_index.tracking(cg).* = .init( |
| | 21650 | try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), false), |
| | 21651 | ); |
| | 21652 | cg.temp_type[@intFromEnum(temp_index)] = ty; |
| | 21653 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); |
| | 21654 | return .{ .index = temp_index.toIndex() }; |
| | 21655 | } |
| | 21656 | |
| 20916 | fn tempFromValue(cg: *CodeGen, ty: Type, value: MCValue) !Temp { | 21657 | fn tempFromValue(cg: *CodeGen, ty: Type, value: MCValue) !Temp { |
| 20917 | const temp_index = cg.next_temp_index; | 21658 | const temp_index = cg.next_temp_index; |
| 20918 | temp_index.tracking(cg).* = .init(value); | 21659 | temp_index.tracking(cg).* = .init(value); |
| ... | @@ -20993,30 +21734,23 @@ const Operand = union(enum) { | ... | @@ -20993,30 +21734,23 @@ const Operand = union(enum) { |
| 20993 | inst: Mir.Inst.Index, | 21734 | inst: Mir.Inst.Index, |
| 20994 | }; | 21735 | }; |
| 20995 | | 21736 | |
| 20996 | const SelectLoop = struct { | | |
| 20997 | element_reloc: Mir.Inst.Index, | | |
| 20998 | element_offset: union(enum) { | | |
| 20999 | unused, | | |
| 21000 | known: u31, | | |
| 21001 | temp: Temp, | | |
| 21002 | }, | | |
| 21003 | element_size: ?u13, | | |
| 21004 | limb_reloc: Mir.Inst.Index, | | |
| 21005 | limb_offset: union(enum) { | | |
| 21006 | unused, | | |
| 21007 | known: u31, | | |
| 21008 | temp: Temp, | | |
| 21009 | }, | | |
| 21010 | limb_size: ?u8, | | |
| 21011 | remaining_size: ?u64, | | |
| 21012 | }; | | |
| 21013 | | | |
| 21014 | const Pattern = struct { | 21737 | const Pattern = struct { |
| 21015 | ops: []const Op, | 21738 | ops: []const Op, |
| 21016 | commute: struct { u8, u8 } = .{ 0, 0 }, | 21739 | commute: struct { u8, u8 } = .{ 0, 0 }, |
| 21017 | | 21740 | |
| 21018 | const Set = struct { | 21741 | const Set = struct { |
| 21019 | required_features: []const std.Target.x86.Feature = &.{}, | 21742 | required_features: []const std.Target.x86.Feature = &.{}, |
| | 21743 | scalar: union(enum) { |
| | 21744 | any, |
| | 21745 | bool, |
| | 21746 | float: Memory.Size, |
| | 21747 | any_int: Memory.Size, |
| | 21748 | signed_int: Memory.Size, |
| | 21749 | unsigned_int: Memory.Size, |
| | 21750 | any_float_or_int: Memory.Size, |
| | 21751 | } = .any, |
| | 21752 | clobbers: struct { eflags: bool = false } = .{}, |
| | 21753 | invert_result: bool = false, |
| 21020 | loop: enum { | 21754 | loop: enum { |
| 21021 | /// only execute the instruction once | 21755 | /// only execute the instruction once |
| 21022 | once, | 21756 | once, |
| ... | @@ -21050,6 +21784,18 @@ const Pattern = struct { | ... | @@ -21050,6 +21784,18 @@ const Pattern = struct { |
| 21050 | xmm, | 21784 | xmm, |
| 21051 | /// any 256-bit sse register | 21785 | /// any 256-bit sse register |
| 21052 | ymm, | 21786 | ymm, |
| | 21787 | /// a 64-bit mmx register mask |
| | 21788 | mm_mask, |
| | 21789 | /// a 128-bit sse register mask |
| | 21790 | xmm_mask, |
| | 21791 | /// a 256-bit sse register mask |
| | 21792 | ymm_mask, |
| | 21793 | /// a 64-bit mmx register sign mask |
| | 21794 | mm_sign_mask, |
| | 21795 | /// a 128-bit sse register sign mask |
| | 21796 | xmm_sign_mask, |
| | 21797 | /// a 256-bit sse register sign mask |
| | 21798 | ymm_sign_mask, |
| 21053 | /// any memory | 21799 | /// any memory |
| 21054 | mem, | 21800 | mem, |
| 21055 | /// a limb stored in a gpr | 21801 | /// a limb stored in a gpr |
| ... | @@ -21062,41 +21808,82 @@ const Pattern = struct { | ... | @@ -21062,41 +21808,82 @@ const Pattern = struct { |
| 21062 | ymm_limb, | 21808 | ymm_limb, |
| 21063 | /// a limb stored in memory | 21809 | /// a limb stored in memory |
| 21064 | mem_limb, | 21810 | mem_limb, |
| | 21811 | /// a limb stored in a 64-bit mmx register mask |
| | 21812 | mm_mask_limb, |
| | 21813 | /// a limb stored in a 128-bit sse register masuk |
| | 21814 | xmm_mask_limb, |
| | 21815 | /// a limb stored in a 256-bit sse register masuk |
| | 21816 | ymm_mask_limb, |
| 21065 | /// specific immediate | 21817 | /// specific immediate |
| 21066 | imm: i8, | 21818 | imm: i8, |
| 21067 | /// any immediate signed extended from 32 bits | 21819 | /// any immediate signed extended from 32 bits |
| 21068 | simm32, | 21820 | simm32, |
| | 21821 | /// a temp general purpose register containing all ones |
| | 21822 | umax_gpr, |
| | 21823 | /// a temp 64-bit mmx register containing all ones |
| | 21824 | umax_mm, |
| | 21825 | /// a temp 128-bit sse register containing all ones |
| | 21826 | umax_xmm, |
| | 21827 | /// a temp 256-bit sse register containing all ones |
| | 21828 | umax_ymm, |
| 21069 | | 21829 | |
| 21070 | fn matches(op: Op, is_mut: bool, temp: Temp, cg: *CodeGen) bool { | 21830 | fn matches(op: Op, is_mut: bool, temp: Temp, cg: *CodeGen) bool { |
| 21071 | const abi_size = temp.typeOf(cg).abiSize(cg.pt.zcu); | 21831 | switch (op) { |
| 21072 | return switch (op) { | | |
| 21073 | .implicit, .explicit => unreachable, | 21832 | .implicit, .explicit => unreachable, |
| | 21833 | else => {}, |
| | 21834 | // temp is undefined |
| | 21835 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => return true, |
| | 21836 | } |
| | 21837 | const temp_ty = temp.typeOf(cg); |
| | 21838 | const abi_size = temp_ty.abiSize(cg.pt.zcu); |
| | 21839 | return switch (op) { |
| | 21840 | .implicit, .explicit, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, |
| 21074 | .gpr => abi_size <= 8 and switch (temp.tracking(cg).short) { | 21841 | .gpr => abi_size <= 8 and switch (temp.tracking(cg).short) { |
| 21075 | .register => |reg| reg.class() == .general_purpose, | 21842 | .register => |reg| reg.class() == .general_purpose, |
| 21076 | .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and | 21843 | .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and |
| 21077 | reg_off.off == 0, | 21844 | reg_off.off == 0, |
| 21078 | else => cg.regClassForType(temp.typeOf(cg)) == .general_purpose, | 21845 | else => cg.regClassForType(temp_ty) == .general_purpose, |
| 21079 | }, | 21846 | }, |
| 21080 | .mm => abi_size <= 8 and switch (temp.tracking(cg).short) { | 21847 | .mm, .mm_mask, .mm_sign_mask => abi_size <= 8 and switch (temp.tracking(cg).short) { |
| 21081 | .register => |reg| reg.class() == .mmx, | 21848 | .register => |reg| reg.class() == .mmx, |
| 21082 | .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0, | 21849 | .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0, |
| 21083 | else => cg.regClassForType(temp.typeOf(cg)) == .mmx, | 21850 | else => cg.regClassForType(temp_ty) == .mmx, |
| 21084 | }, | 21851 | }, |
| 21085 | .xmm => abi_size > 8 and abi_size <= 16 and switch (temp.tracking(cg).short) { | 21852 | .xmm, .xmm_mask, .xmm_sign_mask => abi_size > 8 and abi_size <= 16 and switch (temp.tracking(cg).short) { |
| 21086 | .register => |reg| reg.class() == .sse, | 21853 | .register => |reg| reg.class() == .sse, |
| 21087 | .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0, | 21854 | .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0, |
| 21088 | else => cg.regClassForType(temp.typeOf(cg)) == .sse, | 21855 | else => cg.regClassForType(temp_ty) == .sse, |
| 21089 | }, | 21856 | }, |
| 21090 | .ymm => abi_size > 16 and abi_size <= 32 and switch (temp.tracking(cg).short) { | 21857 | .ymm, .ymm_mask, .ymm_sign_mask => abi_size > 16 and abi_size <= 32 and switch (temp.tracking(cg).short) { |
| 21091 | .register => |reg| reg.class() == .sse, | 21858 | .register => |reg| reg.class() == .sse, |
| 21092 | .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0, | 21859 | .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0, |
| 21093 | else => cg.regClassForType(temp.typeOf(cg)) == .sse, | 21860 | else => cg.regClassForType(temp_ty) == .sse, |
| 21094 | }, | 21861 | }, |
| 21095 | .mem, .mem_limb => (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(), | 21862 | .mem, .mem_limb => (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(), |
| 21096 | .gpr_limb => abi_size > 8, | 21863 | .gpr_limb => abi_size > 8 and switch (temp.tracking(cg).short) { |
| 21097 | .mm_limb => abi_size > 8 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory() and cg.regClassForType(temp.typeOf(cg)) == .mmx, | 21864 | inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| { |
| 21098 | .xmm_limb => abi_size > 16 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(), | 21865 | if (reg.class() != .general_purpose) break false; |
| 21099 | .ymm_limb => abi_size > 32 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(), | 21866 | } else true, |
| | 21867 | else => |mcv| mcv.isMemory(), |
| | 21868 | }, |
| | 21869 | .mm_limb, .mm_mask_limb => abi_size > 8 and switch (temp.tracking(cg).short) { |
| | 21870 | inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| { |
| | 21871 | if (reg.class() != .mmx) break false; |
| | 21872 | } else true, |
| | 21873 | else => |mcv| mcv.isMemory() and cg.regClassForType(temp_ty) == .mmx, |
| | 21874 | }, |
| | 21875 | .xmm_limb, .xmm_mask_limb => abi_size > 16 and switch (temp.tracking(cg).short) { |
| | 21876 | inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| { |
| | 21877 | if (reg.class() != .sse) break false; |
| | 21878 | } else true, |
| | 21879 | else => |mcv| mcv.isMemory(), |
| | 21880 | }, |
| | 21881 | .ymm_limb, .ymm_mask_limb => abi_size > 32 and switch (temp.tracking(cg).short) { |
| | 21882 | inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| { |
| | 21883 | if (reg.class() != .sse) break false; |
| | 21884 | } else true, |
| | 21885 | else => |mcv| mcv.isMemory(), |
| | 21886 | }, |
| 21100 | .imm => |specific_imm| if (is_mut) unreachable else switch (temp.tracking(cg).short) { | 21887 | .imm => |specific_imm| if (is_mut) unreachable else switch (temp.tracking(cg).short) { |
| 21101 | .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm, | 21888 | .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm, |
| 21102 | else => false, | 21889 | else => false, |
| ... | @@ -21109,78 +21896,176 @@ const Pattern = struct { | ... | @@ -21109,78 +21896,176 @@ const Pattern = struct { |
| 21109 | } | 21896 | } |
| 21110 | }; | 21897 | }; |
| 21111 | }; | 21898 | }; |
| 21112 | fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_sets: []const Pattern.Set) !void { | 21899 | const SelectOptions = struct { |
| 21113 | var loop: SelectLoop = .{ | 21900 | invert_result: bool = false, |
| | 21901 | }; |
| | 21902 | fn select( |
| | 21903 | cg: *CodeGen, |
| | 21904 | dst_temps: []Temp, |
| | 21905 | dst_tys: []const Type, |
| | 21906 | src_temps: []Temp, |
| | 21907 | pattern_sets: []const Pattern.Set, |
| | 21908 | opts: SelectOptions, |
| | 21909 | ) !void { |
| | 21910 | var loop: struct { |
| | 21911 | element_reloc: Mir.Inst.Index, |
| | 21912 | element_offset: Offset, |
| | 21913 | element_size: ?u13, |
| | 21914 | limb_reloc: Mir.Inst.Index, |
| | 21915 | limb_offset: Offset, |
| | 21916 | limb_size: ?u8, |
| | 21917 | mask_limb_temp: ?Temp, |
| | 21918 | mask_limb_offset: Offset, |
| | 21919 | mask_limb_offset_lock: ?RegisterLock, |
| | 21920 | mask_limb_bit_size: ?u7, |
| | 21921 | mask_store_temp: ?Temp, |
| | 21922 | mask_store_reg: ?Register, |
| | 21923 | mask_store_bit_size: ?u7, |
| | 21924 | remaining_size: ?u64, |
| | 21925 | |
| | 21926 | const Offset = union(enum) { |
| | 21927 | unused, |
| | 21928 | known: u31, |
| | 21929 | temp: Temp, |
| | 21930 | }; |
| | 21931 | } = .{ |
| 21114 | .element_reloc = undefined, | 21932 | .element_reloc = undefined, |
| 21115 | .element_offset = .unused, | 21933 | .element_offset = .unused, |
| 21116 | .element_size = null, | 21934 | .element_size = null, |
| 21117 | .limb_reloc = undefined, | 21935 | .limb_reloc = undefined, |
| 21118 | .limb_offset = .unused, | 21936 | .limb_offset = .unused, |
| 21119 | .limb_size = null, | 21937 | .limb_size = null, |
| | 21938 | .mask_limb_temp = null, |
| | 21939 | .mask_limb_offset = .unused, |
| | 21940 | .mask_limb_offset_lock = null, |
| | 21941 | .mask_limb_bit_size = null, |
| | 21942 | .mask_store_temp = null, |
| | 21943 | .mask_store_reg = null, |
| | 21944 | .mask_store_bit_size = null, |
| 21120 | .remaining_size = null, | 21945 | .remaining_size = null, |
| 21121 | }; | 21946 | }; |
| 21122 | var extra_temps: [4]?Temp = @splat(null); | 21947 | var extra_temps: [4]?Temp = @splat(null); |
| 21123 | pattern_sets: for (pattern_sets) |pattern_set| { | 21948 | pattern_sets: for (pattern_sets) |pattern_set| { |
| 21124 | for (pattern_set.required_features) |required_feature| if (!cg.hasFeature(required_feature)) continue :pattern_sets; | 21949 | for (pattern_set.required_features) |required_feature| if (!cg.hasFeature(required_feature)) continue :pattern_sets; |
| | 21950 | for (src_temps) |src_temp| switch (pattern_set.scalar) { |
| | 21951 | .any => {}, |
| | 21952 | .bool => if (src_temp.typeOf(cg).scalarType(cg.pt.zcu).toIntern() != .bool_type) continue :pattern_sets, |
| | 21953 | .float => |size| { |
| | 21954 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); |
| | 21955 | if (!scalar_ty.isRuntimeFloat()) continue :pattern_sets; |
| | 21956 | if (scalar_ty.floatBits(cg.target.*) != size.bitSize()) continue :pattern_sets; |
| | 21957 | }, |
| | 21958 | .any_int => |size| { |
| | 21959 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); |
| | 21960 | if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets; |
| | 21961 | if (scalar_ty.intInfo(cg.pt.zcu).bits > size.bitSize()) continue :pattern_sets; |
| | 21962 | }, |
| | 21963 | .signed_int => |size| { |
| | 21964 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); |
| | 21965 | if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets; |
| | 21966 | const scalar_info = scalar_ty.intInfo(cg.pt.zcu); |
| | 21967 | if (scalar_info.signedness != .signed) continue :pattern_sets; |
| | 21968 | if (scalar_info.bits > size.bitSize()) continue :pattern_sets; |
| | 21969 | }, |
| | 21970 | .unsigned_int => |size| { |
| | 21971 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); |
| | 21972 | if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets; |
| | 21973 | const scalar_info = scalar_ty.intInfo(cg.pt.zcu); |
| | 21974 | if (scalar_info.signedness != .unsigned) continue :pattern_sets; |
| | 21975 | if (scalar_info.bits > size.bitSize()) continue :pattern_sets; |
| | 21976 | }, |
| | 21977 | .any_float_or_int => |size| { |
| | 21978 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); |
| | 21979 | if (scalar_ty.isRuntimeFloat()) { |
| | 21980 | if (scalar_ty.floatBits(cg.target.*) != size.bitSize()) continue :pattern_sets; |
| | 21981 | } else if (scalar_ty.isAbiInt(cg.pt.zcu)) { |
| | 21982 | if (scalar_ty.intInfo(cg.pt.zcu).bits > size.bitSize()) continue :pattern_sets; |
| | 21983 | } else continue :pattern_sets; |
| | 21984 | }, |
| | 21985 | }; |
| 21125 | patterns: for (pattern_set.patterns) |pattern| { | 21986 | patterns: for (pattern_set.patterns) |pattern| { |
| 21126 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { | 21987 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { |
| 21127 | const ref_src_op, const is_mut = switch (src_op) { | 21988 | const ref_src_op, const is_mut = switch (src_op) { |
| 21128 | .implicit, .explicit => |op_index| .{ pattern.ops[op_index], true }, | 21989 | .implicit, .explicit => |linked_index| .{ pattern.ops[linked_index], true }, |
| 21129 | else => .{ src_op, false }, | 21990 | else => .{ src_op, false }, |
| 21130 | }; | 21991 | }; |
| 21131 | if (!ref_src_op.matches(is_mut, src_temp.*, cg)) continue :patterns; | 21992 | if (!ref_src_op.matches(is_mut, src_temp, cg)) continue :patterns; |
| 21132 | } | 21993 | } |
| 21133 | while (true) for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { | 21994 | |
| 21134 | if (switch (switch (src_op) { | 21995 | while (true) for (src_temps, pattern.ops[dst_temps.len..]) |*src_temp, src_op| { |
| 21135 | .implicit, .explicit => |op_index| pattern.ops[op_index], | 21996 | if (changed: switch (switch (src_op) { |
| | 21997 | .implicit, .explicit => |linked_index| pattern.ops[linked_index], |
| 21136 | else => src_op, | 21998 | else => src_op, |
| 21137 | }) { | 21999 | }) { |
| 21138 | .implicit, .explicit => unreachable, | 22000 | .implicit, .explicit => unreachable, |
| 21139 | .gpr => try src_temp.toRegClass(.general_purpose, cg), | 22001 | .gpr => try src_temp.toRegClass(.general_purpose, cg), |
| 21140 | .mm => try src_temp.toRegClass(.mmx, cg), | 22002 | .mm, .mm_mask, .mm_sign_mask => try src_temp.toRegClass(.mmx, cg), |
| 21141 | .xmm, .ymm => try src_temp.toRegClass(.sse, cg), | 22003 | .xmm, |
| 21142 | .mem, .imm, .simm32 => false, | 22004 | .ymm, |
| 21143 | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb, .mem_limb => switch (src_temp.tracking(cg).short) { | 22005 | .xmm_mask, |
| 21144 | .register_pair => false, | 22006 | .ymm_mask, |
| | 22007 | .xmm_sign_mask, |
| | 22008 | .ymm_sign_mask, |
| | 22009 | => try src_temp.toRegClass(.sse, cg), |
| | 22010 | .mem => try src_temp.toBase(cg), |
| | 22011 | .imm, .simm32 => false, |
| | 22012 | .gpr_limb, |
| | 22013 | .mm_limb, |
| | 22014 | .xmm_limb, |
| | 22015 | .ymm_limb, |
| | 22016 | .mem_limb, |
| | 22017 | => switch (src_temp.tracking(cg).short) { |
| | 22018 | .register_pair, .register_triple, .register_quadruple => false, |
| 21145 | else => try src_temp.toBase(cg), | 22019 | else => try src_temp.toBase(cg), |
| 21146 | }, | 22020 | }, |
| | 22021 | .mm_mask_limb, |
| | 22022 | .xmm_mask_limb, |
| | 22023 | .ymm_mask_limb, |
| | 22024 | => if (!cg.hasFeature(.bmi2) and !cg.register_manager.isKnownRegFree(.rcx)) { |
| | 22025 | try cg.register_manager.getKnownReg(.rcx, null); |
| | 22026 | loop.mask_limb_offset_lock = cg.register_manager.lockKnownRegAssumeUnused(.rcx); |
| | 22027 | break :changed true; |
| | 22028 | } else false, |
| | 22029 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => false, |
| 21147 | }) break; | 22030 | }) break; |
| 21148 | } else break; | 22031 | } else break; |
| | 22032 | |
| | 22033 | var dst_is_linked: std.StaticBitSet(4) = .initEmpty(); |
| 21149 | var mir_ops_len = dst_temps.len; | 22034 | var mir_ops_len = dst_temps.len; |
| 21150 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { | 22035 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { |
| 21151 | const ref_src_op, const extra_temp = op: switch (src_op) { | 22036 | defer mir_ops_len += @intFromBool(src_op != .implicit); |
| 21152 | .implicit => |op_index| { | 22037 | const linked_src_op, const extra_temp = op: switch (src_op) { |
| 21153 | dst_temps[op_index] = if (src_temp.isMut(cg)) | 22038 | .implicit, .explicit => |linked_index| { |
| 21154 | src_temp.* | 22039 | if (src_temp.isMut(cg)) { |
| 21155 | else | 22040 | dst_temps[linked_index] = src_temp; |
| 21156 | try cg.tempAlloc(src_temp.typeOf(cg)); | 22041 | dst_is_linked.set(linked_index); |
| 21157 | break :op .{ pattern.ops[op_index], &extra_temps[op_index] }; | 22042 | } |
| 21158 | }, | 22043 | break :op .{ pattern.ops[linked_index], &extra_temps[linked_index] }; |
| 21159 | .explicit => |op_index| { | | |
| 21160 | dst_temps[op_index] = if (src_temp.isMut(cg)) | | |
| 21161 | src_temp.* | | |
| 21162 | else | | |
| 21163 | try cg.tempAlloc(src_temp.typeOf(cg)); | | |
| 21164 | defer mir_ops_len += 1; | | |
| 21165 | break :op .{ pattern.ops[op_index], &extra_temps[mir_ops_len] }; | | |
| 21166 | }, | | |
| 21167 | else => { | | |
| 21168 | defer mir_ops_len += 1; | | |
| 21169 | break :op .{ src_op, &extra_temps[mir_ops_len] }; | | |
| 21170 | }, | 22044 | }, |
| | 22045 | else => .{ src_op, &extra_temps[mir_ops_len] }, |
| 21171 | }; | 22046 | }; |
| 21172 | const limb_size: u8, const rc = switch (ref_src_op) { | 22047 | const limb_size: u8, const rc = switch (linked_src_op) { |
| 21173 | else => continue, | 22048 | else => continue, |
| 21174 | .gpr_limb => .{ 8, abi.RegisterClass.gp }, | 22049 | .gpr_limb => .{ 8, abi.RegisterClass.gp }, |
| 21175 | .mm_limb => .{ 8, @panic("TODO") }, | 22050 | .mm_limb, .mm_mask_limb => .{ 8, @panic("TODO") }, |
| 21176 | .xmm_limb => .{ 16, abi.RegisterClass.sse }, | 22051 | .xmm_limb, .xmm_mask_limb => .{ 16, abi.RegisterClass.sse }, |
| 21177 | .ymm_limb => .{ 32, abi.RegisterClass.sse }, | 22052 | .ymm_limb, .ymm_mask_limb => .{ 32, abi.RegisterClass.sse }, |
| | 22053 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => { |
| | 22054 | assert(extra_temp.* == null); |
| | 22055 | extra_temp.* = try cg.tempAllocReg(Type.usize, switch (linked_src_op) { |
| | 22056 | else => unreachable, |
| | 22057 | .umax_gpr => abi.RegisterClass.gp, |
| | 22058 | .umax_mm => @panic("TODO"), |
| | 22059 | .umax_xmm, .umax_ymm => abi.RegisterClass.sse, |
| | 22060 | }); |
| | 22061 | continue; |
| | 22062 | }, |
| 21178 | }; | 22063 | }; |
| 21179 | assert(loop.limb_size == null or loop.limb_size == limb_size); | 22064 | assert(loop.limb_size == null or loop.limb_size == limb_size); |
| 21180 | loop.limb_size = limb_size; | 22065 | loop.limb_size = limb_size; |
| 21181 | loop.remaining_size = loop.remaining_size orelse src_temp.typeOf(cg).abiSize(cg.pt.zcu); | 22066 | loop.remaining_size = loop.remaining_size orelse src_temp.typeOf(cg).abiSize(cg.pt.zcu); |
| 21182 | switch (src_temp.tracking(cg).short) { | 22067 | switch (src_temp.tracking(cg).short) { |
| 21183 | .register_pair => switch (loop.limb_offset) { | 22068 | .register_pair, .register_triple, .register_quadruple => switch (loop.limb_offset) { |
| 21184 | .unused, .temp => loop.limb_offset = .{ .known = 0 }, | 22069 | .unused, .temp => loop.limb_offset = .{ .known = 0 }, |
| 21185 | .known => {}, | 22070 | .known => {}, |
| 21186 | }, | 22071 | }, |
| ... | @@ -21189,25 +22074,130 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set | ... | @@ -21189,25 +22074,130 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21189 | .unused => loop.limb_offset = .{ .temp = undefined }, | 22074 | .unused => loop.limb_offset = .{ .temp = undefined }, |
| 21190 | .known, .temp => {}, | 22075 | .known, .temp => {}, |
| 21191 | } | 22076 | } |
| | 22077 | assert(extra_temp.* == null); |
| 21192 | extra_temp.* = try cg.tempAllocReg(Type.usize, rc); | 22078 | extra_temp.* = try cg.tempAllocReg(Type.usize, rc); |
| 21193 | }, | 22079 | }, |
| 21194 | } | 22080 | } |
| 21195 | } | 22081 | } |
| | 22082 | for ( |
| | 22083 | 0.., |
| | 22084 | dst_temps, |
| | 22085 | pattern.ops[0..dst_temps.len], |
| | 22086 | dst_tys, |
| | 22087 | extra_temps[0..dst_temps.len], |
| | 22088 | ) |dst_index, *dst_temp, dst_op, dst_ty, *extra_temp| switch (dst_op) { |
| | 22089 | else => if (!dst_is_linked.isSet(dst_index)) { |
| | 22090 | dst_temp.* = dst_temp: switch (dst_op) { |
| | 22091 | .implicit => unreachable, |
| | 22092 | .explicit => |linked_index| dst_temps[linked_index], |
| | 22093 | .gpr => try cg.tempAllocReg(dst_ty, abi.RegisterClass.gp), |
| | 22094 | .mm, .mm_mask, .mm_sign_mask => @panic("TODO"), |
| | 22095 | .xmm, .xmm_mask, .xmm_sign_mask => try cg.tempAllocReg(dst_ty, abi.RegisterClass.sse), |
| | 22096 | .ymm, .ymm_mask, .ymm_sign_mask => try cg.tempAllocReg(dst_ty, abi.RegisterClass.sse), |
| | 22097 | .mem => @panic("TODO"), |
| | 22098 | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => { |
| | 22099 | if (extra_temp.* == null) extra_temp.* = try cg.tempAllocReg(Type.usize, switch (dst_op) { |
| | 22100 | else => unreachable, |
| | 22101 | .gpr_limb => abi.RegisterClass.gp, |
| | 22102 | .mm_limb => @panic("TODO"), |
| | 22103 | .xmm_limb, .ymm_limb => abi.RegisterClass.sse, |
| | 22104 | }); |
| | 22105 | break :dst_temp try cg.tempAlloc(dst_ty); |
| | 22106 | }, |
| | 22107 | .mem_limb => try cg.tempAlloc(dst_ty), |
| | 22108 | .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => unreachable, // already checked |
| | 22109 | .imm, .simm32, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, // unmodifiable destination |
| | 22110 | }; |
| | 22111 | }, |
| | 22112 | .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => { |
| | 22113 | const scalar_size = @divExact(switch (pattern_set.scalar) { |
| | 22114 | .any, .bool => unreachable, |
| | 22115 | .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size, |
| | 22116 | }.bitSize(), 8); |
| | 22117 | const mask_bit_size = @divExact(loop.remaining_size.?, scalar_size); |
| | 22118 | const mask_limb_bit_size: u7 = @intCast(@divExact(loop.limb_size.?, scalar_size)); |
| | 22119 | assert(loop.mask_limb_bit_size == null or loop.mask_limb_bit_size == mask_limb_bit_size); |
| | 22120 | loop.mask_limb_bit_size = mask_limb_bit_size; |
| | 22121 | const mask_store_bit_size = mask_store_bit_size: { |
| | 22122 | // Try to match limb size so that no shifting will be needed. |
| | 22123 | if (mask_limb_bit_size % 8 == 0) break :mask_store_bit_size mask_limb_bit_size; |
| | 22124 | // If abi size <= 8 the entire value can be stored at once, |
| | 22125 | // enabling store forwarding and minimizing store buffer usage. |
| | 22126 | // Otherwise, we will be performing shifts that need to wrap at |
| | 22127 | // store size, which for x86 requires 32 or 64, so just pick 64 |
| | 22128 | // for the same reasons as above. |
| | 22129 | break :mask_store_bit_size @min(mask_bit_size, 64); |
| | 22130 | }; |
| | 22131 | assert(loop.mask_store_bit_size == null or loop.mask_store_bit_size == mask_store_bit_size); |
| | 22132 | loop.mask_store_bit_size = mask_store_bit_size; |
| | 22133 | loop.mask_limb_offset = loop.limb_offset; |
| | 22134 | if (extra_temp.* == null) extra_temp.* = try cg.tempAllocReg(Type.usize, switch (dst_op) { |
| | 22135 | else => unreachable, |
| | 22136 | .mm_mask_limb => @panic("TODO"), |
| | 22137 | .xmm_mask_limb, .ymm_mask_limb => abi.RegisterClass.sse, |
| | 22138 | }); |
| | 22139 | if (loop.mask_limb_temp == null) loop.mask_limb_temp = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); |
| | 22140 | if (mask_limb_bit_size < mask_store_bit_size and loop.mask_store_reg == null) { |
| | 22141 | loop.mask_store_temp = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); |
| | 22142 | loop.mask_store_reg = loop.mask_store_temp.?.tracking(cg).short.register; |
| | 22143 | } |
| | 22144 | dst_temp.* = if (mask_store_bit_size < mask_bit_size) |
| | 22145 | try cg.tempAllocMem(dst_ty) |
| | 22146 | else if (loop.mask_store_temp) |mask_store_temp| dst_temp: { |
| | 22147 | loop.mask_store_temp = null; |
| | 22148 | break :dst_temp mask_store_temp; |
| | 22149 | } else try cg.tempAlloc(dst_ty); |
| | 22150 | }, |
| | 22151 | }; |
| | 22152 | switch (loop.mask_limb_offset) { |
| | 22153 | .unused, .known => {}, |
| | 22154 | .temp => |*mask_limb_offset| { |
| | 22155 | if (cg.hasFeature(.bmi2)) { |
| | 22156 | assert(loop.mask_limb_offset_lock == null); |
| | 22157 | mask_limb_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); |
| | 22158 | } else { |
| | 22159 | if (loop.mask_limb_offset_lock) |lock| cg.register_manager.unlockReg(lock); |
| | 22160 | loop.mask_limb_offset_lock = null; |
| | 22161 | mask_limb_offset.* = try cg.tempFromValue(Type.usize, .{ .register = .rcx }); |
| | 22162 | } |
| | 22163 | if (loop.mask_store_reg) |mask_store_reg| { |
| | 22164 | const mask_store_alias = registerAlias( |
| | 22165 | mask_store_reg, |
| | 22166 | @min(std.math.divCeil(u7, loop.mask_store_bit_size.?, 8) catch unreachable, 4), |
| | 22167 | ); |
| | 22168 | try cg.spillEflagsIfOccupied(); |
| | 22169 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias); |
| | 22170 | } |
| | 22171 | }, |
| | 22172 | } |
| 21196 | switch (loop.element_offset) { | 22173 | switch (loop.element_offset) { |
| 21197 | .unused, .known => {}, | 22174 | .unused, .known => {}, |
| 21198 | .temp => |*element_offset| { | 22175 | .temp => |*element_offset| { |
| 21199 | element_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); | 22176 | element_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); |
| 21200 | const element_offset_reg = element_offset.tracking(cg).short.register; | 22177 | const element_offset_reg = element_offset.tracking(cg).short.register; |
| | 22178 | try cg.spillEflagsIfOccupied(); |
| 21201 | try cg.asmRegisterRegister(.{ ._, .xor }, element_offset_reg.to32(), element_offset_reg.to32()); | 22179 | try cg.asmRegisterRegister(.{ ._, .xor }, element_offset_reg.to32(), element_offset_reg.to32()); |
| 21202 | loop.element_reloc = @intCast(cg.mir_instructions.len); | 22180 | loop.element_reloc = @intCast(cg.mir_instructions.len); |
| 21203 | }, | 22181 | }, |
| 21204 | } | 22182 | } |
| | 22183 | switch (loop.limb_offset) { |
| | 22184 | .unused, .known => {}, |
| | 22185 | .temp => |*limb_offset| limb_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp), |
| | 22186 | } |
| 21205 | while (true) { | 22187 | while (true) { |
| | 22188 | switch (loop.mask_limb_offset) { |
| | 22189 | .unused, .known => {}, |
| | 22190 | .temp => |mask_limb_offset| { |
| | 22191 | const mask_limb_offset_reg = mask_limb_offset.tracking(cg).short.register.to32(); |
| | 22192 | try cg.spillEflagsIfOccupied(); |
| | 22193 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_limb_offset_reg, mask_limb_offset_reg); |
| | 22194 | }, |
| | 22195 | } |
| 21206 | switch (loop.limb_offset) { | 22196 | switch (loop.limb_offset) { |
| 21207 | .unused, .known => {}, | 22197 | .unused, .known => {}, |
| 21208 | .temp => |*limb_offset| { | 22198 | .temp => |limb_offset| { |
| 21209 | limb_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); | 22199 | const limb_offset_reg = limb_offset.tracking(cg).short.register.to32(); |
| 21210 | const limb_offset_reg = limb_offset.tracking(cg).short.register; | 22200 | try cg.spillEflagsIfOccupied(); |
| 21211 | try cg.asmRegisterRegister(.{ ._, .xor }, limb_offset_reg.to32(), limb_offset_reg.to32()); | 22201 | try cg.asmRegisterRegister(.{ ._, .xor }, limb_offset_reg.to32(), limb_offset_reg.to32()); |
| 21212 | loop.limb_reloc = @intCast(cg.mir_instructions.len); | 22202 | loop.limb_reloc = @intCast(cg.mir_instructions.len); |
| 21213 | }, | 22203 | }, |
| ... | @@ -21216,56 +22206,67 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set | ... | @@ -21216,56 +22206,67 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21216 | var mir_ops: [4]Operand = @splat(.none); | 22206 | var mir_ops: [4]Operand = @splat(.none); |
| 21217 | mir_ops_len = dst_temps.len; | 22207 | mir_ops_len = dst_temps.len; |
| 21218 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { | 22208 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { |
| 21219 | const mir_op, const ref_src_op, const extra_temp = op: switch (src_op) { | 22209 | defer mir_ops_len += @intFromBool(src_op != .implicit); |
| 21220 | .implicit => |op_index| .{ &mir_ops[op_index], pattern.ops[op_index], &extra_temps[op_index] }, | 22210 | const mir_op, const linked_src_op, const extra_temp = switch (src_op) { |
| 21221 | .explicit => |op_index| { | 22211 | .implicit => |linked_index| .{ &mir_ops[linked_index], pattern.ops[linked_index], extra_temps[linked_index] }, |
| 21222 | defer mir_ops_len += 1; | 22212 | .explicit => |linked_index| .{ &mir_ops[mir_ops_len], pattern.ops[linked_index], extra_temps[linked_index] }, |
| 21223 | break :op .{ &mir_ops[mir_ops_len], pattern.ops[op_index], &extra_temps[mir_ops_len] }; | 22213 | else => .{ &mir_ops[mir_ops_len], src_op, extra_temps[mir_ops_len] }, |
| 21224 | }, | | |
| 21225 | else => { | | |
| 21226 | defer mir_ops_len += 1; | | |
| 21227 | break :op .{ &mir_ops[mir_ops_len], src_op, &extra_temps[mir_ops_len] }; | | |
| 21228 | }, | | |
| 21229 | }; | 22214 | }; |
| 21230 | const src_mcv = src_temp.tracking(cg).short; | 22215 | const src_mcv = switch (linked_src_op) { |
| 21231 | switch (ref_src_op) { | 22216 | else => src_temp, |
| 21232 | else => {}, | 22217 | // src_temp is undefined |
| 21233 | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (src_mcv) { | 22218 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => extra_temp.?, |
| 21234 | .register_pair => {}, | 22219 | }.tracking(cg).short; |
| 21235 | else => try cg.asmRegisterMemory( | 22220 | copy_limb: switch (src_mcv) { |
| 21236 | switch (ref_src_op) { | 22221 | .register_pair, .register_triple, .register_quadruple => {}, |
| 21237 | else => unreachable, | 22222 | else => try cg.asmRegisterMemory( |
| 21238 | .gpr_limb => .{ ._, .mov }, | 22223 | switch (linked_src_op) { |
| 21239 | .mm_limb => .{ ._q, .mov }, | 22224 | else => break :copy_limb, |
| 21240 | .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu }, | 22225 | .gpr_limb => .{ ._, .mov }, |
| | 22226 | .mm_limb, .mm_mask_limb => .{ ._q, .mov }, |
| | 22227 | .xmm_limb, |
| | 22228 | .ymm_limb, |
| | 22229 | .xmm_mask_limb, |
| | 22230 | .ymm_mask_limb, |
| | 22231 | => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu }, |
| | 22232 | }, |
| | 22233 | registerAlias(extra_temp.?.tracking(cg).short.register, loop.limb_size.?), |
| | 22234 | try src_mcv.mem(cg, switch (loop.limb_offset) { |
| | 22235 | .unused => unreachable, |
| | 22236 | .known => |limb_offset| .{ |
| | 22237 | .size = .fromSize(loop.limb_size.?), |
| | 22238 | .disp = limb_offset, |
| 21241 | }, | 22239 | }, |
| 21242 | registerAlias(extra_temp.*.?.tracking(cg).short.register, loop.limb_size.?), | 22240 | .temp => |limb_offset| .{ |
| 21243 | try src_mcv.mem(cg, switch (loop.limb_offset) { | 22241 | .size = .fromSize(loop.limb_size.?), |
| 21244 | .unused => unreachable, | 22242 | .index = limb_offset.tracking(cg).short.register.to64(), |
| 21245 | .known => |limb_offset| .{ | 22243 | }, |
| 21246 | .size = .fromSize(loop.limb_size.?), | 22244 | }), |
| 21247 | .disp = limb_offset, | 22245 | ), |
| 21248 | }, | | |
| 21249 | .temp => |limb_offset| .{ | | |
| 21250 | .size = .fromSize(loop.limb_size.?), | | |
| 21251 | .index = limb_offset.tracking(cg).short.register.to64(), | | |
| 21252 | }, | | |
| 21253 | }), | | |
| 21254 | ), | | |
| 21255 | }, | | |
| 21256 | } | 22246 | } |
| 21257 | mir_op.* = switch (ref_src_op) { | 22247 | mir_op.* = switch (linked_src_op) { |
| 21258 | .implicit, .explicit => unreachable, | 22248 | .implicit, .explicit => unreachable, |
| 21259 | .gpr => .{ .reg = registerAlias( | 22249 | .gpr => .{ .reg = registerAlias( |
| 21260 | src_mcv.register, | 22250 | src_mcv.register, |
| 21261 | @intCast(src_temp.typeOf(cg).abiSize(cg.pt.zcu)), | 22251 | @intCast(src_temp.typeOf(cg).abiSize(cg.pt.zcu)), |
| 21262 | ) }, | 22252 | ) }, |
| 21263 | .mm => .{ .reg = src_mcv.register }, | 22253 | .umax_gpr => .{ .reg = src_mcv.register.to64() }, // TODO: use other op size? |
| 21264 | .xmm => .{ .reg = src_mcv.register.to128() }, | 22254 | .mm, .mm_mask, .mm_sign_mask, .umax_mm => .{ .reg = src_mcv.register }, |
| 21265 | .ymm => .{ .reg = src_mcv.register.to256() }, | 22255 | .xmm, .xmm_mask, .xmm_sign_mask, .umax_xmm => .{ .reg = src_mcv.register.to128() }, |
| | 22256 | .ymm, .ymm_mask, .ymm_sign_mask, .umax_ymm => .{ .reg = src_mcv.register.to256() }, |
| 21266 | .mem => .{ .mem = try src_mcv.mem(cg, .{ .size = cg.memSize(src_temp.typeOf(cg)) }) }, | 22257 | .mem => .{ .mem = try src_mcv.mem(cg, .{ .size = cg.memSize(src_temp.typeOf(cg)) }) }, |
| 21267 | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (src_mcv) { | 22258 | .gpr_limb, |
| 21268 | .register_pair => |src_regs| switch (loop.limb_offset) { | 22259 | .mm_limb, |
| | 22260 | .xmm_limb, |
| | 22261 | .ymm_limb, |
| | 22262 | .mm_mask_limb, |
| | 22263 | .xmm_mask_limb, |
| | 22264 | .ymm_mask_limb, |
| | 22265 | => switch (src_mcv) { |
| | 22266 | inline .register_pair, |
| | 22267 | .register_triple, |
| | 22268 | .register_quadruple, |
| | 22269 | => |src_regs| switch (loop.limb_offset) { |
| 21269 | .unused => unreachable, | 22270 | .unused => unreachable, |
| 21270 | .known => |limb_offset| .{ .reg = registerAlias( | 22271 | .known => |limb_offset| .{ .reg = registerAlias( |
| 21271 | src_regs[@divExact(limb_offset, loop.limb_size.?)], | 22272 | src_regs[@divExact(limb_offset, loop.limb_size.?)], |
| ... | @@ -21274,12 +22275,12 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set | ... | @@ -21274,12 +22275,12 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21274 | .temp => unreachable, | 22275 | .temp => unreachable, |
| 21275 | }, | 22276 | }, |
| 21276 | else => .{ .reg = registerAlias( | 22277 | else => .{ .reg = registerAlias( |
| 21277 | extra_temp.*.?.tracking(cg).short.register, | 22278 | extra_temp.?.tracking(cg).short.register, |
| 21278 | loop.limb_size.?, | 22279 | loop.limb_size.?, |
| 21279 | ) }, | 22280 | ) }, |
| 21280 | }, | 22281 | }, |
| 21281 | .mem_limb => .{ .mem = switch (src_mcv) { | 22282 | .mem_limb => .{ .mem = switch (src_mcv) { |
| 21282 | .register_pair => unreachable, | 22283 | .register_pair, .register_triple, .register_quadruple => unreachable, |
| 21283 | else => switch (loop.limb_offset) { | 22284 | else => switch (loop.limb_offset) { |
| 21284 | .unused => unreachable, | 22285 | .unused => unreachable, |
| 21285 | .known => |limb_offset| try src_mcv.mem(cg, .{ | 22286 | .known => |limb_offset| try src_mcv.mem(cg, .{ |
| ... | @@ -21309,76 +22310,53 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set | ... | @@ -21309,76 +22310,53 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21309 | .u(@as(u32, @intCast(src_mcv.immediate))) }, | 22310 | .u(@as(u32, @intCast(src_mcv.immediate))) }, |
| 21310 | }, | 22311 | }, |
| 21311 | }; | 22312 | }; |
| | 22313 | switch (src_op) { |
| | 22314 | else => {}, |
| | 22315 | .explicit => |linked_index| mir_ops[linked_index] = mir_op.*, |
| | 22316 | } |
| 21312 | } | 22317 | } |
| 21313 | for ( | 22318 | for ( |
| | 22319 | mir_ops[0..dst_temps.len], |
| 21314 | pattern.ops[0..dst_temps.len], | 22320 | pattern.ops[0..dst_temps.len], |
| 21315 | dst_temps, | 22321 | dst_temps, |
| 21316 | mir_ops[0..dst_temps.len], | 22322 | dst_tys, |
| 21317 | extra_temps[0..dst_temps.len], | 22323 | extra_temps[0..dst_temps.len], |
| 21318 | ) |dst_op, *dst_temp, *mir_op, *extra_temp| { | 22324 | ) |*mir_op, dst_op, dst_temp, dst_ty, extra_temp| { |
| 21319 | if (mir_op.* != .none) continue; | 22325 | if (mir_op.* != .none) continue; |
| 21320 | const ty = src_temps[0].typeOf(cg); | 22326 | mir_op.* = switch (dst_op) { |
| 21321 | switch (dst_op) { | | |
| 21322 | .implicit => unreachable, | 22327 | .implicit => unreachable, |
| 21323 | .explicit => |op_index| { | 22328 | .explicit => |linked_index| mir_ops[linked_index], |
| 21324 | dst_temp.* = dst_temps[op_index]; | 22329 | .gpr => .{ .reg = registerAlias( |
| 21325 | mir_op.* = mir_ops[op_index]; | 22330 | dst_temp.tracking(cg).short.register, |
| 21326 | }, | 22331 | @intCast(dst_ty.abiSize(cg.pt.zcu)), |
| 21327 | .gpr => { | 22332 | ) }, |
| 21328 | dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.gp); | 22333 | .mm, .mm_mask, .mm_sign_mask => @panic("TODO"), |
| 21329 | mir_op.* = .{ .reg = registerAlias( | 22334 | .xmm, .xmm_mask, .xmm_sign_mask => .{ .reg = dst_temp.tracking(cg).short.register.to128() }, |
| 21330 | dst_temp.tracking(cg).short.register, | 22335 | .ymm, .ymm_mask, .ymm_sign_mask => .{ .reg = dst_temp.tracking(cg).short.register.to256() }, |
| 21331 | @intCast(ty.abiSize(cg.pt.zcu)), | | |
| 21332 | ) }; | | |
| 21333 | }, | | |
| 21334 | .mm => @panic("TODO"), | | |
| 21335 | .xmm => { | | |
| 21336 | dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.sse); | | |
| 21337 | mir_op.* = .{ .reg = dst_temp.tracking(cg).short.register.to128() }; | | |
| 21338 | }, | | |
| 21339 | .ymm => { | | |
| 21340 | dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.sse); | | |
| 21341 | mir_op.* = .{ .reg = dst_temp.tracking(cg).short.register.to256() }; | | |
| 21342 | }, | | |
| 21343 | .mem => @panic("TODO"), | 22336 | .mem => @panic("TODO"), |
| 21344 | .gpr_limb => { | 22337 | .gpr_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to64() }, |
| 21345 | dst_temp.* = try cg.tempAlloc(ty); | 22338 | .mm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register }, |
| 21346 | extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); | 22339 | .xmm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to128() }, |
| 21347 | mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to64() }; | 22340 | .ymm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to256() }, |
| 21348 | }, | 22341 | .mem_limb => .{ .mem = try dst_temp.tracking(cg).short.mem(cg, switch (loop.limb_offset) { |
| 21349 | .mm_limb => { | 22342 | .unused => unreachable, |
| 21350 | dst_temp.* = try cg.tempAlloc(ty); | 22343 | .known => |limb_offset| .{ |
| 21351 | extra_temp.* = try cg.tempAllocReg(Type.usize, @panic("TODO")); | 22344 | .size = .fromSize(loop.limb_size.?), |
| 21352 | mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register }; | 22345 | .disp = limb_offset, |
| 21353 | }, | 22346 | }, |
| 21354 | .xmm_limb => { | 22347 | .temp => |limb_offset| .{ |
| 21355 | dst_temp.* = try cg.tempAlloc(ty); | 22348 | .size = .fromSize(loop.limb_size.?), |
| 21356 | extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.sse); | 22349 | .index = limb_offset.tracking(cg).short.register.to64(), |
| 21357 | mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to128() }; | 22350 | }, |
| 21358 | }, | 22351 | }) }, |
| 21359 | .ymm_limb => { | 22352 | .mm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register }, |
| 21360 | dst_temp.* = try cg.tempAlloc(ty); | 22353 | .xmm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to128() }, |
| 21361 | extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.sse); | 22354 | .ymm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to256() }, |
| 21362 | mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to256() }; | 22355 | .imm, .simm32, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, // unmodifiable destination |
| 21363 | }, | 22356 | }; |
| 21364 | .mem_limb => { | | |
| 21365 | dst_temp.* = try cg.tempAlloc(ty); | | |
| 21366 | mir_op.* = .{ .mem = try dst_temp.tracking(cg).short.mem(cg, switch (loop.limb_offset) { | | |
| 21367 | .unused => unreachable, | | |
| 21368 | .known => |limb_offset| .{ | | |
| 21369 | .size = .fromSize(loop.limb_size.?), | | |
| 21370 | .disp = limb_offset, | | |
| 21371 | }, | | |
| 21372 | .temp => |limb_offset| .{ | | |
| 21373 | .size = .fromSize(loop.limb_size.?), | | |
| 21374 | .index = limb_offset.tracking(cg).short.register.to64(), | | |
| 21375 | }, | | |
| 21376 | }) }; | | |
| 21377 | }, | | |
| 21378 | .imm, .simm32 => unreachable, // unmodifiable destination | | |
| 21379 | } | | |
| 21380 | } | 22357 | } |
| 21381 | std.mem.swap(Operand, &mir_ops[pattern.commute[0]], &mir_ops[pattern.commute[1]]); | 22358 | std.mem.swap(Operand, &mir_ops[pattern.commute[0]], &mir_ops[pattern.commute[1]]); |
| | 22359 | if (pattern_set.clobbers.eflags) try cg.spillEflagsIfOccupied(); |
| 21382 | cg.asmOps(pattern_set.mir_tag, mir_ops) catch |err| switch (err) { | 22360 | cg.asmOps(pattern_set.mir_tag, mir_ops) catch |err| switch (err) { |
| 21383 | error.InvalidInstruction => { | 22361 | error.InvalidInstruction => { |
| 21384 | const fixes = @tagName(pattern_set.mir_tag[0]); | 22362 | const fixes = @tagName(pattern_set.mir_tag[0]); |
| ... | @@ -21398,42 +22376,223 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set | ... | @@ -21398,42 +22376,223 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21398 | }, | 22376 | }, |
| 21399 | else => |e| return e, | 22377 | else => |e| return e, |
| 21400 | }; | 22378 | }; |
| | 22379 | const invert_result = opts.invert_result != pattern_set.invert_result; |
| 21401 | for ( | 22380 | for ( |
| 21402 | extra_temps[0..dst_temps.len], | 22381 | extra_temps[0..dst_temps.len], |
| 21403 | pattern.ops[0..dst_temps.len], | 22382 | pattern.ops[0..dst_temps.len], |
| | 22383 | mir_ops[0..dst_temps.len], |
| 21404 | dst_temps, | 22384 | dst_temps, |
| 21405 | ) |maybe_extra_temp, dst_op, dst_temp| if (maybe_extra_temp) |extra_temp| switch (dst_op) { | 22385 | ) |extra_temp, dst_op, mir_op, dst_temp| switch (dst_op) { |
| 21406 | else => {}, | 22386 | else => if (invert_result) { |
| 21407 | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (dst_temp.tracking(cg).short) { | 22387 | try cg.spillEflagsIfOccupied(); |
| 21408 | .register_pair => |dst_regs| switch (loop.limb_offset) { | 22388 | cg.asmOps( |
| 21409 | .unused => unreachable, | 22389 | .{ ._, .not }, |
| 21410 | .known => |limb_offset| try cg.asmRegisterRegister( | 22390 | .{ mir_op, .none, .none, .none }, |
| 21411 | .{ ._, .mov }, | 22391 | ) catch |err| switch (err) { |
| 21412 | dst_regs[@divExact(limb_offset, loop.limb_size.?)].to64(), | 22392 | error.InvalidInstruction => return cg.fail( |
| 21413 | extra_temp.tracking(cg).short.register.to64(), | 22393 | "invalid instruction: 'not {s} none none none'", |
| | 22394 | .{@tagName(mir_op)}, |
| 21414 | ), | 22395 | ), |
| 21415 | .temp => unreachable, | 22396 | else => |e| return e, |
| 21416 | }, | 22397 | }; |
| 21417 | else => |dst_mcv| try cg.asmMemoryRegister( | 22398 | }, |
| 21418 | switch (dst_op) { | 22399 | .mm_mask, |
| 21419 | else => unreachable, | 22400 | .xmm_mask, |
| 21420 | .gpr_limb => .{ ._, .mov }, | 22401 | .ymm_mask, |
| 21421 | .mm_limb => .{ ._q, .mov }, | 22402 | .mm_sign_mask, |
| 21422 | .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu }, | 22403 | .xmm_sign_mask, |
| | 22404 | .ymm_sign_mask, |
| | 22405 | => dst_temp.asMask(switch (dst_op) { |
| | 22406 | else => unreachable, |
| | 22407 | .mm_mask, .xmm_mask, .ymm_mask => .all, |
| | 22408 | .mm_sign_mask, .xmm_sign_mask, .ymm_sign_mask => .sign, |
| | 22409 | }, invert_result, switch (pattern_set.scalar) { |
| | 22410 | .any, .bool => unreachable, |
| | 22411 | .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size, |
| | 22412 | }, cg), |
| | 22413 | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => if (extra_temp) |limb_temp| |
| | 22414 | switch (dst_temp.tracking(cg).short) { |
| | 22415 | inline .register_pair, |
| | 22416 | .register_triple, |
| | 22417 | .register_quadruple, |
| | 22418 | => |dst_regs| switch (loop.limb_offset) { |
| | 22419 | .unused => unreachable, |
| | 22420 | .known => |limb_offset| try cg.asmRegisterRegister( |
| | 22421 | .{ ._, .mov }, |
| | 22422 | dst_regs[@divExact(limb_offset, loop.limb_size.?)].to64(), |
| | 22423 | limb_temp.tracking(cg).short.register.to64(), |
| | 22424 | ), |
| | 22425 | .temp => unreachable, |
| 21423 | }, | 22426 | }, |
| 21424 | try dst_mcv.mem(cg, switch (loop.limb_offset) { | 22427 | else => |dst_mcv| try cg.asmMemoryRegister( |
| | 22428 | switch (dst_op) { |
| | 22429 | else => unreachable, |
| | 22430 | .gpr_limb => .{ ._, .mov }, |
| | 22431 | .mm_limb => .{ ._q, .mov }, |
| | 22432 | .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu }, |
| | 22433 | }, |
| | 22434 | try dst_mcv.mem(cg, switch (loop.limb_offset) { |
| | 22435 | .unused => unreachable, |
| | 22436 | .known => |limb_offset| .{ |
| | 22437 | .size = .fromSize(loop.limb_size.?), |
| | 22438 | .disp = limb_offset, |
| | 22439 | }, |
| | 22440 | .temp => |limb_offset| .{ |
| | 22441 | .size = .fromSize(loop.limb_size.?), |
| | 22442 | .index = limb_offset.tracking(cg).short.register.to64(), |
| | 22443 | }, |
| | 22444 | }), |
| | 22445 | registerAlias(limb_temp.tracking(cg).short.register, loop.limb_size.?), |
| | 22446 | ), |
| | 22447 | }, |
| | 22448 | .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => { |
| | 22449 | const scalar_size = switch (pattern_set.scalar) { |
| | 22450 | .any, .bool => unreachable, |
| | 22451 | .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size, |
| | 22452 | }; |
| | 22453 | switch (scalar_size) { |
| | 22454 | else => {}, |
| | 22455 | .word => if (cg.hasFeature(.avx)) try cg.asmRegisterRegisterRegister( |
| | 22456 | .{ .vp_b, .ackssw }, |
| | 22457 | mir_op.reg, |
| | 22458 | mir_op.reg, |
| | 22459 | mir_op.reg, |
| | 22460 | ) else try cg.asmRegisterRegister( |
| | 22461 | .{ .p_b, .ackssw }, |
| | 22462 | mir_op.reg, |
| | 22463 | mir_op.reg, |
| | 22464 | ), |
| | 22465 | } |
| | 22466 | const mask_store_size: u4 = |
| | 22467 | @intCast(std.math.divCeil(u7, loop.mask_store_bit_size.?, 8) catch unreachable); |
| | 22468 | const mask_limb_reg = registerAlias( |
| | 22469 | loop.mask_limb_temp.?.tracking(cg).short.register, |
| | 22470 | mask_store_size, |
| | 22471 | ); |
| | 22472 | try cg.asmRegisterRegister(switch (scalar_size) { |
| | 22473 | else => unreachable, |
| | 22474 | .byte, .word => .{ if (cg.hasFeature(.avx)) .vp_b else .p_b, .movmsk }, |
| | 22475 | .dword => .{ if (cg.hasFeature(.avx)) .v_ps else ._ps, .movmsk }, |
| | 22476 | .qword => .{ if (cg.hasFeature(.avx)) .v_pd else ._pd, .movmsk }, |
| | 22477 | }, mask_limb_reg.to32(), mir_op.reg); |
| | 22478 | if (invert_result) if (loop.mask_store_reg) |_| { |
| | 22479 | try cg.spillEflagsIfOccupied(); |
| | 22480 | try cg.asmRegisterImmediate( |
| | 22481 | .{ ._, .xor }, |
| | 22482 | registerAlias(mask_limb_reg, @min(mask_store_size, 4)), |
| | 22483 | .u((@as(u32, 1) << @intCast(loop.mask_limb_bit_size.?)) - 1), |
| | 22484 | ); |
| | 22485 | } else try cg.asmRegister(.{ ._, .not }, mask_limb_reg); |
| | 22486 | if (loop.mask_store_reg) |mask_store_reg| { |
| | 22487 | const mask_store_alias = registerAlias(mask_store_reg, mask_store_size); |
| | 22488 | switch (loop.mask_limb_offset) { |
| 21425 | .unused => unreachable, | 22489 | .unused => unreachable, |
| 21426 | .known => |limb_offset| .{ | 22490 | .known => |mask_limb_offset| switch (mask_limb_offset & (loop.mask_store_bit_size.? - 1)) { |
| 21427 | .size = .fromSize(loop.limb_size.?), | 22491 | 0 => try cg.asmRegisterRegister(.{ ._, .mov }, mask_store_alias, mask_limb_reg), |
| 21428 | .disp = limb_offset, | 22492 | else => |shl_count| { |
| | 22493 | try cg.spillEflagsIfOccupied(); |
| | 22494 | try cg.asmRegisterImmediate(.{ ._l, .sh }, mask_limb_reg, .u(shl_count)); |
| | 22495 | try cg.spillEflagsIfOccupied(); |
| | 22496 | try cg.asmRegisterRegister(.{ ._, .@"or" }, mask_store_alias, mask_limb_reg); |
| | 22497 | }, |
| 21429 | }, | 22498 | }, |
| 21430 | .temp => |limb_offset| .{ | 22499 | .temp => |mask_limb_offset| { |
| 21431 | .size = .fromSize(loop.limb_size.?), | 22500 | if (cg.hasFeature(.bmi2)) { |
| 21432 | .index = limb_offset.tracking(cg).short.register.to64(), | 22501 | const shlx_size = @max(mask_store_size, 4); |
| | 22502 | const shlx_mask_limb_reg = registerAlias(mask_limb_reg, shlx_size); |
| | 22503 | try cg.asmRegisterRegisterRegister( |
| | 22504 | .{ ._lx, .sh }, |
| | 22505 | shlx_mask_limb_reg, |
| | 22506 | shlx_mask_limb_reg, |
| | 22507 | registerAlias(mask_limb_offset.tracking(cg).short.register, shlx_size), |
| | 22508 | ); |
| | 22509 | } else { |
| | 22510 | try cg.spillEflagsIfOccupied(); |
| | 22511 | try cg.asmRegisterRegister( |
| | 22512 | .{ ._l, .sh }, |
| | 22513 | mask_limb_reg, |
| | 22514 | mask_limb_offset.tracking(cg).short.register.to8(), |
| | 22515 | ); |
| | 22516 | } |
| | 22517 | try cg.spillEflagsIfOccupied(); |
| | 22518 | try cg.asmRegisterRegister(.{ ._, .@"or" }, mask_store_alias, mask_limb_reg); |
| 21433 | }, | 22519 | }, |
| 21434 | }), | 22520 | } |
| 21435 | registerAlias(extra_temp.tracking(cg).short.register, loop.limb_size.?), | 22521 | } |
| 21436 | ), | 22522 | const dst_mcv = dst_temp.tracking(cg).short; |
| | 22523 | switch (loop.mask_limb_offset) { |
| | 22524 | .unused => unreachable, |
| | 22525 | .known => |*mask_limb_offset| { |
| | 22526 | mask_limb_offset.* += loop.mask_limb_bit_size.?; |
| | 22527 | if (mask_limb_offset.* & (loop.mask_store_bit_size.? - 1) == 0) { |
| | 22528 | switch (dst_mcv) { |
| | 22529 | .register => {}, |
| | 22530 | else => try cg.asmMemoryRegister( |
| | 22531 | .{ ._, .mov }, |
| | 22532 | try dst_mcv.mem(cg, .{ |
| | 22533 | .size = .fromSize(mask_store_size), |
| | 22534 | .disp = @divExact(mask_limb_offset.*, 8) - mask_store_size, |
| | 22535 | }), |
| | 22536 | registerAlias(loop.mask_store_reg orelse mask_limb_reg, mask_store_size), |
| | 22537 | ), |
| | 22538 | } |
| | 22539 | if (loop.mask_store_reg) |mask_store_reg| { |
| | 22540 | const mask_store_alias = registerAlias(mask_store_reg, @min(mask_store_size, 4)); |
| | 22541 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias); |
| | 22542 | } |
| | 22543 | } |
| | 22544 | }, |
| | 22545 | .temp => |mask_limb_offset| { |
| | 22546 | const mask_limb_offset_reg = mask_limb_offset.tracking(cg).short.register.to32(); |
| | 22547 | if (loop.mask_store_reg) |mask_store_reg| { |
| | 22548 | try cg.asmRegisterMemory(.{ ._, .lea }, mask_limb_offset_reg, .{ |
| | 22549 | .base = .{ .reg = mask_limb_offset_reg.to64() }, |
| | 22550 | .mod = .{ .rm = .{ |
| | 22551 | .size = .qword, |
| | 22552 | .disp = loop.mask_limb_bit_size.?, |
| | 22553 | } }, |
| | 22554 | }); |
| | 22555 | switch (dst_mcv) { |
| | 22556 | .register => {}, |
| | 22557 | else => { |
| | 22558 | try cg.spillEflagsIfOccupied(); |
| | 22559 | try cg.asmRegisterImmediate( |
| | 22560 | .{ ._, .@"test" }, |
| | 22561 | mask_limb_offset_reg, |
| | 22562 | .u(loop.mask_store_bit_size.? - 1), |
| | 22563 | ); |
| | 22564 | const skip_store_reloc = try cg.asmJccReloc(.nz, undefined); |
| | 22565 | const mask_store_offset_reg = mask_limb_reg.to32(); |
| | 22566 | try cg.asmRegisterRegister(.{ ._, .mov }, mask_store_offset_reg, mask_limb_offset_reg); |
| | 22567 | try cg.asmRegisterImmediate(.{ ._r, .sh }, mask_store_offset_reg, .u(3)); |
| | 22568 | try cg.asmMemoryRegister(.{ ._, .mov }, try dst_mcv.mem(cg, .{ |
| | 22569 | .size = .fromSize(mask_store_size), |
| | 22570 | .index = mask_store_offset_reg.to64(), |
| | 22571 | .disp = -@as(i8, mask_store_size), |
| | 22572 | }), registerAlias(mask_store_reg, mask_store_size)); |
| | 22573 | const mask_store_alias = registerAlias(mask_store_reg, @min(mask_store_size, 4)); |
| | 22574 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias); |
| | 22575 | cg.performReloc(skip_store_reloc); |
| | 22576 | }, |
| | 22577 | } |
| | 22578 | } else { |
| | 22579 | switch (dst_mcv) { |
| | 22580 | .register => {}, |
| | 22581 | else => try cg.asmMemoryRegister(.{ ._, .mov }, try dst_mcv.mem(cg, .{ |
| | 22582 | .size = .fromSize(mask_store_size), |
| | 22583 | .index = mask_limb_offset_reg.to64(), |
| | 22584 | }), mask_limb_reg), |
| | 22585 | } |
| | 22586 | try cg.asmRegisterMemory(.{ ._, .lea }, mask_limb_offset_reg, .{ |
| | 22587 | .base = .{ .reg = mask_limb_offset_reg.to64() }, |
| | 22588 | .mod = .{ .rm = .{ |
| | 22589 | .size = .qword, |
| | 22590 | .disp = mask_store_size, |
| | 22591 | } }, |
| | 22592 | }); |
| | 22593 | } |
| | 22594 | }, |
| | 22595 | } |
| 21437 | }, | 22596 | }, |
| 21438 | }; | 22597 | }; |
| 21439 | switch (pattern_set.loop) { | 22598 | switch (pattern_set.loop) { |
| ... | @@ -21442,7 +22601,7 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set | ... | @@ -21442,7 +22601,7 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21442 | .limbwise_carry => @panic("TODO"), | 22601 | .limbwise_carry => @panic("TODO"), |
| 21443 | .limbwise_pairs_forward => @panic("TODO"), | 22602 | .limbwise_pairs_forward => @panic("TODO"), |
| 21444 | .limbwise_pairs_reverse => @panic("TODO"), | 22603 | .limbwise_pairs_reverse => @panic("TODO"), |
| 21445 | .elementwise => @panic("TODO"), | 22604 | .elementwise => {}, |
| 21446 | } | 22605 | } |
| 21447 | switch (loop.limb_offset) { | 22606 | switch (loop.limb_offset) { |
| 21448 | .unused => break, | 22607 | .unused => break, |
| ... | @@ -21452,6 +22611,11 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set | ... | @@ -21452,6 +22611,11 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21452 | if (loop.remaining_size.? < loop.limb_size.? or | 22611 | if (loop.remaining_size.? < loop.limb_size.? or |
| 21453 | (loop.element_size != null and limb_offset.* >= loop.element_size.?)) | 22612 | (loop.element_size != null and limb_offset.* >= loop.element_size.?)) |
| 21454 | { | 22613 | { |
| | 22614 | switch (loop.mask_limb_offset) { |
| | 22615 | .unused => {}, |
| | 22616 | .known => |*mask_limb_offset| mask_limb_offset.* = 0, |
| | 22617 | .temp => unreachable, |
| | 22618 | } |
| 21455 | limb_offset.* = 0; | 22619 | limb_offset.* = 0; |
| 21456 | break; | 22620 | break; |
| 21457 | } | 22621 | } |
| ... | @@ -21465,6 +22629,7 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set | ... | @@ -21465,6 +22629,7 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21465 | .disp = loop.limb_size.?, | 22629 | .disp = loop.limb_size.?, |
| 21466 | } }, | 22630 | } }, |
| 21467 | }); | 22631 | }); |
| | 22632 | try cg.spillEflagsIfOccupied(); |
| 21468 | try cg.asmRegisterImmediate( | 22633 | try cg.asmRegisterImmediate( |
| 21469 | .{ ._, .cmp }, | 22634 | .{ ._, .cmp }, |
| 21470 | limb_offset_reg.to32(), | 22635 | limb_offset_reg.to32(), |
| ... | @@ -21476,6 +22641,12 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set | ... | @@ -21476,6 +22641,12 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21476 | }, | 22641 | }, |
| 21477 | } | 22642 | } |
| 21478 | } | 22643 | } |
| | 22644 | if (loop.mask_limb_temp) |mask_limb_temp| try mask_limb_temp.die(cg); |
| | 22645 | if (loop.mask_store_temp) |mask_store_temp| try mask_store_temp.die(cg); |
| | 22646 | switch (loop.mask_limb_offset) { |
| | 22647 | .unused, .known => {}, |
| | 22648 | .temp => |mask_limb_offset| try mask_limb_offset.die(cg), |
| | 22649 | } |
| 21479 | switch (loop.element_offset) { | 22650 | switch (loop.element_offset) { |
| 21480 | .unused => break :pattern_sets, | 22651 | .unused => break :pattern_sets, |
| 21481 | .known => |*element_offset| { | 22652 | .known => |*element_offset| { |
Jacob Young <jacobly@ziglang.org> 2024-12-01 23:53:22-05:00
test