authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-09 09:48:29-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-16 20:47:30-05:00
log666d76d85c1dc2f107d0a57b424983082672943e
tree0c80f3d3fce09874a00cca67c6cc80a79923f65e
parent6d1fc0f51cf835e8ed503a2089450b1f280d1216

x86_64: implement load and store

2 files changed, 302 insertions(+), 125 deletions(-)

src/arch/x86_64/CodeGen.zig+301-124
......@@ -257,6 +257,7 @@ pub const MCValue = union(enum) {
257257 };
258258 }
259259
260 // hack around linker relocation bugs
260261 fn isBase(mcv: MCValue) bool {
261262 return switch (mcv) {
262263 .memory, .indirect, .load_frame => true,
......@@ -2398,8 +2399,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
23982399 .add_wrap,
23992400 .sub,
24002401 .sub_wrap,
2401 .bool_and,
2402 .bool_or,
24032402 .min,
24042403 .max,
24052404 => |air_tag| try cg.airBinOp(inst, air_tag),
......@@ -2454,9 +2453,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
24542453 .is_null => try cg.airIsNull(inst),
24552454 .is_non_err => try cg.airIsNonErr(inst),
24562455 .is_err => try cg.airIsErr(inst),
2457 .load => try cg.airLoad(inst),
2458 .store => try cg.airStore(inst, false),
2459 .store_safe => try cg.airStore(inst, true),
24602456 .float_from_int => try cg.airFloatFromInt(inst),
24612457 .int_from_float => try cg.airIntFromFloat(inst),
24622458 .cmpxchg_strong => try cg.airCmpxchg(inst),
......@@ -2791,14 +2787,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
27912787 try slot.moveTo(inst, cg);
27922788 },
27932789 .assembly => try cg.airAsm(inst),
2794 .bit_and, .bit_or, .xor => |air_tag| if (use_old) try cg.airBinOp(inst, air_tag) else {
2790 .bit_and, .bit_or, .xor, .bool_and, .bool_or => |air_tag| if (use_old) try cg.airBinOp(inst, air_tag) else {
27952791 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
27962792 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
27972793 var res: [1]Temp = undefined;
27982794 cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, switch (@as(Mir.Inst.Tag, switch (air_tag) {
27992795 else => unreachable,
2800 .bit_and => .@"and",
2801 .bit_or => .@"or",
2796 .bit_and, .bool_and => .@"and",
2797 .bit_or, .bool_or => .@"or",
28022798 .xor => .xor,
28032799 })) {
28042800 else => unreachable,
......@@ -9601,6 +9597,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
96019597 try ops[0].die(cg);
96029598 try is_non_err.moveTo(inst, cg);
96039599 },
9600 .load => if (use_old) try cg.airLoad(inst) else fallback: {
9601 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9602 const val_ty = ty_op.ty.toType();
9603 const ptr_ty = cg.typeOf(ty_op.operand);
9604 const ptr_info = ptr_ty.ptrInfo(zcu);
9605 if (ptr_info.packed_offset.host_size > 0 and
9606 (ptr_info.flags.vector_index == .none or val_ty.toIntern() == .bool_type))
9607 break :fallback try cg.airLoad(inst);
9608 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9609 var res = try ops[0].load(val_ty, .{
9610 .disp = switch (ptr_info.flags.vector_index) {
9611 .none => 0,
9612 .runtime => unreachable,
9613 else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)),
9614 },
9615 }, cg);
9616 for (ops) |op| if (op.index != res.index) try op.die(cg);
9617 try res.moveTo(inst, cg);
9618 },
96049619 .int_from_ptr => if (use_old) try cg.airIntFromPtr(inst) else {
96059620 const un_op = air_datas[@intFromEnum(inst)].un_op;
96069621 var ops = try cg.tempsFromOperands(inst, .{un_op});
......@@ -9615,6 +9630,37 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
96159630 .ret => try cg.airRet(inst, false),
96169631 .ret_safe => try cg.airRet(inst, true),
96179632 .ret_load => try cg.airRetLoad(inst),
9633 .store, .store_safe => |air_tag| if (use_old) try cg.airStore(inst, switch (air_tag) {
9634 else => unreachable,
9635 .store => false,
9636 .store_safe => true,
9637 }) else fallback: {
9638 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
9639 const ptr_ty = cg.typeOf(bin_op.lhs);
9640 const ptr_info = ptr_ty.ptrInfo(zcu);
9641 const val_ty = cg.typeOf(bin_op.rhs);
9642 if (ptr_info.packed_offset.host_size > 0 and
9643 (ptr_info.flags.vector_index == .none or val_ty.toIntern() == .bool_type))
9644 break :fallback try cg.airStore(inst, switch (air_tag) {
9645 else => unreachable,
9646 .store => false,
9647 .store_safe => true,
9648 });
9649 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
9650 try ops[0].store(&ops[1], .{
9651 .disp = switch (ptr_info.flags.vector_index) {
9652 .none => 0,
9653 .runtime => unreachable,
9654 else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)),
9655 },
9656 .safe = switch (air_tag) {
9657 else => unreachable,
9658 .store => false,
9659 .store_safe => true,
9660 },
9661 }, cg);
9662 for (ops) |op| try op.die(cg);
9663 },
96189664 .unreach => {},
96199665 .optional_payload_ptr => if (use_old) try cg.airOptionalPayloadPtr(inst) else {
96209666 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
......@@ -9630,7 +9676,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
96309676 const opt_child_abi_size: i32 = @intCast(opt_child_ty.abiSize(zcu));
96319677 try ops[0].toOffset(opt_child_abi_size, cg);
96329678 var has_value = try cg.tempInit(.bool, .{ .immediate = 1 });
9633 try ops[0].store(0, &has_value, cg);
9679 try ops[0].store(&has_value, .{}, cg);
96349680 try has_value.die(cg);
96359681 try ops[0].toOffset(-opt_child_abi_size, cg);
96369682 }
......@@ -9652,7 +9698,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
96529698 const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu));
96539699 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
96549700 try ops[0].toOffset(eu_err_off, cg);
9655 var err = try ops[0].load(0, eu_ty.errorUnionSet(zcu), cg);
9701 var err = try ops[0].load(eu_ty.errorUnionSet(zcu), .{}, cg);
96569702 try ops[0].die(cg);
96579703 try err.moveTo(inst, cg);
96589704 },
......@@ -9666,7 +9712,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
96669712 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
96679713 try ops[0].toOffset(eu_err_off, cg);
96689714 var no_err = try cg.tempInit(eu_err_ty, .{ .immediate = 0 });
9669 try ops[0].store(0, &no_err, cg);
9715 try ops[0].store(&no_err, .{}, cg);
96709716 try no_err.die(cg);
96719717 try ops[0].toOffset(eu_pl_off - eu_err_off, cg);
96729718 try ops[0].moveTo(inst, cg);
......@@ -9682,43 +9728,29 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
96829728 ), cg);
96839729 try ops[0].moveTo(inst, cg);
96849730 },
9685 .struct_field_ptr_index_0 => if (use_old) try cg.airStructFieldPtrIndex(inst, 0) else {
9686 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9687 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9688 try ops[0].toOffset(cg.fieldOffset(
9689 cg.typeOf(ty_op.operand),
9690 ty_op.ty.toType(),
9691 0,
9692 ), cg);
9693 try ops[0].moveTo(inst, cg);
9694 },
9695 .struct_field_ptr_index_1 => if (use_old) try cg.airStructFieldPtrIndex(inst, 1) else {
9696 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9697 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9698 try ops[0].toOffset(cg.fieldOffset(
9699 cg.typeOf(ty_op.operand),
9700 ty_op.ty.toType(),
9701 1,
9702 ), cg);
9703 try ops[0].moveTo(inst, cg);
9704 },
9705 .struct_field_ptr_index_2 => if (use_old) try cg.airStructFieldPtrIndex(inst, 2) else {
9706 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
9707 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
9708 try ops[0].toOffset(cg.fieldOffset(
9709 cg.typeOf(ty_op.operand),
9710 ty_op.ty.toType(),
9711 2,
9712 ), cg);
9713 try ops[0].moveTo(inst, cg);
9714 },
9715 .struct_field_ptr_index_3 => if (use_old) try cg.airStructFieldPtrIndex(inst, 3) else {
9731 .struct_field_ptr_index_0,
9732 .struct_field_ptr_index_1,
9733 .struct_field_ptr_index_2,
9734 .struct_field_ptr_index_3,
9735 => |air_tag| if (use_old) try cg.airStructFieldPtrIndex(inst, switch (air_tag) {
9736 else => unreachable,
9737 .struct_field_ptr_index_0 => 0,
9738 .struct_field_ptr_index_1 => 1,
9739 .struct_field_ptr_index_2 => 2,
9740 .struct_field_ptr_index_3 => 3,
9741 }) else {
97169742 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
97179743 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
97189744 try ops[0].toOffset(cg.fieldOffset(
97199745 cg.typeOf(ty_op.operand),
97209746 ty_op.ty.toType(),
9721 3,
9747 switch (air_tag) {
9748 else => unreachable,
9749 .struct_field_ptr_index_0 => 0,
9750 .struct_field_ptr_index_1 => 1,
9751 .struct_field_ptr_index_2 => 2,
9752 .struct_field_ptr_index_3 => 3,
9753 },
97229754 ), cg);
97239755 try ops[0].moveTo(inst, cg);
97249756 },
......@@ -9733,7 +9765,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
97339765 };
97349766 if (field_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
97359767 var ops = try cg.tempsFromOperands(inst, .{extra.struct_operand});
9736 var res = try ops[0].read(field_off, field_ty, cg);
9768 var res = try ops[0].read(field_ty, .{ .disp = field_off }, cg);
97379769 for (ops) |op| if (op.index != res.index) try op.die(cg);
97389770 try res.moveTo(inst, cg);
97399771 } else {
......@@ -9748,7 +9780,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
97489780 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
97499781 const union_layout = union_ty.unionGetLayout(zcu);
97509782 // hack around Sema OPV bugs
9751 if (union_layout.tag_size > 0) try ops[0].store(@intCast(union_layout.tagOffset()), &ops[1], cg);
9783 if (union_layout.tag_size > 0) try ops[0].store(&ops[1], .{
9784 .disp = @intCast(union_layout.tagOffset()),
9785 }, cg);
97529786 for (ops) |op| try op.die(cg);
97539787 },
97549788 .get_union_tag => if (use_old) try cg.airGetUnionTag(inst) else {
......@@ -9757,7 +9791,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
97579791 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
97589792 const union_layout = union_ty.unionGetLayout(zcu);
97599793 assert(union_layout.tag_size > 0);
9760 var res = try ops[0].read(@intCast(union_layout.tagOffset()), ty_op.ty.toType(), cg);
9794 var res = try ops[0].read(ty_op.ty.toType(), .{
9795 .disp = @intCast(union_layout.tagOffset()),
9796 }, cg);
97619797 for (ops) |op| if (op.index != res.index) try op.die(cg);
97629798 try res.moveTo(inst, cg);
97639799 },
......@@ -9916,7 +9952,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
99169952 .scale = .fromFactor(@intCast(elem_size)),
99179953 } },
99189954 });
9919 res[0] = try ops[0].load(0, res_ty, cg);
9955 res[0] = try ops[0].load(res_ty, .{}, cg);
99209956 },
99219957 },
99229958 else => |e| return e,
......@@ -10002,10 +10038,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1000210038 union_ty.unionTagTypeSafety(zcu).?,
1000310039 extra.field_index,
1000410040 ));
10005 try res.write(@intCast(union_layout.tagOffset()), &tag_temp, cg);
10041 try res.write(&tag_temp, .{
10042 .disp = @intCast(union_layout.tagOffset()),
10043 }, cg);
1000610044 try tag_temp.die(cg);
1000710045 }
10008 try res.write(@intCast(union_layout.payloadOffset()), &ops[0], cg);
10046 try res.write(&ops[0], .{
10047 .disp = @intCast(union_layout.payloadOffset()),
10048 }, cg);
1000910049 try ops[0].die(cg);
1001010050 try res.moveTo(inst, cg);
1001110051 },
......@@ -28338,6 +28378,7 @@ const Temp = struct {
2833828378 return true;
2833928379 }
2834028380
28381 // hack around linker relocation bugs
2834128382 fn toBase(temp: *Temp, cg: *CodeGen) !bool {
2834228383 const temp_tracking = temp.tracking(cg);
2834328384 if (temp_tracking.short.isBase()) return false;
......@@ -28354,17 +28395,38 @@ const Temp = struct {
2835428395 return true;
2835528396 }
2835628397
28357 fn load(ptr: *Temp, disp: i32, val_ty: Type, cg: *CodeGen) !Temp {
28398 const AccessOptions = struct {
28399 disp: i32 = 0,
28400 safe: bool = false,
28401 };
28402
28403 fn load(ptr: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) !Temp {
2835828404 const val = try cg.tempAlloc(val_ty);
28405 try ptr.toOffset(opts.disp, cg);
28406 while (try ptr.toLea(cg)) {}
2835928407 const val_mcv = val.tracking(cg).short;
2836028408 switch (val_mcv) {
2836128409 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28362 .register => |val_reg| {
28410 .register => |val_reg| try ptr.loadReg(val_ty, registerAlias(
28411 val_reg,
28412 @intCast(val_ty.abiSize(cg.pt.zcu)),
28413 ), cg),
28414 inline .register_pair,
28415 .register_triple,
28416 .register_quadruple,
28417 => |val_regs| for (val_regs) |val_reg| {
28418 try ptr.loadReg(val_ty, val_reg, cg);
28419 try ptr.toOffset(@divExact(val_reg.bitSize(), 8), cg);
2836328420 while (try ptr.toLea(cg)) {}
28364 try cg.genSetReg(val_reg, val_ty, ptr.tracking(cg).short.offset(disp).deref(), .{});
28421 },
28422 .register_offset => |val_reg_off| switch (val_reg_off.off) {
28423 0 => try ptr.loadReg(val_ty, registerAlias(
28424 val_reg_off.reg,
28425 @intCast(val_ty.abiSize(cg.pt.zcu)),
28426 ), cg),
28427 else => unreachable,
2836528428 },
2836628429 .memory, .indirect, .load_frame, .load_symbol => {
28367 try ptr.toOffset(disp, cg);
2836828430 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
2836928431 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
2837028432 try val_ptr.memcpy(ptr, &len, cg);
......@@ -28375,103 +28437,158 @@ const Temp = struct {
2837528437 return val;
2837628438 }
2837728439
28378 fn store(ptr: *Temp, disp: i32, val: *Temp, cg: *CodeGen) !void {
28440 fn store(ptr: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) !void {
2837928441 const val_ty = val.typeOf(cg);
28380 val: switch (val.tracking(cg).short) {
28381 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28382 .immediate => |val_imm| {
28383 const val_op: Immediate = if (std.math.cast(u32, val_imm)) |val_uimm32|
28384 .u(val_uimm32)
28385 else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32|
28386 .s(val_simm32)
28387 else
28388 continue :val .{ .register = undefined };
28389 while (try ptr.toLea(cg)) {}
28390 try cg.asmMemoryImmediate(
28391 .{ ._, .mov },
28392 try ptr.tracking(cg).short.deref().mem(cg, .{
28393 .size = cg.memSize(val_ty),
28394 .disp = disp,
28395 }),
28396 val_op,
28397 );
28398 },
28399 .register => {
28400 while (try ptr.toLea(cg) or try val.toRegClass(true, .general_purpose, cg)) {}
28401 const val_reg = val.tracking(cg).short.register;
28402 switch (val_reg.class()) {
28403 .general_purpose => try cg.asmMemoryRegister(
28442 try ptr.toOffset(opts.disp, cg);
28443 while (try ptr.toLea(cg)) {}
28444 val_to_gpr: while (true) : (while (try ptr.toLea(cg) or
28445 try val.toRegClass(false, .general_purpose, cg))
28446 {}) {
28447 const val_mcv = val.tracking(cg).short;
28448 switch (val_mcv) {
28449 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28450 .undef => if (opts.safe) {
28451 var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa });
28452 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
28453 try ptr.memset(&pat, &len, cg);
28454 try pat.die(cg);
28455 try len.die(cg);
28456 },
28457 .immediate => |val_imm| {
28458 const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31|
28459 .u(val_uimm31)
28460 else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32|
28461 .s(val_simm32)
28462 else
28463 continue :val_to_gpr;
28464 // hack around linker relocation bugs
28465 switch (ptr.tracking(cg).short) {
28466 else => {},
28467 .lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
28468 }
28469 try cg.asmMemoryImmediate(
2840428470 .{ ._, .mov },
2840528471 try ptr.tracking(cg).short.deref().mem(cg, .{
2840628472 .size = cg.memSize(val_ty),
28407 .disp = disp,
2840828473 }),
28409 registerAlias(val_reg, @intCast(val_ty.abiSize(cg.pt.zcu))),
28410 ),
28411 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28412 }
28413 },
28414 }
28415 }
28416
28417 fn read(src: *Temp, disp: i32, val_ty: Type, cg: *CodeGen) !Temp {
28418 var val = try cg.tempAlloc(val_ty);
28419 while (try src.toBase(cg)) {}
28420 val_to_gpr: while (true) : (while (try val.toRegClass(false, .general_purpose, cg)) {}) {
28421 const val_mcv = val.tracking(cg).short;
28422 switch (val_mcv) {
28423 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28424 .register => |val_reg| try src.readReg(disp, val_ty, registerAlias(
28474 val_op,
28475 );
28476 },
28477 .eflags => |cc| {
28478 // hack around linker relocation bugs
28479 switch (ptr.tracking(cg).short) {
28480 else => {},
28481 .lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
28482 }
28483 try cg.asmSetccMemory(
28484 cc,
28485 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }),
28486 );
28487 },
28488 .register => |val_reg| try ptr.storeReg(val_ty, registerAlias(
2842528489 val_reg,
2842628490 @intCast(val_ty.abiSize(cg.pt.zcu)),
2842728491 ), cg),
28428 inline .register_pair, .register_triple, .register_quadruple => |val_regs| {
28429 var part_disp = disp;
28430 for (val_regs) |val_reg| {
28431 try src.readReg(disp, val_ty, val_reg, cg);
28432 part_disp += @divExact(val_reg.bitSize(), 8);
28433 }
28492 inline .register_pair,
28493 .register_triple,
28494 .register_quadruple,
28495 => |val_regs| for (val_regs) |val_reg| {
28496 try ptr.storeReg(val_ty, val_reg, cg);
28497 try ptr.toOffset(@divExact(val_reg.bitSize(), 8), cg);
28498 while (try ptr.toLea(cg)) {}
2843428499 },
2843528500 .register_offset => |val_reg_off| switch (val_reg_off.off) {
28436 0 => try src.readReg(disp, val_ty, registerAlias(
28501 0 => try ptr.storeReg(val_ty, registerAlias(
2843728502 val_reg_off.reg,
2843828503 @intCast(val_ty.abiSize(cg.pt.zcu)),
2843928504 ), cg),
2844028505 else => continue :val_to_gpr,
2844128506 },
28507 .register_overflow => |val_reg_ov| {
28508 const ip = &cg.pt.zcu.intern_pool;
28509 const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) {
28510 .tuple_type => |tuple_type| {
28511 const tuple_field_types = tuple_type.types.get(ip);
28512 assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type);
28513 break :first_ty tuple_field_types[0];
28514 },
28515 .opt_type => |opt_child| {
28516 assert(!val_ty.optionalReprIsPayload(cg.pt.zcu));
28517 break :first_ty opt_child;
28518 },
28519 else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }),
28520 });
28521 const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu));
28522 try ptr.storeReg(first_ty, registerAlias(val_reg_ov.reg, first_size), cg);
28523 try ptr.toOffset(first_size, cg);
28524 try cg.asmSetccMemory(
28525 val_reg_ov.eflags,
28526 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }),
28527 );
28528 },
2844228529 .lea_frame, .lea_symbol => continue :val_to_gpr,
2844328530 .memory, .indirect, .load_frame, .load_symbol => {
2844428531 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
28445 var src_ptr = try cg.tempInit(.usize, src.tracking(cg).short.address().offset(disp));
2844628532 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
28447 try val_ptr.memcpy(&src_ptr, &len, cg);
28533 try ptr.memcpy(&val_ptr, &len, cg);
2844828534 try val_ptr.die(cg);
28449 try src_ptr.die(cg);
2845028535 try len.die(cg);
2845128536 },
2845228537 }
28453 return val;
28538 break;
2845428539 }
2845528540 }
2845628541
28457 fn readReg(src: Temp, disp: i32, dst_ty: Type, dst_reg: Register, cg: *CodeGen) !void {
28458 const strat = try cg.moveStrategy(dst_ty, dst_reg.class(), false);
28459 try strat.read(cg, dst_reg, try src.tracking(cg).short.mem(cg, .{
28460 .size = .fromBitSize(@min(8 * dst_ty.abiSize(cg.pt.zcu), dst_reg.bitSize())),
28461 .disp = disp,
28462 }));
28542 fn read(src: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) !Temp {
28543 var val = try cg.tempAlloc(val_ty);
28544 while (try src.toBase(cg)) {}
28545 const val_mcv = val.tracking(cg).short;
28546 switch (val_mcv) {
28547 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
28548 .register => |val_reg| try src.readReg(opts.disp, val_ty, registerAlias(
28549 val_reg,
28550 @intCast(val_ty.abiSize(cg.pt.zcu)),
28551 ), cg),
28552 inline .register_pair, .register_triple, .register_quadruple => |val_regs| {
28553 var disp = opts.disp;
28554 for (val_regs) |val_reg| {
28555 try src.readReg(disp, val_ty, val_reg, cg);
28556 disp += @divExact(val_reg.bitSize(), 8);
28557 }
28558 },
28559 .register_offset => |val_reg_off| switch (val_reg_off.off) {
28560 0 => try src.readReg(opts.disp, val_ty, registerAlias(
28561 val_reg_off.reg,
28562 @intCast(val_ty.abiSize(cg.pt.zcu)),
28563 ), cg),
28564 else => unreachable,
28565 },
28566 .memory, .indirect, .load_frame, .load_symbol => {
28567 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
28568 var src_ptr =
28569 try cg.tempInit(.usize, src.tracking(cg).short.address().offset(opts.disp));
28570 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
28571 try val_ptr.memcpy(&src_ptr, &len, cg);
28572 try val_ptr.die(cg);
28573 try src_ptr.die(cg);
28574 try len.die(cg);
28575 },
28576 }
28577 return val;
2846328578 }
2846428579
28465 fn write(dst: *Temp, disp: i32, val: *Temp, cg: *CodeGen) !void {
28580 fn write(dst: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) !void {
2846628581 const val_ty = val.typeOf(cg);
2846728582 while (try dst.toBase(cg)) {}
28468 val_to_gpr: while (true) : (while (try val.toRegClass(false, .general_purpose, cg)) {}) {
28583 val_to_gpr: while (true) : (while (try dst.toBase(cg) or
28584 try val.toRegClass(false, .general_purpose, cg))
28585 {}) {
2846928586 const val_mcv = val.tracking(cg).short;
2847028587 switch (val_mcv) {
2847128588 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
2847228589 .immediate => |val_imm| {
28473 const val_op: Immediate = if (std.math.cast(u32, val_imm)) |val_uimm32|
28474 .u(val_uimm32)
28590 const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31|
28591 .u(val_uimm31)
2847528592 else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32|
2847628593 .s(val_simm32)
2847728594 else
......@@ -28480,24 +28597,24 @@ const Temp = struct {
2848028597 .{ ._, .mov },
2848128598 try dst.tracking(cg).short.mem(cg, .{
2848228599 .size = cg.memSize(val_ty),
28483 .disp = disp,
28600 .disp = opts.disp,
2848428601 }),
2848528602 val_op,
2848628603 );
2848728604 },
28488 .register => |val_reg| try dst.writeReg(disp, val_ty, registerAlias(
28605 .register => |val_reg| try dst.writeReg(opts.disp, val_ty, registerAlias(
2848928606 val_reg,
2849028607 @intCast(val_ty.abiSize(cg.pt.zcu)),
2849128608 ), cg),
2849228609 inline .register_pair, .register_triple, .register_quadruple => |val_regs| {
28493 var part_disp = disp;
28610 var disp = opts.disp;
2849428611 for (val_regs) |val_reg| {
28495 try dst.writeReg(part_disp, val_ty, val_reg, cg);
28496 part_disp += @divExact(val_reg.bitSize(), 8);
28612 try dst.writeReg(disp, val_ty, val_reg, cg);
28613 disp += @divExact(val_reg.bitSize(), 8);
2849728614 }
2849828615 },
2849928616 .register_offset => |val_reg_off| switch (val_reg_off.off) {
28500 0 => try dst.writeReg(disp, val_ty, registerAlias(
28617 0 => try dst.writeReg(opts.disp, val_ty, registerAlias(
2850128618 val_reg_off.reg,
2850228619 @intCast(val_ty.abiSize(cg.pt.zcu)),
2850328620 ), cg),
......@@ -28505,7 +28622,8 @@ const Temp = struct {
2850528622 },
2850628623 .lea_frame, .lea_symbol => continue :val_to_gpr,
2850728624 .memory, .indirect, .load_frame, .load_symbol => {
28508 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(disp));
28625 var dst_ptr =
28626 try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
2850928627 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
2851028628 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
2851128629 try dst_ptr.memcpy(&val_ptr, &len, cg);
......@@ -28514,10 +28632,62 @@ const Temp = struct {
2851428632 try len.die(cg);
2851528633 },
2851628634 }
28517 return;
28635 break;
28636 }
28637 }
28638
28639 fn loadReg(ptr: *Temp, dst_ty: Type, dst_reg: Register, cg: *CodeGen) !void {
28640 const dst_rc = dst_reg.class();
28641 const strat = try cg.moveStrategy(dst_ty, dst_rc, false);
28642 // hack around linker relocation bugs
28643 switch (ptr.tracking(cg).short) {
28644 else => {},
28645 .lea_symbol => |sym_off| if (dst_rc != .general_purpose or sym_off.off != 0)
28646 while (try ptr.toRegClass(false, .general_purpose, cg)) {},
28647 }
28648 try strat.read(cg, dst_reg, try ptr.tracking(cg).short.deref().mem(cg, .{
28649 .size = .fromBitSize(@min(8 * dst_ty.abiSize(cg.pt.zcu), dst_reg.bitSize())),
28650 }));
28651 }
28652
28653 fn storeReg(ptr: *Temp, src_ty: Type, src_reg: Register, cg: *CodeGen) !void {
28654 const src_rc = src_reg.class();
28655 const src_abi_size = src_ty.abiSize(cg.pt.zcu);
28656 const strat = try cg.moveStrategy(src_ty, src_rc, false);
28657 // hack around linker relocation bugs
28658 switch (ptr.tracking(cg).short) {
28659 else => {},
28660 .lea_symbol => |sym_off| if (src_rc != .general_purpose or sym_off.off != 0)
28661 while (try ptr.toRegClass(false, .general_purpose, cg)) {},
28662 }
28663 if (src_rc == .x87 or std.math.isPowerOfTwo(src_abi_size)) {
28664 try strat.write(cg, try ptr.tracking(cg).short.deref().mem(cg, .{
28665 .size = .fromBitSize(@min(8 * src_abi_size, src_reg.bitSize())),
28666 }), src_reg);
28667 } else {
28668 const frame_alloc: FrameAlloc = .initSpill(src_ty, cg.pt.zcu);
28669 const frame_index = try cg.allocFrameIndex(frame_alloc);
28670 const frame_size: Memory.Size = .fromSize(frame_alloc.abi_size);
28671 try strat.write(cg, .{
28672 .base = .{ .frame = frame_index },
28673 .mod = .{ .rm = .{ .size = frame_size } },
28674 }, src_reg);
28675 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
28676 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
28677 try ptr.memcpy(&src_ptr, &len, cg);
28678 try src_ptr.die(cg);
28679 try len.die(cg);
2851828680 }
2851928681 }
2852028682
28683 fn readReg(src: Temp, disp: i32, dst_ty: Type, dst_reg: Register, cg: *CodeGen) !void {
28684 const strat = try cg.moveStrategy(dst_ty, dst_reg.class(), false);
28685 try strat.read(cg, dst_reg, try src.tracking(cg).short.mem(cg, .{
28686 .size = .fromBitSize(@min(8 * dst_ty.abiSize(cg.pt.zcu), dst_reg.bitSize())),
28687 .disp = disp,
28688 }));
28689 }
28690
2852128691 fn writeReg(dst: Temp, disp: i32, src_ty: Type, src_reg: Register, cg: *CodeGen) !void {
2852228692 const src_rc = src_reg.class();
2852328693 const src_abi_size = src_ty.abiSize(cg.pt.zcu);
......@@ -28552,6 +28722,13 @@ const Temp = struct {
2855228722 try cg.asmOpOnly(.{ .@"rep _sb", .mov });
2855328723 }
2855428724
28725 fn memset(dst: *Temp, val: *Temp, len: *Temp, cg: *CodeGen) !void {
28726 while (true) for ([_]*Temp{ dst, val, len }, [_]Register{ .rdi, .rax, .rcx }) |temp, reg| {
28727 if (try temp.toReg(reg, cg)) break;
28728 } else break;
28729 try cg.asmOpOnly(.{ .@"rep _sb", .sto });
28730 }
28731
2855528732 fn moveTo(temp: Temp, inst: Air.Inst.Index, cg: *CodeGen) !void {
2855628733 if (cg.liveness.isUnused(inst)) try temp.die(cg) else switch (temp.unwrap(cg)) {
2855728734 .ref => {
test/behavior/basic.zig+1-1
......@@ -1169,10 +1169,10 @@ test "arrays and vectors with big integers" {
11691169 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
11701170 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
11711171 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
1172 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
11731172 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
11741173 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
11751174 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1175 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
11761176
11771177 inline for (.{ u65528, u65529, u65535 }) |Int| {
11781178 var a: [1]Int = undefined;