authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-12 02:11:37-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-15 03:07:51-04:00
logf83ebd8e6c95cf37d498936cae26d3a743cddc7f
tree79c213371dcb42883fb1b3f625b39336db4023a5
parent3681da25f865d499cffe923b7f0721cf759d3591

x86_64: implement stack probing

4 files changed, 163 insertions(+), 11 deletions(-)

src/arch/x86_64/CodeGen.zig+54-9
......@@ -1550,7 +1550,9 @@ fn gen(self: *Self) InnerError!void {
15501550 const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
15511551 try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
15521552 const backpatch_frame_align = try self.asmPlaceholder();
1553 const backpatch_frame_align_extra = try self.asmPlaceholder();
15531554 const backpatch_stack_alloc = try self.asmPlaceholder();
1555 const backpatch_stack_alloc_extra = try self.asmPlaceholder();
15541556
15551557 switch (self.ret_mcv.long) {
15561558 .none, .unreach => {},
......@@ -1599,24 +1601,67 @@ fn gen(self: *Self) InnerError!void {
15991601 const need_stack_adjust = frame_layout.stack_adjust > 0;
16001602 const need_save_reg = frame_layout.save_reg_list.count() > 0;
16011603 if (need_frame_align) {
1604 const page_align = @as(u32, math.maxInt(u32)) << 12;
16021605 self.mir_instructions.set(backpatch_frame_align, .{
16031606 .tag = .@"and",
16041607 .ops = .ri_s,
16051608 .data = .{ .ri = .{
16061609 .r1 = .rsp,
1607 .i = frame_layout.stack_mask,
1610 .i = @max(frame_layout.stack_mask, page_align),
16081611 } },
16091612 });
1613 if (frame_layout.stack_mask < page_align) {
1614 self.mir_instructions.set(backpatch_frame_align_extra, .{
1615 .tag = .pseudo,
1616 .ops = .pseudo_probe_align_ri_s,
1617 .data = .{ .ri = .{
1618 .r1 = .rsp,
1619 .i = ~frame_layout.stack_mask & page_align,
1620 } },
1621 });
1622 }
16101623 }
16111624 if (need_stack_adjust) {
1612 self.mir_instructions.set(backpatch_stack_alloc, .{
1613 .tag = .sub,
1614 .ops = .ri_s,
1615 .data = .{ .ri = .{
1616 .r1 = .rsp,
1617 .i = frame_layout.stack_adjust,
1618 } },
1619 });
1625 const page_size: u32 = 1 << 12;
1626 if (frame_layout.stack_adjust <= page_size) {
1627 self.mir_instructions.set(backpatch_stack_alloc, .{
1628 .tag = .sub,
1629 .ops = .ri_s,
1630 .data = .{ .ri = .{
1631 .r1 = .rsp,
1632 .i = frame_layout.stack_adjust,
1633 } },
1634 });
1635 } else if (frame_layout.stack_adjust <
1636 page_size * Lower.pseudo_probe_adjust_unrolled_max_insts)
1637 {
1638 self.mir_instructions.set(backpatch_stack_alloc, .{
1639 .tag = .pseudo,
1640 .ops = .pseudo_probe_adjust_unrolled_ri_s,
1641 .data = .{ .ri = .{
1642 .r1 = .rsp,
1643 .i = frame_layout.stack_adjust,
1644 } },
1645 });
1646 } else {
1647 self.mir_instructions.set(backpatch_stack_alloc, .{
1648 .tag = .pseudo,
1649 .ops = .pseudo_probe_adjust_setup_rri_s,
1650 .data = .{ .rri = .{
1651 .r1 = .rsp,
1652 .r2 = .rax,
1653 .i = frame_layout.stack_adjust,
1654 } },
1655 });
1656 self.mir_instructions.set(backpatch_stack_alloc_extra, .{
1657 .tag = .pseudo,
1658 .ops = .pseudo_probe_adjust_loop_rr,
1659 .data = .{ .rr = .{
1660 .r1 = .rsp,
1661 .r2 = .rax,
1662 } },
1663 });
1664 }
16201665 }
16211666 if (need_frame_align or need_stack_adjust) {
16221667 self.mir_instructions.set(backpatch_stack_dealloc, .{
src/arch/x86_64/Lower.zig+97
......@@ -9,19 +9,33 @@ result_insts_len: u8 = undefined,
99result_relocs_len: u8 = undefined,
1010result_insts: [
1111 std.mem.max(usize, &.{
12 1, // non-pseudo instructions
1213 2, // cmovcc: cmovcc \ cmovcc
1314 3, // setcc: setcc \ setcc \ logicop
1415 2, // jcc: jcc \ jcc
16 pseudo_probe_align_insts,
17 pseudo_probe_adjust_unrolled_max_insts,
18 pseudo_probe_adjust_setup_insts,
19 pseudo_probe_adjust_loop_insts,
1520 abi.Win64.callee_preserved_regs.len, // push_regs/pop_regs
1621 abi.SysV.callee_preserved_regs.len, // push_regs/pop_regs
1722 })
1823]Instruction = undefined,
1924result_relocs: [
2025 std.mem.max(usize, &.{
26 1, // jmp/jcc/call/mov/lea: jmp/jcc/call/mov/lea
2127 2, // jcc: jcc \ jcc
28 2, // test \ jcc \ probe \ sub \ jmp
29 1, // probe \ sub \ jcc
2230 })
2331]Reloc = undefined,
2432
33pub const pseudo_probe_align_insts = 5; // test \ jcc \ probe \ sub \ jmp
34pub const pseudo_probe_adjust_unrolled_max_insts =
35 pseudo_probe_adjust_setup_insts + pseudo_probe_adjust_loop_insts;
36pub const pseudo_probe_adjust_setup_insts = 2; // mov \ sub
37pub const pseudo_probe_adjust_loop_insts = 3; // probe \ sub \ jcc
38
2539pub const Error = error{
2640 OutOfMemory,
2741 LowerFail,
......@@ -62,6 +76,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
6276 else => try lower.generic(inst),
6377 .pseudo => switch (inst.ops) {
6478 .pseudo_cmov_z_and_np_rr => {
79 assert(inst.data.rr.fixes == ._);
6580 try lower.emit(.none, .cmovnz, &.{
6681 .{ .reg = inst.data.rr.r2 },
6782 .{ .reg = inst.data.rr.r1 },
......@@ -72,6 +87,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
7287 });
7388 },
7489 .pseudo_cmov_nz_or_p_rr => {
90 assert(inst.data.rr.fixes == ._);
7591 try lower.emit(.none, .cmovnz, &.{
7692 .{ .reg = inst.data.rr.r1 },
7793 .{ .reg = inst.data.rr.r2 },
......@@ -84,6 +100,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
84100 .pseudo_cmov_nz_or_p_rm_sib,
85101 .pseudo_cmov_nz_or_p_rm_rip,
86102 => {
103 assert(inst.data.rx.fixes == ._);
87104 try lower.emit(.none, .cmovnz, &.{
88105 .{ .reg = inst.data.rx.r1 },
89106 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
......@@ -94,6 +111,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
94111 });
95112 },
96113 .pseudo_set_z_and_np_r => {
114 assert(inst.data.rr.fixes == ._);
97115 try lower.emit(.none, .setz, &.{
98116 .{ .reg = inst.data.rr.r1 },
99117 });
......@@ -108,6 +126,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
108126 .pseudo_set_z_and_np_m_sib,
109127 .pseudo_set_z_and_np_m_rip,
110128 => {
129 assert(inst.data.rx.fixes == ._);
111130 try lower.emit(.none, .setz, &.{
112131 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
113132 });
......@@ -120,6 +139,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
120139 });
121140 },
122141 .pseudo_set_nz_or_p_r => {
142 assert(inst.data.rr.fixes == ._);
123143 try lower.emit(.none, .setnz, &.{
124144 .{ .reg = inst.data.rr.r1 },
125145 });
......@@ -134,6 +154,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
134154 .pseudo_set_nz_or_p_m_sib,
135155 .pseudo_set_nz_or_p_m_rip,
136156 => {
157 assert(inst.data.rx.fixes == ._);
137158 try lower.emit(.none, .setnz, &.{
138159 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
139160 });
......@@ -146,6 +167,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
146167 });
147168 },
148169 .pseudo_j_z_and_np_inst => {
170 assert(inst.data.inst.fixes == ._);
149171 try lower.emit(.none, .jnz, &.{
150172 .{ .imm = lower.reloc(.{ .inst = index + 1 }) },
151173 });
......@@ -154,6 +176,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
154176 });
155177 },
156178 .pseudo_j_nz_or_p_inst => {
179 assert(inst.data.inst.fixes == ._);
157180 try lower.emit(.none, .jnz, &.{
158181 .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
159182 });
......@@ -162,6 +185,78 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
162185 });
163186 },
164187
188 .pseudo_probe_align_ri_s => {
189 try lower.emit(.none, .@"test", &.{
190 .{ .reg = inst.data.ri.r1 },
191 .{ .imm = Immediate.s(@bitCast(i32, inst.data.ri.i)) },
192 });
193 try lower.emit(.none, .jz, &.{
194 .{ .imm = lower.reloc(.{ .inst = index + 1 }) },
195 });
196 try lower.emit(.none, .lea, &.{
197 .{ .reg = inst.data.ri.r1 },
198 .{ .mem = Memory.sib(.qword, .{
199 .base = .{ .reg = inst.data.ri.r1 },
200 .disp = -page_size,
201 }) },
202 });
203 try lower.emit(.none, .@"test", &.{
204 .{ .mem = Memory.sib(.dword, .{
205 .base = .{ .reg = inst.data.ri.r1 },
206 }) },
207 .{ .reg = inst.data.ri.r1.to32() },
208 });
209 try lower.emit(.none, .jmp, &.{
210 .{ .imm = lower.reloc(.{ .inst = index }) },
211 });
212 assert(lower.result_insts_len == pseudo_probe_align_insts);
213 },
214 .pseudo_probe_adjust_unrolled_ri_s => {
215 var offset = page_size;
216 while (offset < @bitCast(i32, inst.data.ri.i)) : (offset += page_size) {
217 try lower.emit(.none, .@"test", &.{
218 .{ .mem = Memory.sib(.dword, .{
219 .base = .{ .reg = inst.data.ri.r1 },
220 .disp = -offset,
221 }) },
222 .{ .reg = inst.data.ri.r1.to32() },
223 });
224 }
225 try lower.emit(.none, .sub, &.{
226 .{ .reg = inst.data.ri.r1 },
227 .{ .imm = Immediate.s(@bitCast(i32, inst.data.ri.i)) },
228 });
229 assert(lower.result_insts_len <= pseudo_probe_adjust_unrolled_max_insts);
230 },
231 .pseudo_probe_adjust_setup_rri_s => {
232 try lower.emit(.none, .mov, &.{
233 .{ .reg = inst.data.rri.r2.to32() },
234 .{ .imm = Immediate.s(@bitCast(i32, inst.data.rri.i)) },
235 });
236 try lower.emit(.none, .sub, &.{
237 .{ .reg = inst.data.rri.r1 },
238 .{ .reg = inst.data.rri.r2 },
239 });
240 assert(lower.result_insts_len == pseudo_probe_adjust_setup_insts);
241 },
242 .pseudo_probe_adjust_loop_rr => {
243 try lower.emit(.none, .@"test", &.{
244 .{ .mem = Memory.sib(.dword, .{
245 .base = .{ .reg = inst.data.rr.r1 },
246 .scale_index = .{ .scale = 1, .index = inst.data.rr.r2 },
247 .disp = -page_size,
248 }) },
249 .{ .reg = inst.data.rr.r1.to32() },
250 });
251 try lower.emit(.none, .sub, &.{
252 .{ .reg = inst.data.rr.r2 },
253 .{ .imm = Immediate.s(page_size) },
254 });
255 try lower.emit(.none, .jae, &.{
256 .{ .imm = lower.reloc(.{ .inst = index }) },
257 });
258 assert(lower.result_insts_len == pseudo_probe_adjust_loop_insts);
259 },
165260 .pseudo_push_reg_list => try lower.pushPopRegList(.push, inst),
166261 .pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst),
167262
......@@ -440,6 +535,8 @@ fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Er
440535 }});
441536}
442537
538const page_size: i32 = 1 << 12;
539
443540const abi = @import("abi.zig");
444541const assert = std.debug.assert;
445542const bits = @import("bits.zig");
src/arch/x86_64/Mir.zig+12
......@@ -740,6 +740,18 @@ pub const Inst = struct {
740740 /// Uses `inst` payload.
741741 pseudo_j_nz_or_p_inst,
742742
743 /// Probe alignment
744 /// Uses `ri` payload
745 pseudo_probe_align_ri_s,
746 /// Probe adjust unrolled
747 /// Uses `ri` payload
748 pseudo_probe_adjust_unrolled_ri_s,
749 /// Probe adjust setup
750 /// Uses `rri` payload
751 pseudo_probe_adjust_setup_rri_s,
752 /// Probe adjust loop
753 /// Uses `rr` payload
754 pseudo_probe_adjust_loop_rr,
743755 /// Push registers
744756 /// Uses `reg_list` payload.
745757 pseudo_push_reg_list,
test/behavior/memset.zig-2
......@@ -120,7 +120,6 @@ test "memset with large array element, runtime known" {
120120 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
121121 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
122122 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
123 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
124123
125124 const A = [128]u64;
126125 var buf: [5]A = undefined;
......@@ -139,7 +138,6 @@ test "memset with large array element, comptime known" {
139138 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
140139 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
141140 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
142 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
143141
144142 const A = [128]u64;
145143 var buf: [5]A = undefined;