authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-12 02:11:37-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-15 03:07:51-04:00
logf83ebd8e6c95cf37d498936cae26d3a743cddc7f
tree79c213371dcb42883fb1b3f625b39336db4023a5
parent3681da25f865d499cffe923b7f0721cf759d3591

x86_64: implement stack probing

4 files changed, 163 insertions(+), 11 deletions(-)

src/arch/x86_64/CodeGen.zig+54-9
...@@ -1550,7 +1550,9 @@ fn gen(self: *Self) InnerError!void {...@@ -1550,7 +1550,9 @@ fn gen(self: *Self) InnerError!void {
1550 const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();1550 const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
1551 try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);1551 try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
1552 const backpatch_frame_align = try self.asmPlaceholder();1552 const backpatch_frame_align = try self.asmPlaceholder();
1553 const backpatch_frame_align_extra = try self.asmPlaceholder();
1553 const backpatch_stack_alloc = try self.asmPlaceholder();1554 const backpatch_stack_alloc = try self.asmPlaceholder();
1555 const backpatch_stack_alloc_extra = try self.asmPlaceholder();
15541556
1555 switch (self.ret_mcv.long) {1557 switch (self.ret_mcv.long) {
1556 .none, .unreach => {},1558 .none, .unreach => {},
...@@ -1599,24 +1601,67 @@ fn gen(self: *Self) InnerError!void {...@@ -1599,24 +1601,67 @@ fn gen(self: *Self) InnerError!void {
1599 const need_stack_adjust = frame_layout.stack_adjust > 0;1601 const need_stack_adjust = frame_layout.stack_adjust > 0;
1600 const need_save_reg = frame_layout.save_reg_list.count() > 0;1602 const need_save_reg = frame_layout.save_reg_list.count() > 0;
1601 if (need_frame_align) {1603 if (need_frame_align) {
1604 const page_align = @as(u32, math.maxInt(u32)) << 12;
1602 self.mir_instructions.set(backpatch_frame_align, .{1605 self.mir_instructions.set(backpatch_frame_align, .{
1603 .tag = .@"and",1606 .tag = .@"and",
1604 .ops = .ri_s,1607 .ops = .ri_s,
1605 .data = .{ .ri = .{1608 .data = .{ .ri = .{
1606 .r1 = .rsp,1609 .r1 = .rsp,
1607 .i = frame_layout.stack_mask,1610 .i = @max(frame_layout.stack_mask, page_align),
1608 } },1611 } },
1609 });1612 });
1613 if (frame_layout.stack_mask < page_align) {
1614 self.mir_instructions.set(backpatch_frame_align_extra, .{
1615 .tag = .pseudo,
1616 .ops = .pseudo_probe_align_ri_s,
1617 .data = .{ .ri = .{
1618 .r1 = .rsp,
1619 .i = ~frame_layout.stack_mask & page_align,
1620 } },
1621 });
1622 }
1610 }1623 }
1611 if (need_stack_adjust) {1624 if (need_stack_adjust) {
1612 self.mir_instructions.set(backpatch_stack_alloc, .{1625 const page_size: u32 = 1 << 12;
1613 .tag = .sub,1626 if (frame_layout.stack_adjust <= page_size) {
1614 .ops = .ri_s,1627 self.mir_instructions.set(backpatch_stack_alloc, .{
1615 .data = .{ .ri = .{1628 .tag = .sub,
1616 .r1 = .rsp,1629 .ops = .ri_s,
1617 .i = frame_layout.stack_adjust,1630 .data = .{ .ri = .{
1618 } },1631 .r1 = .rsp,
1619 });1632 .i = frame_layout.stack_adjust,
1633 } },
1634 });
1635 } else if (frame_layout.stack_adjust <
1636 page_size * Lower.pseudo_probe_adjust_unrolled_max_insts)
1637 {
1638 self.mir_instructions.set(backpatch_stack_alloc, .{
1639 .tag = .pseudo,
1640 .ops = .pseudo_probe_adjust_unrolled_ri_s,
1641 .data = .{ .ri = .{
1642 .r1 = .rsp,
1643 .i = frame_layout.stack_adjust,
1644 } },
1645 });
1646 } else {
1647 self.mir_instructions.set(backpatch_stack_alloc, .{
1648 .tag = .pseudo,
1649 .ops = .pseudo_probe_adjust_setup_rri_s,
1650 .data = .{ .rri = .{
1651 .r1 = .rsp,
1652 .r2 = .rax,
1653 .i = frame_layout.stack_adjust,
1654 } },
1655 });
1656 self.mir_instructions.set(backpatch_stack_alloc_extra, .{
1657 .tag = .pseudo,
1658 .ops = .pseudo_probe_adjust_loop_rr,
1659 .data = .{ .rr = .{
1660 .r1 = .rsp,
1661 .r2 = .rax,
1662 } },
1663 });
1664 }
1620 }1665 }
1621 if (need_frame_align or need_stack_adjust) {1666 if (need_frame_align or need_stack_adjust) {
1622 self.mir_instructions.set(backpatch_stack_dealloc, .{1667 self.mir_instructions.set(backpatch_stack_dealloc, .{
src/arch/x86_64/Lower.zig+97
...@@ -9,19 +9,33 @@ result_insts_len: u8 = undefined,...@@ -9,19 +9,33 @@ result_insts_len: u8 = undefined,
9result_relocs_len: u8 = undefined,9result_relocs_len: u8 = undefined,
10result_insts: [10result_insts: [
11 std.mem.max(usize, &.{11 std.mem.max(usize, &.{
12 1, // non-pseudo instructions
12 2, // cmovcc: cmovcc \ cmovcc13 2, // cmovcc: cmovcc \ cmovcc
13 3, // setcc: setcc \ setcc \ logicop14 3, // setcc: setcc \ setcc \ logicop
14 2, // jcc: jcc \ jcc15 2, // jcc: jcc \ jcc
16 pseudo_probe_align_insts,
17 pseudo_probe_adjust_unrolled_max_insts,
18 pseudo_probe_adjust_setup_insts,
19 pseudo_probe_adjust_loop_insts,
15 abi.Win64.callee_preserved_regs.len, // push_regs/pop_regs20 abi.Win64.callee_preserved_regs.len, // push_regs/pop_regs
16 abi.SysV.callee_preserved_regs.len, // push_regs/pop_regs21 abi.SysV.callee_preserved_regs.len, // push_regs/pop_regs
17 })22 })
18]Instruction = undefined,23]Instruction = undefined,
19result_relocs: [24result_relocs: [
20 std.mem.max(usize, &.{25 std.mem.max(usize, &.{
26 1, // jmp/jcc/call/mov/lea: jmp/jcc/call/mov/lea
21 2, // jcc: jcc \ jcc27 2, // jcc: jcc \ jcc
28 2, // test \ jcc \ probe \ sub \ jmp
29 1, // probe \ sub \ jcc
22 })30 })
23]Reloc = undefined,31]Reloc = undefined,
2432
33pub const pseudo_probe_align_insts = 5; // test \ jcc \ probe \ sub \ jmp
34pub const pseudo_probe_adjust_unrolled_max_insts =
35 pseudo_probe_adjust_setup_insts + pseudo_probe_adjust_loop_insts;
36pub const pseudo_probe_adjust_setup_insts = 2; // mov \ sub
37pub const pseudo_probe_adjust_loop_insts = 3; // probe \ sub \ jcc
38
25pub const Error = error{39pub const Error = error{
26 OutOfMemory,40 OutOfMemory,
27 LowerFail,41 LowerFail,
...@@ -62,6 +76,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -62,6 +76,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
62 else => try lower.generic(inst),76 else => try lower.generic(inst),
63 .pseudo => switch (inst.ops) {77 .pseudo => switch (inst.ops) {
64 .pseudo_cmov_z_and_np_rr => {78 .pseudo_cmov_z_and_np_rr => {
79 assert(inst.data.rr.fixes == ._);
65 try lower.emit(.none, .cmovnz, &.{80 try lower.emit(.none, .cmovnz, &.{
66 .{ .reg = inst.data.rr.r2 },81 .{ .reg = inst.data.rr.r2 },
67 .{ .reg = inst.data.rr.r1 },82 .{ .reg = inst.data.rr.r1 },
...@@ -72,6 +87,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -72,6 +87,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
72 });87 });
73 },88 },
74 .pseudo_cmov_nz_or_p_rr => {89 .pseudo_cmov_nz_or_p_rr => {
90 assert(inst.data.rr.fixes == ._);
75 try lower.emit(.none, .cmovnz, &.{91 try lower.emit(.none, .cmovnz, &.{
76 .{ .reg = inst.data.rr.r1 },92 .{ .reg = inst.data.rr.r1 },
77 .{ .reg = inst.data.rr.r2 },93 .{ .reg = inst.data.rr.r2 },
...@@ -84,6 +100,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -84,6 +100,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
84 .pseudo_cmov_nz_or_p_rm_sib,100 .pseudo_cmov_nz_or_p_rm_sib,
85 .pseudo_cmov_nz_or_p_rm_rip,101 .pseudo_cmov_nz_or_p_rm_rip,
86 => {102 => {
103 assert(inst.data.rx.fixes == ._);
87 try lower.emit(.none, .cmovnz, &.{104 try lower.emit(.none, .cmovnz, &.{
88 .{ .reg = inst.data.rx.r1 },105 .{ .reg = inst.data.rx.r1 },
89 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },106 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
...@@ -94,6 +111,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -94,6 +111,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
94 });111 });
95 },112 },
96 .pseudo_set_z_and_np_r => {113 .pseudo_set_z_and_np_r => {
114 assert(inst.data.rr.fixes == ._);
97 try lower.emit(.none, .setz, &.{115 try lower.emit(.none, .setz, &.{
98 .{ .reg = inst.data.rr.r1 },116 .{ .reg = inst.data.rr.r1 },
99 });117 });
...@@ -108,6 +126,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -108,6 +126,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
108 .pseudo_set_z_and_np_m_sib,126 .pseudo_set_z_and_np_m_sib,
109 .pseudo_set_z_and_np_m_rip,127 .pseudo_set_z_and_np_m_rip,
110 => {128 => {
129 assert(inst.data.rx.fixes == ._);
111 try lower.emit(.none, .setz, &.{130 try lower.emit(.none, .setz, &.{
112 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },131 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
113 });132 });
...@@ -120,6 +139,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -120,6 +139,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
120 });139 });
121 },140 },
122 .pseudo_set_nz_or_p_r => {141 .pseudo_set_nz_or_p_r => {
142 assert(inst.data.rr.fixes == ._);
123 try lower.emit(.none, .setnz, &.{143 try lower.emit(.none, .setnz, &.{
124 .{ .reg = inst.data.rr.r1 },144 .{ .reg = inst.data.rr.r1 },
125 });145 });
...@@ -134,6 +154,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -134,6 +154,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
134 .pseudo_set_nz_or_p_m_sib,154 .pseudo_set_nz_or_p_m_sib,
135 .pseudo_set_nz_or_p_m_rip,155 .pseudo_set_nz_or_p_m_rip,
136 => {156 => {
157 assert(inst.data.rx.fixes == ._);
137 try lower.emit(.none, .setnz, &.{158 try lower.emit(.none, .setnz, &.{
138 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },159 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
139 });160 });
...@@ -146,6 +167,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -146,6 +167,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
146 });167 });
147 },168 },
148 .pseudo_j_z_and_np_inst => {169 .pseudo_j_z_and_np_inst => {
170 assert(inst.data.inst.fixes == ._);
149 try lower.emit(.none, .jnz, &.{171 try lower.emit(.none, .jnz, &.{
150 .{ .imm = lower.reloc(.{ .inst = index + 1 }) },172 .{ .imm = lower.reloc(.{ .inst = index + 1 }) },
151 });173 });
...@@ -154,6 +176,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -154,6 +176,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
154 });176 });
155 },177 },
156 .pseudo_j_nz_or_p_inst => {178 .pseudo_j_nz_or_p_inst => {
179 assert(inst.data.inst.fixes == ._);
157 try lower.emit(.none, .jnz, &.{180 try lower.emit(.none, .jnz, &.{
158 .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },181 .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
159 });182 });
...@@ -162,6 +185,78 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -162,6 +185,78 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
162 });185 });
163 },186 },
164187
188 .pseudo_probe_align_ri_s => {
189 try lower.emit(.none, .@"test", &.{
190 .{ .reg = inst.data.ri.r1 },
191 .{ .imm = Immediate.s(@bitCast(i32, inst.data.ri.i)) },
192 });
193 try lower.emit(.none, .jz, &.{
194 .{ .imm = lower.reloc(.{ .inst = index + 1 }) },
195 });
196 try lower.emit(.none, .lea, &.{
197 .{ .reg = inst.data.ri.r1 },
198 .{ .mem = Memory.sib(.qword, .{
199 .base = .{ .reg = inst.data.ri.r1 },
200 .disp = -page_size,
201 }) },
202 });
203 try lower.emit(.none, .@"test", &.{
204 .{ .mem = Memory.sib(.dword, .{
205 .base = .{ .reg = inst.data.ri.r1 },
206 }) },
207 .{ .reg = inst.data.ri.r1.to32() },
208 });
209 try lower.emit(.none, .jmp, &.{
210 .{ .imm = lower.reloc(.{ .inst = index }) },
211 });
212 assert(lower.result_insts_len == pseudo_probe_align_insts);
213 },
214 .pseudo_probe_adjust_unrolled_ri_s => {
215 var offset = page_size;
216 while (offset < @bitCast(i32, inst.data.ri.i)) : (offset += page_size) {
217 try lower.emit(.none, .@"test", &.{
218 .{ .mem = Memory.sib(.dword, .{
219 .base = .{ .reg = inst.data.ri.r1 },
220 .disp = -offset,
221 }) },
222 .{ .reg = inst.data.ri.r1.to32() },
223 });
224 }
225 try lower.emit(.none, .sub, &.{
226 .{ .reg = inst.data.ri.r1 },
227 .{ .imm = Immediate.s(@bitCast(i32, inst.data.ri.i)) },
228 });
229 assert(lower.result_insts_len <= pseudo_probe_adjust_unrolled_max_insts);
230 },
231 .pseudo_probe_adjust_setup_rri_s => {
232 try lower.emit(.none, .mov, &.{
233 .{ .reg = inst.data.rri.r2.to32() },
234 .{ .imm = Immediate.s(@bitCast(i32, inst.data.rri.i)) },
235 });
236 try lower.emit(.none, .sub, &.{
237 .{ .reg = inst.data.rri.r1 },
238 .{ .reg = inst.data.rri.r2 },
239 });
240 assert(lower.result_insts_len == pseudo_probe_adjust_setup_insts);
241 },
242 .pseudo_probe_adjust_loop_rr => {
243 try lower.emit(.none, .@"test", &.{
244 .{ .mem = Memory.sib(.dword, .{
245 .base = .{ .reg = inst.data.rr.r1 },
246 .scale_index = .{ .scale = 1, .index = inst.data.rr.r2 },
247 .disp = -page_size,
248 }) },
249 .{ .reg = inst.data.rr.r1.to32() },
250 });
251 try lower.emit(.none, .sub, &.{
252 .{ .reg = inst.data.rr.r2 },
253 .{ .imm = Immediate.s(page_size) },
254 });
255 try lower.emit(.none, .jae, &.{
256 .{ .imm = lower.reloc(.{ .inst = index }) },
257 });
258 assert(lower.result_insts_len == pseudo_probe_adjust_loop_insts);
259 },
165 .pseudo_push_reg_list => try lower.pushPopRegList(.push, inst),260 .pseudo_push_reg_list => try lower.pushPopRegList(.push, inst),
166 .pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst),261 .pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst),
167262
...@@ -440,6 +535,8 @@ fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Er...@@ -440,6 +535,8 @@ fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Er
440 }});535 }});
441}536}
442537
538const page_size: i32 = 1 << 12;
539
443const abi = @import("abi.zig");540const abi = @import("abi.zig");
444const assert = std.debug.assert;541const assert = std.debug.assert;
445const bits = @import("bits.zig");542const bits = @import("bits.zig");
src/arch/x86_64/Mir.zig+12
...@@ -740,6 +740,18 @@ pub const Inst = struct {...@@ -740,6 +740,18 @@ pub const Inst = struct {
740 /// Uses `inst` payload.740 /// Uses `inst` payload.
741 pseudo_j_nz_or_p_inst,741 pseudo_j_nz_or_p_inst,
742742
743 /// Probe alignment
744 /// Uses `ri` payload
745 pseudo_probe_align_ri_s,
746 /// Probe adjust unrolled
747 /// Uses `ri` payload
748 pseudo_probe_adjust_unrolled_ri_s,
749 /// Probe adjust setup
750 /// Uses `rri` payload
751 pseudo_probe_adjust_setup_rri_s,
752 /// Probe adjust loop
753 /// Uses `rr` payload
754 pseudo_probe_adjust_loop_rr,
743 /// Push registers755 /// Push registers
744 /// Uses `reg_list` payload.756 /// Uses `reg_list` payload.
745 pseudo_push_reg_list,757 pseudo_push_reg_list,
test/behavior/memset.zig-2
...@@ -120,7 +120,6 @@ test "memset with large array element, runtime known" {...@@ -120,7 +120,6 @@ test "memset with large array element, runtime known" {
120 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;120 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
121 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;121 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
122 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;122 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
123 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
124123
125 const A = [128]u64;124 const A = [128]u64;
126 var buf: [5]A = undefined;125 var buf: [5]A = undefined;
...@@ -139,7 +138,6 @@ test "memset with large array element, comptime known" {...@@ -139,7 +138,6 @@ test "memset with large array element, comptime known" {
139 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;138 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
140 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;139 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
141 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;140 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
142 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
143141
144 const A = [128]u64;142 const A = [128]u64;
145 var buf: [5]A = undefined;143 var buf: [5]A = undefined;