authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-27 15:45:01-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-04-27 15:45:01-07:00
log011bc59e8aa4b62a286e4dd6e92371af5dac8b9f
tree9205ef1625f18435be9dfbffcc709dee73b27724
parentaa3405aabc539c66a9f110bc044924db07e892b6
parentf56054d129d19a006ff19c56b48bc9c7d9c04593
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15435 from jacobly0/x86_64-frame

x86_64: add frame indices

27 files changed, 2751 insertions(+), 2647 deletions(-)

lib/test_runner.zig+20-4
...@@ -233,14 +233,30 @@ pub fn log(...@@ -233,14 +233,30 @@ pub fn log(
233/// Simpler main(), exercising fewer language features, so that233/// Simpler main(), exercising fewer language features, so that
234/// work-in-progress backends can handle it.234/// work-in-progress backends can handle it.
235pub fn mainSimple() anyerror!void {235pub fn mainSimple() anyerror!void {
236 //const stderr = std.io.getStdErr();236 const enable_print = false;
237
238 var passed: u64 = 0;
239 var skipped: u64 = 0;
240 var failed: u64 = 0;
241 const stderr = if (enable_print) std.io.getStdErr() else {};
237 for (builtin.test_functions) |test_fn| {242 for (builtin.test_functions) |test_fn| {
238 test_fn.func() catch |err| {243 test_fn.func() catch |err| {
244 if (enable_print) stderr.writeAll(test_fn.name) catch {};
239 if (err != error.SkipZigTest) {245 if (err != error.SkipZigTest) {
240 //stderr.writeAll(test_fn.name) catch {};246 if (enable_print) stderr.writeAll("... FAIL\n") catch {};
241 //stderr.writeAll("\n") catch {};247 failed += 1;
242 return err;248 if (!enable_print) return err;
249 continue;
243 }250 }
251 if (enable_print) stderr.writeAll("... SKIP\n") catch {};
252 skipped += 1;
253 continue;
244 };254 };
255 //if (enable_print) stderr.writeAll("... PASS\n") catch {};
256 passed += 1;
257 }
258 if (enable_print) {
259 stderr.writer().print("{} passed, {} skipped, {} failed\n", .{ passed, skipped, failed }) catch {};
260 if (failed != 0) std.process.exit(1);
245 }261 }
246}262}
src/arch/x86_64/CodeGen.zig+2310-2389
...@@ -6,6 +6,7 @@ const codegen = @import("../../codegen.zig");...@@ -6,6 +6,7 @@ const codegen = @import("../../codegen.zig");
6const leb128 = std.leb;6const leb128 = std.leb;
7const link = @import("../../link.zig");7const link = @import("../../link.zig");
8const log = std.log.scoped(.codegen);8const log = std.log.scoped(.codegen);
9const tracking_log = std.log.scoped(.tracking);
9const math = std.math;10const math = std.math;
10const mem = std.mem;11const mem = std.mem;
11const trace = @import("../../tracy.zig").trace;12const trace = @import("../../tracy.zig").trace;
...@@ -40,6 +41,7 @@ const Memory = bits.Memory;...@@ -40,6 +41,7 @@ const Memory = bits.Memory;
40const Register = bits.Register;41const Register = bits.Register;
41const RegisterManager = abi.RegisterManager;42const RegisterManager = abi.RegisterManager;
42const RegisterLock = RegisterManager.RegisterLock;43const RegisterLock = RegisterManager.RegisterLock;
44const FrameIndex = bits.FrameIndex;
4345
44const gp = abi.RegisterClass.gp;46const gp = abi.RegisterClass.gp;
45const sse = abi.RegisterClass.sse;47const sse = abi.RegisterClass.sse;
...@@ -47,6 +49,7 @@ const sse = abi.RegisterClass.sse;...@@ -47,6 +49,7 @@ const sse = abi.RegisterClass.sse;
47const InnerError = CodeGenError || error{OutOfRegisters};49const InnerError = CodeGenError || error{OutOfRegisters};
4850
49const debug_wip_mir = false;51const debug_wip_mir = false;
52const debug_tracking = false;
5053
51gpa: Allocator,54gpa: Allocator,
52air: Air,55air: Air,
...@@ -57,11 +60,10 @@ target: *const std.Target,...@@ -57,11 +60,10 @@ target: *const std.Target,
57mod_fn: *const Module.Fn,60mod_fn: *const Module.Fn,
58err_msg: ?*ErrorMsg,61err_msg: ?*ErrorMsg,
59args: []MCValue,62args: []MCValue,
60ret_mcv: MCValue,63ret_mcv: InstTracking,
61fn_type: Type,64fn_type: Type,
62arg_index: u32,65arg_index: u32,
63src_loc: Module.SrcLoc,66src_loc: Module.SrcLoc,
64stack_align: u32,
6567
66eflags_inst: ?Air.Inst.Index = null,68eflags_inst: ?Air.Inst.Index = null,
6769
...@@ -86,17 +88,13 @@ inst_tracking: InstTrackingMap = .{},...@@ -86,17 +88,13 @@ inst_tracking: InstTrackingMap = .{},
86blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},88blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
8789
88register_manager: RegisterManager = .{},90register_manager: RegisterManager = .{},
89/// Maps offset to what is stored there.
90stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
9191
92/// Generation of the current scope, increments by 1 for every entered scope.92/// Generation of the current scope, increments by 1 for every entered scope.
93scope_generation: u32 = 0,93scope_generation: u32 = 0,
9494
95/// Offset from the stack base, representing the end of the stack frame.95frame_allocs: std.MultiArrayList(FrameAlloc) = .{},
96max_end_stack: u32 = 0,96free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .{},
97/// Represents the current end stack offset. If there is no existing slot97frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{},
98/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
99next_stack_offset: u32 = 0,
10098
101/// Debug field, used to find bugs in the compiler.99/// Debug field, used to find bugs in the compiler.
102air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,100air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
...@@ -108,6 +106,9 @@ const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}...@@ -108,6 +106,9 @@ const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}
108106
109const mir_to_air_map_init = if (builtin.mode == .Debug) std.AutoHashMapUnmanaged(Mir.Inst.Index, Air.Inst.Index){} else {};107const mir_to_air_map_init = if (builtin.mode == .Debug) std.AutoHashMapUnmanaged(Mir.Inst.Index, Air.Inst.Index){} else {};
110108
109const FrameAddr = struct { index: FrameIndex, off: i32 = 0 };
110const RegisterOffset = struct { reg: Register, off: i32 = 0 };
111
111pub const MCValue = union(enum) {112pub const MCValue = union(enum) {
112 /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.113 /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
113 /// TODO Look into deleting this tag and using `dead` instead, since every use114 /// TODO Look into deleting this tag and using `dead` instead, since every use
...@@ -123,46 +124,59 @@ pub const MCValue = union(enum) {...@@ -123,46 +124,59 @@ pub const MCValue = union(enum) {
123 /// A pointer-sized integer that fits in a register.124 /// A pointer-sized integer that fits in a register.
124 /// If the type is a pointer, this is the pointer address in virtual address space.125 /// If the type is a pointer, this is the pointer address in virtual address space.
125 immediate: u64,126 immediate: u64,
126 /// The value is in a GP register.127 /// The value resides in the EFLAGS register.
128 eflags: Condition,
129 /// The value is in a register.
127 register: Register,130 register: Register,
131 /// The value is a constant offset from the value in a register.
132 register_offset: RegisterOffset,
128 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.133 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.
129 register_overflow: struct { reg: Register, eflags: Condition },134 register_overflow: struct { reg: Register, eflags: Condition },
130 /// The value is in memory at a hard-coded address.135 /// The value is in memory at a hard-coded address.
131 /// If the type is a pointer, it means the pointer address is at this memory location.136 /// If the type is a pointer, it means the pointer address is at this memory location.
132 memory: u64,137 memory: u64,
138 /// The value is in memory at a constant offset from the address in a register.
139 indirect: RegisterOffset,
133 /// The value is in memory.140 /// The value is in memory.
134 /// Payload is a symbol index.141 /// Payload is a symbol index.
135 load_direct: u32,142 load_direct: u32,
136 /// The value is a pointer to value in memory.143 /// The value is a pointer to a value in memory.
137 /// Payload is a symbol index.144 /// Payload is a symbol index.
138 lea_direct: u32,145 lea_direct: u32,
139 /// The value is in memory referenced indirectly via GOT.146 /// The value is in memory referenced indirectly via GOT.
140 /// Payload is a symbol index.147 /// Payload is a symbol index.
141 load_got: u32,148 load_got: u32,
149 /// The value is a pointer to a value referenced indirectly via GOT.
150 /// Payload is a symbol index.
151 lea_got: u32,
142 /// The value is a threadlocal variable.152 /// The value is a threadlocal variable.
143 /// Payload is a symbol index.153 /// Payload is a symbol index.
144 load_tlv: u32,154 load_tlv: u32,
145 /// The value is a pointer to threadlocal variable.155 /// The value is a pointer to a threadlocal variable.
146 /// Payload is a symbol index.156 /// Payload is a symbol index.
147 lea_tlv: u32,157 lea_tlv: u32,
148 /// The value is one of the stack variables.158 /// The value stored at an offset from a frame index
149 /// If the type is a pointer, it means the pointer address is in the stack at this offset.159 /// Payload is a frame address.
150 stack_offset: i32,160 load_frame: FrameAddr,
151 /// The value is a pointer to one of the stack variables (payload is stack offset).161 /// The address of an offset from a frame index
152 ptr_stack_offset: i32,162 /// Payload is a frame address.
153 /// The value resides in the EFLAGS register.163 lea_frame: FrameAddr,
154 eflags: Condition,164 /// This indicates that we have already allocated a frame index for this instruction,
165 /// but it has not been spilled there yet in the current control flow.
166 /// Payload is a frame index.
167 reserved_frame: FrameIndex,
155168
156 fn isMemory(mcv: MCValue) bool {169 fn isMemory(mcv: MCValue) bool {
157 return switch (mcv) {170 return switch (mcv) {
158 .memory,171 .memory,
159 .stack_offset,
160 .ptr_stack_offset,
161 .load_direct,172 .load_direct,
162 .lea_direct,173 .lea_direct,
163 .load_got,174 .load_got,
175 .lea_got,
164 .load_tlv,176 .load_tlv,
165 .lea_tlv,177 .lea_tlv,
178 .load_frame,
179 .lea_frame,
166 => true,180 => true,
167 else => false,181 else => false,
168 };182 };
...@@ -181,6 +195,134 @@ pub const MCValue = union(enum) {...@@ -181,6 +195,134 @@ pub const MCValue = union(enum) {
181 else => false,195 else => false,
182 };196 };
183 }197 }
198
199 fn getReg(mcv: MCValue) ?Register {
200 return switch (mcv) {
201 .register => |reg| reg,
202 .register_offset, .indirect => |ro| ro.reg,
203 .register_overflow => |ro| ro.reg,
204 else => null,
205 };
206 }
207
208 fn getCondition(mcv: MCValue) ?Condition {
209 return switch (mcv) {
210 .eflags => |cc| cc,
211 .register_overflow => |reg_ov| reg_ov.eflags,
212 else => null,
213 };
214 }
215
216 fn address(mcv: MCValue) MCValue {
217 return switch (mcv) {
218 .none,
219 .unreach,
220 .dead,
221 .undef,
222 .immediate,
223 .register,
224 .register_offset,
225 .eflags,
226 .register_overflow,
227 .lea_direct,
228 .lea_got,
229 .lea_tlv,
230 .lea_frame,
231 .reserved_frame,
232 => unreachable, // not in memory
233 .memory => |addr| .{ .immediate = addr },
234 .indirect => |reg_off| switch (reg_off.off) {
235 0 => .{ .register = reg_off.reg },
236 else => .{ .register_offset = reg_off },
237 },
238 .load_direct => |sym_index| .{ .lea_direct = sym_index },
239 .load_got => |sym_index| .{ .lea_got = sym_index },
240 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
241 .load_frame => |frame_addr| .{ .lea_frame = frame_addr },
242 };
243 }
244
245 fn deref(mcv: MCValue) MCValue {
246 return switch (mcv) {
247 .none,
248 .unreach,
249 .dead,
250 .undef,
251 .eflags,
252 .register_overflow,
253 .memory,
254 .indirect,
255 .load_direct,
256 .load_got,
257 .load_tlv,
258 .load_frame,
259 .reserved_frame,
260 => unreachable, // not a dereferenceable
261 .immediate => |addr| .{ .memory = addr },
262 .register => |reg| .{ .indirect = .{ .reg = reg } },
263 .register_offset => |reg_off| .{ .indirect = reg_off },
264 .lea_direct => |sym_index| .{ .load_direct = sym_index },
265 .lea_got => |sym_index| .{ .load_got = sym_index },
266 .lea_tlv => |sym_index| .{ .load_tlv = sym_index },
267 .lea_frame => |frame_addr| .{ .load_frame = frame_addr },
268 };
269 }
270
271 fn offset(mcv: MCValue, off: i32) MCValue {
272 return switch (mcv) {
273 .none,
274 .unreach,
275 .dead,
276 .undef,
277 .eflags,
278 .register_overflow,
279 .memory,
280 .indirect,
281 .load_direct,
282 .lea_direct,
283 .load_got,
284 .lea_got,
285 .load_tlv,
286 .lea_tlv,
287 .load_frame,
288 .reserved_frame,
289 => unreachable, // not offsettable
290 .immediate => |imm| .{ .immediate = @bitCast(u64, @bitCast(i64, imm) +% off) },
291 .register => |reg| .{ .register_offset = .{ .reg = reg, .off = off } },
292 .register_offset => |reg_off| .{
293 .register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off },
294 },
295 .lea_frame => |frame_addr| .{
296 .lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
297 },
298 };
299 }
300
301 pub fn format(
302 mcv: MCValue,
303 comptime _: []const u8,
304 _: std.fmt.FormatOptions,
305 writer: anytype,
306 ) @TypeOf(writer).Error!void {
307 switch (mcv) {
308 .none, .unreach, .dead, .undef => try writer.print("({s})", .{@tagName(mcv)}),
309 .immediate => |pl| try writer.print("0x{x}", .{pl}),
310 .memory => |pl| try writer.print("[ds:0x{x}]", .{pl}),
311 inline .eflags, .register => |pl| try writer.print("{s}", .{@tagName(pl)}),
312 .register_offset => |pl| try writer.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }),
313 .register_overflow => |pl| try writer.print("{s}:{s}", .{ @tagName(pl.eflags), @tagName(pl.reg) }),
314 .indirect => |pl| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
315 .load_direct => |pl| try writer.print("[direct:{d}]", .{pl}),
316 .lea_direct => |pl| try writer.print("direct:{d}", .{pl}),
317 .load_got => |pl| try writer.print("[got:{d}]", .{pl}),
318 .lea_got => |pl| try writer.print("got:{d}", .{pl}),
319 .load_tlv => |pl| try writer.print("[tlv:{d}]", .{pl}),
320 .lea_tlv => |pl| try writer.print("tlv:{d}", .{pl}),
321 .load_frame => |pl| try writer.print("[{} + 0x{x}]", .{ pl.index, pl.off }),
322 .lea_frame => |pl| try writer.print("{} + 0x{x}", .{ pl.index, pl.off }),
323 .reserved_frame => |pl| try writer.print("(dead:{})", .{pl}),
324 }
325 }
184};326};
185327
186const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking);328const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking);
...@@ -189,89 +331,211 @@ const InstTracking = struct {...@@ -189,89 +331,211 @@ const InstTracking = struct {
189 short: MCValue,331 short: MCValue,
190332
191 fn init(result: MCValue) InstTracking {333 fn init(result: MCValue) InstTracking {
192 return .{ .long = result, .short = result };334 return .{ .long = switch (result) {
335 .none,
336 .unreach,
337 .undef,
338 .immediate,
339 .memory,
340 .load_direct,
341 .lea_direct,
342 .load_got,
343 .lea_got,
344 .load_tlv,
345 .lea_tlv,
346 .load_frame,
347 .lea_frame,
348 => result,
349 .dead,
350 .reserved_frame,
351 => unreachable,
352 .eflags,
353 .register,
354 .register_offset,
355 .register_overflow,
356 .indirect,
357 => .none,
358 }, .short = result };
193 }359 }
194360
195 fn getReg(self: InstTracking) ?Register {361 fn getReg(self: InstTracking) ?Register {
196 return switch (self.short) {362 return self.short.getReg();
197 .register => |reg| reg,
198 .register_overflow => |ro| ro.reg,
199 else => null,
200 };
201 }363 }
202364
203 fn getCondition(self: InstTracking) ?Condition {365 fn getCondition(self: InstTracking) ?Condition {
204 return switch (self.short) {366 return self.short.getCondition();
205 .eflags => |eflags| eflags,
206 .register_overflow => |ro| ro.eflags,
207 else => null,
208 };
209 }367 }
210368
211 fn spill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {369 fn spill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {
370 if (std.meta.eql(self.long, self.short)) return; // Already spilled
371 // Allocate or reuse frame index
372 switch (self.long) {
373 .none => self.long = try function.allocRegOrMem(inst, false),
374 .load_frame => {},
375 .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } },
376 else => unreachable,
377 }
378 tracking_log.debug("spill %{d} from {} to {}", .{ inst, self.short, self.long });
379 try function.genCopy(function.air.typeOfIndex(inst), self.long, self.short);
380 }
381
382 fn reuseFrame(self: *InstTracking) void {
212 switch (self.long) {383 switch (self.long) {
384 .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } },
385 else => {},
386 }
387 self.short = switch (self.long) {
213 .none,388 .none,
214 .dead,
215 .unreach,389 .unreach,
216 => unreachable,390 .undef,
391 .immediate,
392 .memory,
393 .load_direct,
394 .lea_direct,
395 .load_got,
396 .lea_got,
397 .load_tlv,
398 .lea_tlv,
399 .load_frame,
400 .lea_frame,
401 => self.long,
402 .dead,
403 .eflags,
217 .register,404 .register,
405 .register_offset,
218 .register_overflow,406 .register_overflow,
219 .eflags,407 .indirect,
220 => self.long = try function.allocRegOrMem(inst, self.short == .eflags),408 .reserved_frame,
221 .stack_offset => {},409 => unreachable,
410 };
411 }
412
413 fn trackSpill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) void {
414 function.freeValue(self.short);
415 self.reuseFrame();
416 tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* });
417 }
418
419 fn verifyMaterialize(self: *InstTracking, target: InstTracking) void {
420 switch (self.long) {
421 .none,
422 .unreach,
222 .undef,423 .undef,
223 .immediate,424 .immediate,
224 .memory,425 .memory,
225 .load_direct,426 .load_direct,
226 .lea_direct,427 .lea_direct,
227 .load_got,428 .load_got,
429 .lea_got,
228 .load_tlv,430 .load_tlv,
229 .lea_tlv,431 .lea_tlv,
230 .ptr_stack_offset,432 .lea_frame,
231 => return, // these can be rematerialized without using a stack slot433 => assert(std.meta.eql(self.long, target.long)),
434 .load_frame,
435 .reserved_frame,
436 => switch (target.long) {
437 .none,
438 .load_frame,
439 .reserved_frame,
440 => {},
441 else => unreachable,
442 },
443 .dead,
444 .eflags,
445 .register,
446 .register_offset,
447 .register_overflow,
448 .indirect,
449 => unreachable,
232 }450 }
233 log.debug("spilling %{d} from {} to {}", .{ inst, self.short, self.long });
234 const ty = function.air.typeOfIndex(inst);
235 try function.setRegOrMem(ty, self.long, self.short);
236 }451 }
237452
238 fn trackSpill(self: *InstTracking, function: *Self) void {453 fn materialize(
239 if (self.getReg()) |reg| function.register_manager.freeReg(reg);454 self: *InstTracking,
240 switch (self.short) {455 function: *Self,
241 .none, .dead, .unreach => unreachable,456 inst: Air.Inst.Index,
242 else => {},457 target: InstTracking,
243 }458 ) !void {
244 self.short = self.long;459 self.verifyMaterialize(target);
460 try self.materializeUnsafe(function, inst, target);
245 }461 }
246462
247 fn materialize(self: *InstTracking, function: *Self, inst: Air.Inst.Index, reg: Register) !void {463 fn materializeUnsafe(
464 self: *InstTracking,
465 function: *Self,
466 inst: Air.Inst.Index,
467 target: InstTracking,
468 ) !void {
248 const ty = function.air.typeOfIndex(inst);469 const ty = function.air.typeOfIndex(inst);
249 try function.genSetReg(ty, reg, self.long);470 if ((self.long == .none or self.long == .reserved_frame) and target.long == .load_frame)
471 try function.genCopy(ty, target.long, self.short);
472 try function.genCopy(ty, target.short, self.short);
250 }473 }
251474
252 fn trackMaterialize(self: *InstTracking, function: *Self, inst: Air.Inst.Index, reg: Register) void {475 fn trackMaterialize(self: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void {
253 assert(inst == function.register_manager.registers[476 self.verifyMaterialize(target);
254 RegisterManager.indexOfRegIntoTracked(reg).?477 // Don't clobber reserved frame indices
255 ]);478 self.long = if (target.long == .none) switch (self.long) {
256 self.short = .{ .register = reg };479 .load_frame => |addr| .{ .reserved_frame = addr.index },
480 .reserved_frame => self.long,
481 else => target.long,
482 } else target.long;
483 self.short = target.short;
484 tracking_log.debug("%{d} => {} (materialize)", .{ inst, self.* });
257 }485 }
258486
259 fn resurrect(self: *InstTracking, scope_generation: u32) void {487 fn resurrect(self: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {
260 switch (self.short) {488 switch (self.short) {
261 .dead => |die_generation| if (die_generation >= scope_generation) {489 .dead => |die_generation| if (die_generation >= scope_generation) {
262 self.short = self.long;490 self.reuseFrame();
491 tracking_log.debug("%{d} => {} (resurrect)", .{ inst, self.* });
263 },492 },
264 else => {},493 else => {},
265 }494 }
266 }495 }
267496
268 fn die(self: *InstTracking, function: *Self) void {497 fn die(self: *InstTracking, function: *Self, inst: Air.Inst.Index) void {
269 function.freeValue(self.short);498 function.freeValue(self.short);
270 self.reuse(function);499 self.short = .{ .dead = function.scope_generation };
500 tracking_log.debug("%{d} => {} (death)", .{ inst, self.* });
271 }501 }
272502
273 fn reuse(self: *InstTracking, function: *Self) void {503 fn reuse(
504 self: *InstTracking,
505 function: *Self,
506 new_inst: Air.Inst.Index,
507 old_inst: Air.Inst.Index,
508 ) void {
274 self.short = .{ .dead = function.scope_generation };509 self.short = .{ .dead = function.scope_generation };
510 tracking_log.debug("%{d} => {} (reuse %{d})", .{ new_inst, self.*, old_inst });
511 }
512
513 pub fn format(
514 self: InstTracking,
515 comptime _: []const u8,
516 _: std.fmt.FormatOptions,
517 writer: anytype,
518 ) @TypeOf(writer).Error!void {
519 if (!std.meta.eql(self.long, self.short)) try writer.print("|{}| ", .{self.long});
520 try writer.print("{}", .{self.short});
521 }
522};
523
524const FrameAlloc = struct {
525 abi_size: u31,
526 abi_align: u5,
527 ref_count: u16,
528
529 fn init(alloc_abi: struct { size: u64, alignment: u32 }) FrameAlloc {
530 assert(math.isPowerOfTwo(alloc_abi.alignment));
531 return .{
532 .abi_size = @intCast(u31, alloc_abi.size),
533 .abi_align = math.log2_int(u32, alloc_abi.alignment),
534 .ref_count = 0,
535 };
536 }
537 fn initType(ty: Type, target: Target) FrameAlloc {
538 return init(.{ .size = ty.abiSize(target), .alignment = ty.abiAlignment(target) });
275 }539 }
276};540};
277541
...@@ -332,12 +596,13 @@ pub fn generate(...@@ -332,12 +596,13 @@ pub fn generate(
332 .fn_type = fn_type,596 .fn_type = fn_type,
333 .arg_index = 0,597 .arg_index = 0,
334 .src_loc = src_loc,598 .src_loc = src_loc,
335 .stack_align = undefined,
336 .end_di_line = module_fn.rbrace_line,599 .end_di_line = module_fn.rbrace_line,
337 .end_di_column = module_fn.rbrace_column,600 .end_di_column = module_fn.rbrace_column,
338 };601 };
339 defer {602 defer {
340 function.stack.deinit(gpa);603 function.frame_allocs.deinit(gpa);
604 function.free_frame_indices.deinit(gpa);
605 function.frame_locs.deinit(gpa);
341 var block_it = function.blocks.valueIterator();606 var block_it = function.blocks.valueIterator();
342 while (block_it.next()) |block| block.deinit(gpa);607 while (block_it.next()) |block| block.deinit(gpa);
343 function.blocks.deinit(gpa);608 function.blocks.deinit(gpa);
...@@ -349,7 +614,23 @@ pub fn generate(...@@ -349,7 +614,23 @@ pub fn generate(
349 if (builtin.mode == .Debug) function.mir_to_air_map.deinit(gpa);614 if (builtin.mode == .Debug) function.mir_to_air_map.deinit(gpa);
350 }615 }
351616
352 var call_info = function.resolveCallingConventionValues(fn_type, &.{}) catch |err| switch (err) {617 try function.frame_allocs.resize(gpa, FrameIndex.named_count);
618 function.frame_allocs.set(
619 @enumToInt(FrameIndex.stack_frame),
620 FrameAlloc.init(.{
621 .size = 0,
622 .alignment = if (mod.align_stack_fns.get(module_fn)) |set_align_stack|
623 set_align_stack.alignment
624 else
625 1,
626 }),
627 );
628 function.frame_allocs.set(
629 @enumToInt(FrameIndex.call_frame),
630 FrameAlloc.init(.{ .size = 0, .alignment = 1 }),
631 );
632
633 var call_info = function.resolveCallingConventionValues(fn_type, &.{}, .args_frame) catch |err| switch (err) {
353 error.CodegenFail => return Result{ .fail = function.err_msg.? },634 error.CodegenFail => return Result{ .fail = function.err_msg.? },
354 error.OutOfRegisters => return Result{635 error.OutOfRegisters => return Result{
355 .fail = try ErrorMsg.create(636 .fail = try ErrorMsg.create(
...@@ -365,8 +646,18 @@ pub fn generate(...@@ -365,8 +646,18 @@ pub fn generate(
365646
366 function.args = call_info.args;647 function.args = call_info.args;
367 function.ret_mcv = call_info.return_value;648 function.ret_mcv = call_info.return_value;
368 function.stack_align = call_info.stack_align;649 function.frame_allocs.set(@enumToInt(FrameIndex.ret_addr), FrameAlloc.init(.{
369 function.max_end_stack = call_info.stack_byte_count;650 .size = Type.usize.abiSize(function.target.*),
651 .alignment = @min(Type.usize.abiAlignment(function.target.*), call_info.stack_align),
652 }));
653 function.frame_allocs.set(@enumToInt(FrameIndex.base_ptr), FrameAlloc.init(.{
654 .size = Type.usize.abiSize(function.target.*),
655 .alignment = @min(Type.usize.abiAlignment(function.target.*) * 2, call_info.stack_align),
656 }));
657 function.frame_allocs.set(
658 @enumToInt(FrameIndex.args_frame),
659 FrameAlloc.init(.{ .size = call_info.stack_byte_count, .alignment = call_info.stack_align }),
660 );
370661
371 function.gen() catch |err| switch (err) {662 function.gen() catch |err| switch (err) {
372 error.CodegenFail => return Result{ .fail = function.err_msg.? },663 error.CodegenFail => return Result{ .fail = function.err_msg.? },
...@@ -379,6 +670,7 @@ pub fn generate(...@@ -379,6 +670,7 @@ pub fn generate(
379 var mir = Mir{670 var mir = Mir{
380 .instructions = function.mir_instructions.toOwnedSlice(),671 .instructions = function.mir_instructions.toOwnedSlice(),
381 .extra = try function.mir_extra.toOwnedSlice(bin_file.allocator),672 .extra = try function.mir_extra.toOwnedSlice(bin_file.allocator),
673 .frame_locs = function.frame_locs.toOwnedSlice(),
382 };674 };
383 defer mir.deinit(bin_file.allocator);675 defer mir.deinit(bin_file.allocator);
384676
...@@ -432,6 +724,7 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {...@@ -432,6 +724,7 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {
432 .mir = .{724 .mir = .{
433 .instructions = self.mir_instructions.slice(),725 .instructions = self.mir_instructions.slice(),
434 .extra = self.mir_extra.items,726 .extra = self.mir_extra.items,
727 .frame_locs = (std.MultiArrayList(Mir.FrameLoc){}).slice(),
435 },728 },
436 .target = self.target,729 .target = self.target,
437 .src_loc = self.src_loc,730 .src_loc = self.src_loc,
...@@ -458,6 +751,14 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {...@@ -458,6 +751,14 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {
458 }751 }
459}752}
460753
754fn dumpTracking(self: *Self) !void {
755 if (!debug_tracking) return;
756 const stderr = std.io.getStdErr().writer();
757
758 var it = self.inst_tracking.iterator();
759 while (it.next()) |entry| try stderr.print("%{d} = {}\n", .{ entry.key_ptr.*, entry.value_ptr.* });
760}
761
461fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {762fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
462 const gpa = self.gpa;763 const gpa = self.gpa;
463 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);764 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
...@@ -550,6 +851,14 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins...@@ -550,6 +851,14 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins
550 });851 });
551}852}
552853
854fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
855 return self.addInst(.{
856 .tag = .dead,
857 .ops = undefined,
858 .data = undefined,
859 });
860}
861
553fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {862fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
554 _ = try self.addInst(.{863 _ = try self.addInst(.{
555 .tag = tag,864 .tag = tag,
...@@ -765,44 +1074,32 @@ fn gen(self: *Self) InnerError!void {...@@ -765,44 +1074,32 @@ fn gen(self: *Self) InnerError!void {
765 const cc = self.fn_type.fnCallingConvention();1074 const cc = self.fn_type.fnCallingConvention();
766 if (cc != .Naked) {1075 if (cc != .Naked) {
767 try self.asmRegister(.push, .rbp);1076 try self.asmRegister(.push, .rbp);
1077 const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
768 try self.asmRegisterRegister(.mov, .rbp, .rsp);1078 try self.asmRegisterRegister(.mov, .rbp, .rsp);
7691079 const backpatch_frame_align = try self.asmPlaceholder();
770 // We want to subtract the aligned stack frame size from rsp here, but we don't1080 const backpatch_stack_alloc = try self.asmPlaceholder();
771 // yet know how big it will be, so we leave room for a 4-byte stack size.1081
772 // TODO During semantic analysis, check if there are no function calls. If there1082 switch (self.ret_mcv.long) {
773 // are none, here we can omit the part where we subtract and then add rsp.1083 .none, .unreach => {},
774 const backpatch_stack_sub = try self.addInst(.{1084 .indirect => {
775 .tag = .dead,1085 // The address where to store the return value for the caller is in a
776 .ops = undefined,1086 // register which the callee is free to clobber. Therefore, we purposely
777 .data = undefined,1087 // spill it to stack immediately.
778 });1088 const frame_index =
7791089 try self.allocFrameIndex(FrameAlloc.initType(Type.usize, self.target.*));
780 if (self.ret_mcv == .stack_offset) {1090 try self.genSetMem(
781 // The address where to store the return value for the caller is in a1091 .{ .frame = frame_index },
782 // register which the callee is free to clobber. Therefore, we purposely1092 0,
783 // spill it to stack immediately.1093 Type.usize,
784 const stack_offset = mem.alignForwardGeneric(u32, self.next_stack_offset + 8, 8);1094 self.ret_mcv.long.address().offset(-self.ret_mcv.short.indirect.off),
785 self.next_stack_offset = stack_offset;1095 );
786 self.max_end_stack = @max(self.max_end_stack, self.next_stack_offset);1096 self.ret_mcv.long = .{ .load_frame = .{ .index = frame_index } };
7871097 tracking_log.debug("spill {} to {}", .{ self.ret_mcv.long, frame_index });
788 const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0];1098 },
789 try self.genSetStack(Type.usize, @intCast(i32, stack_offset), MCValue{ .register = ret_reg }, .{});1099 else => unreachable,
790 self.ret_mcv = MCValue{ .stack_offset = @intCast(i32, stack_offset) };
791 log.debug("gen: spilling {s} to stack at offset {}", .{ @tagName(ret_reg), stack_offset });
792 }1100 }
7931101
794 _ = try self.addInst(.{1102 try self.asmOpOnly(.dbg_prologue_end);
795 .tag = .dbg_prologue_end,
796 .ops = undefined,
797 .data = undefined,
798 });
799
800 // Push callee-preserved regs that were used actually in use.
801 const backpatch_push_callee_preserved_regs = try self.addInst(.{
802 .tag = .dead,
803 .ops = undefined,
804 .data = undefined,
805 });
8061103
807 try self.genBody(self.air.getMainBody());1104 try self.genBody(self.air.getMainBody());
8081105
...@@ -814,69 +1111,43 @@ fn gen(self: *Self) InnerError!void {...@@ -814,69 +1111,43 @@ fn gen(self: *Self) InnerError!void {
814 // }1111 // }
815 // Eliding the reloc will cause a miscompilation in this case.1112 // Eliding the reloc will cause a miscompilation in this case.
816 for (self.exitlude_jump_relocs.items) |jmp_reloc| {1113 for (self.exitlude_jump_relocs.items) |jmp_reloc| {
817 self.mir_instructions.items(.data)[jmp_reloc].inst = @intCast(u32, self.mir_instructions.len);1114 self.mir_instructions.items(.data)[jmp_reloc].inst =
1115 @intCast(u32, self.mir_instructions.len);
818 }1116 }
8191117
820 // Create list of registers to save in the prologue.1118 try self.asmOpOnly(.dbg_epilogue_begin);
821 // TODO handle register classes1119 const backpatch_stack_dealloc = try self.asmPlaceholder();
822 var reg_list = Mir.RegisterList{};1120 const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
823 const callee_preserved_regs = abi.getCalleePreservedRegs(self.target.*);
824 for (callee_preserved_regs) |reg| {
825 if (self.register_manager.isRegAllocated(reg)) {
826 reg_list.push(callee_preserved_regs, reg);
827 }
828 }
829 const saved_regs_stack_space: u32 = reg_list.count() * 8;
830
831 // Pop saved callee-preserved regs.
832 const backpatch_pop_callee_preserved_regs = try self.addInst(.{
833 .tag = .dead,
834 .ops = undefined,
835 .data = undefined,
836 });
837
838 _ = try self.addInst(.{
839 .tag = .dbg_epilogue_begin,
840 .ops = undefined,
841 .data = undefined,
842 });
843
844 // Maybe add rsp, x if required. This is backpatched later.
845 const backpatch_stack_add = try self.addInst(.{
846 .tag = .dead,
847 .ops = undefined,
848 .data = undefined,
849 });
850
851 try self.asmRegister(.pop, .rbp);1121 try self.asmRegister(.pop, .rbp);
852 try self.asmOpOnly(.ret);1122 try self.asmOpOnly(.ret);
8531123
854 // Adjust the stack1124 const frame_layout = try self.computeFrameLayout();
855 if (self.max_end_stack > math.maxInt(i32)) {1125 const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
856 return self.failSymbol("too much stack used in call parameters", .{});1126 const need_stack_adjust = frame_layout.stack_adjust > 0;
1127 const need_save_reg = frame_layout.save_reg_list.count() > 0;
1128 if (need_frame_align) {
1129 self.mir_instructions.set(backpatch_frame_align, .{
1130 .tag = .@"and",
1131 .ops = .ri_s,
1132 .data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_mask } },
1133 });
857 }1134 }
8581135 if (need_stack_adjust) {
859 const aligned_stack_end = @intCast(1136 self.mir_instructions.set(backpatch_stack_alloc, .{
860 u32,
861 mem.alignForward(self.max_end_stack + saved_regs_stack_space, self.stack_align),
862 );
863 if (aligned_stack_end > 0) {
864 self.mir_instructions.set(backpatch_stack_sub, .{
865 .tag = .sub,1137 .tag = .sub,
866 .ops = .ri_u,1138 .ops = .ri_s,
867 .data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },1139 .data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_adjust } },
868 });1140 });
869 self.mir_instructions.set(backpatch_stack_add, .{1141 }
870 .tag = .add,1142 if (need_frame_align or need_stack_adjust) {
871 .ops = .ri_u,1143 self.mir_instructions.set(backpatch_stack_dealloc, .{
872 .data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },1144 .tag = .mov,
873 });1145 .ops = .rr,
8741146 .data = .{ .rr = .{ .r1 = .rsp, .r2 = .rbp } },
875 const save_reg_list = try self.addExtra(Mir.SaveRegisterList{
876 .base_reg = @enumToInt(Register.rbp),
877 .register_list = reg_list.asInt(),
878 .stack_end = aligned_stack_end,
879 });1147 });
1148 }
1149 if (need_save_reg) {
1150 const save_reg_list = frame_layout.save_reg_list.asInt();
880 self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{1151 self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
881 .tag = .push_regs,1152 .tag = .push_regs,
882 .ops = undefined,1153 .ops = undefined,
...@@ -889,30 +1160,19 @@ fn gen(self: *Self) InnerError!void {...@@ -889,30 +1160,19 @@ fn gen(self: *Self) InnerError!void {
889 });1160 });
890 }1161 }
891 } else {1162 } else {
892 _ = try self.addInst(.{1163 try self.asmOpOnly(.dbg_prologue_end);
893 .tag = .dbg_prologue_end,
894 .ops = undefined,
895 .data = undefined,
896 });
897
898 try self.genBody(self.air.getMainBody());1164 try self.genBody(self.air.getMainBody());
8991165 try self.asmOpOnly(.dbg_epilogue_begin);
900 _ = try self.addInst(.{
901 .tag = .dbg_epilogue_begin,
902 .ops = undefined,
903 .data = undefined,
904 });
905 }1166 }
9061167
907 // Drop them off at the rbrace.1168 // Drop them off at the rbrace.
908 const payload = try self.addExtra(Mir.DbgLineColumn{
909 .line = self.end_di_line,
910 .column = self.end_di_column,
911 });
912 _ = try self.addInst(.{1169 _ = try self.addInst(.{
913 .tag = .dbg_line,1170 .tag = .dbg_line,
914 .ops = undefined,1171 .ops = undefined,
915 .data = .{ .payload = payload },1172 .data = .{ .line_column = .{
1173 .line = self.end_di_line,
1174 .column = self.end_di_column,
1175 } },
916 });1176 });
917}1177}
9181178
...@@ -932,6 +1192,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -932,6 +1192,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
932 self.air,1192 self.air,
933 self.liveness,1193 self.liveness,
934 );1194 );
1195 self.dumpTracking() catch {};
9351196
936 const old_air_bookkeeping = self.air_bookkeeping;1197 const old_air_bookkeeping = self.air_bookkeeping;
937 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);1198 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
...@@ -1187,23 +1448,16 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -1187,23 +1448,16 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
1187 while (it.next()) |index| {1448 while (it.next()) |index| {
1188 const tracked_inst = self.register_manager.registers[index];1449 const tracked_inst = self.register_manager.registers[index];
1189 const tracking = self.getResolvedInstValue(tracked_inst);1450 const tracking = self.getResolvedInstValue(tracked_inst);
1190 assert(RegisterManager.indexOfRegIntoTracked(switch (tracking.short) {1451 assert(RegisterManager.indexOfRegIntoTracked(tracking.getReg().?).? == index);
1191 .register => |reg| reg,
1192 .register_overflow => |ro| ro.reg,
1193 else => unreachable,
1194 }).? == index);
1195 }1452 }
1196 }1453 }
1197 }1454 }
1198 }1455 }
1456 self.dumpTracking() catch {};
1199}1457}
12001458
1201fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {1459fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
1202 const reg = switch (value) {1460 const reg = value.getReg() orelse return;
1203 .register => |reg| reg,
1204 .register_overflow => |ro| ro.reg,
1205 else => return,
1206 };
1207 if (self.register_manager.isRegFree(reg)) {1461 if (self.register_manager.isRegFree(reg)) {
1208 self.register_manager.getRegAssumeFree(reg, inst);1462 self.register_manager.getRegAssumeFree(reg, inst);
1209 }1463 }
...@@ -1214,8 +1468,11 @@ fn freeValue(self: *Self, value: MCValue) void {...@@ -1214,8 +1468,11 @@ fn freeValue(self: *Self, value: MCValue) void {
1214 .register => |reg| {1468 .register => |reg| {
1215 self.register_manager.freeReg(reg);1469 self.register_manager.freeReg(reg);
1216 },1470 },
1217 .register_overflow => |ro| {1471 .register_offset => |reg_off| {
1218 self.register_manager.freeReg(ro.reg);1472 self.register_manager.freeReg(reg_off.reg);
1473 },
1474 .register_overflow => |reg_ov| {
1475 self.register_manager.freeReg(reg_ov.reg);
1219 self.eflags_inst = null;1476 self.eflags_inst = null;
1220 },1477 },
1221 .eflags => {1478 .eflags => {
...@@ -1231,10 +1488,10 @@ fn feed(self: *Self, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) void {...@@ -1231,10 +1488,10 @@ fn feed(self: *Self, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) void {
12311488
1232/// Asserts there is already capacity to insert into top branch inst_table.1489/// Asserts there is already capacity to insert into top branch inst_table.
1233fn processDeath(self: *Self, inst: Air.Inst.Index) void {1490fn processDeath(self: *Self, inst: Air.Inst.Index) void {
1234 const air_tags = self.air.instructions.items(.tag);1491 switch (self.air.instructions.items(.tag)[inst]) {
1235 if (air_tags[inst] == .constant) return;1492 .constant, .const_ty => unreachable,
1236 log.debug("%{d} => {}", .{ inst, MCValue.dead });1493 else => self.inst_tracking.getPtr(inst).?.die(self, inst),
1237 self.inst_tracking.getPtr(inst).?.die(self);1494 }
1238}1495}
12391496
1240/// Called when there are no operands, and the instruction is always unreferenced.1497/// Called when there are no operands, and the instruction is always unreferenced.
...@@ -1249,7 +1506,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {...@@ -1249,7 +1506,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
1249 .none, .dead, .unreach => {},1506 .none, .dead, .unreach => {},
1250 else => unreachable, // Why didn't the result die?1507 else => unreachable, // Why didn't the result die?
1251 } else {1508 } else {
1252 log.debug("%{d} => {}", .{ inst, result });1509 tracking_log.debug("%{d} => {} (birth)", .{ inst, result });
1253 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));1510 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
1254 // In some cases, an operand may be reused as the result.1511 // In some cases, an operand may be reused as the result.
1255 // If that operand died and was a register, it was freed by1512 // If that operand died and was a register, it was freed by
...@@ -1273,36 +1530,123 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live...@@ -1273,36 +1530,123 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live
1273 self.finishAirResult(inst, result);1530 self.finishAirResult(inst, result);
1274}1531}
12751532
1276fn allocMem(self: *Self, inst: ?Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 {1533const FrameLayout = struct {
1277 if (abi_align > self.stack_align)1534 stack_mask: u32,
1278 self.stack_align = abi_align;1535 stack_adjust: u32,
1279 // TODO find a free slot instead of always appending1536 save_reg_list: Mir.RegisterList,
1280 const offset = mem.alignForwardGeneric(u32, self.next_stack_offset + abi_size, abi_align);1537};
1281 self.next_stack_offset = offset;1538
1282 self.max_end_stack = @max(self.max_end_stack, self.next_stack_offset);1539fn setFrameLoc(
1283 try self.stack.putNoClobber(self.gpa, offset, .{1540 self: *Self,
1284 .inst = inst,1541 frame_index: FrameIndex,
1285 .size = abi_size,1542 base: Register,
1286 });1543 offset: *i32,
1287 return offset;1544 comptime aligned: bool,
1545) void {
1546 const frame_i = @enumToInt(frame_index);
1547 if (aligned) {
1548 const alignment = @as(i32, 1) << self.frame_allocs.items(.abi_align)[frame_i];
1549 offset.* = mem.alignForwardGeneric(i32, offset.*, alignment);
1550 }
1551 self.frame_locs.set(frame_i, .{ .base = base, .disp = offset.* });
1552 offset.* += self.frame_allocs.items(.abi_size)[frame_i];
1553}
1554
1555fn computeFrameLayout(self: *Self) !FrameLayout {
1556 const frame_allocs_len = self.frame_allocs.len;
1557 try self.frame_locs.resize(self.gpa, frame_allocs_len);
1558 const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);
1559 defer self.gpa.free(stack_frame_order);
1560
1561 const frame_size = self.frame_allocs.items(.abi_size);
1562 const frame_align = self.frame_allocs.items(.abi_align);
1563 const frame_offset = self.frame_locs.items(.disp);
1564
1565 for (stack_frame_order, FrameIndex.named_count..) |*frame_order, frame_index|
1566 frame_order.* = @intToEnum(FrameIndex, frame_index);
1567 {
1568 const SortContext = struct {
1569 frame_align: @TypeOf(frame_align),
1570 pub fn lessThan(context: @This(), lhs: FrameIndex, rhs: FrameIndex) bool {
1571 return context.frame_align[@enumToInt(lhs)] > context.frame_align[@enumToInt(rhs)];
1572 }
1573 };
1574 const sort_context = SortContext{ .frame_align = frame_align };
1575 std.sort.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
1576 }
1577
1578 const call_frame_align = frame_align[@enumToInt(FrameIndex.call_frame)];
1579 const stack_frame_align = frame_align[@enumToInt(FrameIndex.stack_frame)];
1580 const args_frame_align = frame_align[@enumToInt(FrameIndex.args_frame)];
1581 const needed_align = @max(call_frame_align, stack_frame_align);
1582 const need_align_stack = needed_align > args_frame_align;
1583
1584 // Create list of registers to save in the prologue.
1585 // TODO handle register classes
1586 var save_reg_list = Mir.RegisterList{};
1587 const callee_preserved_regs = abi.getCalleePreservedRegs(self.target.*);
1588 for (callee_preserved_regs) |reg| {
1589 if (self.register_manager.isRegAllocated(reg)) {
1590 save_reg_list.push(callee_preserved_regs, reg);
1591 }
1592 }
1593
1594 var rbp_offset = @intCast(i32, save_reg_list.count() * 8);
1595 self.setFrameLoc(.base_ptr, .rbp, &rbp_offset, false);
1596 self.setFrameLoc(.ret_addr, .rbp, &rbp_offset, false);
1597 self.setFrameLoc(.args_frame, .rbp, &rbp_offset, false);
1598 const stack_frame_align_offset =
1599 if (need_align_stack) 0 else frame_offset[@enumToInt(FrameIndex.args_frame)];
1600
1601 var rsp_offset: i32 = 0;
1602 self.setFrameLoc(.call_frame, .rsp, &rsp_offset, true);
1603 self.setFrameLoc(.stack_frame, .rsp, &rsp_offset, true);
1604 for (stack_frame_order) |frame_index| self.setFrameLoc(frame_index, .rsp, &rsp_offset, true);
1605 rsp_offset += stack_frame_align_offset;
1606 rsp_offset = mem.alignForwardGeneric(i32, rsp_offset, @as(i32, 1) << needed_align);
1607 rsp_offset -= stack_frame_align_offset;
1608 frame_size[@enumToInt(FrameIndex.call_frame)] =
1609 @intCast(u31, rsp_offset - frame_offset[@enumToInt(FrameIndex.stack_frame)]);
1610
1611 return .{
1612 .stack_mask = @as(u32, math.maxInt(u32)) << (if (need_align_stack) needed_align else 0),
1613 .stack_adjust = @intCast(u32, rsp_offset - frame_offset[@enumToInt(FrameIndex.call_frame)]),
1614 .save_reg_list = save_reg_list,
1615 };
1288}1616}
12891617
1290/// Use a pointer instruction as the basis for allocating stack memory.1618fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
1291fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {1619 const frame_allocs_slice = self.frame_allocs.slice();
1292 const ptr_ty = self.air.typeOfIndex(inst);1620 const frame_size = frame_allocs_slice.items(.abi_size);
1293 const elem_ty = ptr_ty.elemType();1621 const frame_align = frame_allocs_slice.items(.abi_align);
1622 for (self.free_frame_indices.keys(), 0..) |frame_index, free_i| {
1623 const abi_size = frame_size[@enumToInt(frame_index)];
1624 if (abi_size != alloc.abi_size) continue;
1625 const abi_align = &frame_align[@enumToInt(frame_index)];
1626 abi_align.* = @max(abi_align.*, alloc.abi_align);
12941627
1295 if (!elem_ty.hasRuntimeBitsIgnoreComptime()) {1628 const stack_frame_align = &frame_align[@enumToInt(FrameIndex.stack_frame)];
1296 return self.allocMem(inst, @sizeOf(usize), @alignOf(usize));1629 stack_frame_align.* = @max(stack_frame_align.*, alloc.abi_align);
1630
1631 _ = self.free_frame_indices.swapRemoveAt(free_i);
1632 return frame_index;
1297 }1633 }
1634 const frame_index = @intToEnum(FrameIndex, self.frame_allocs.len);
1635 try self.frame_allocs.append(self.gpa, alloc);
1636 return frame_index;
1637}
12981638
1299 const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) orelse {1639/// Use a pointer instruction as the basis for allocating stack memory.
1300 const mod = self.bin_file.options.module.?;1640fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
1301 return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(mod)});1641 const ptr_ty = self.air.typeOfIndex(inst);
1302 };1642 const val_ty = ptr_ty.childType();
1303 // TODO swap this for inst.ty.ptrAlign1643 return self.allocFrameIndex(FrameAlloc.init(.{
1304 const abi_align = ptr_ty.ptrAlignment(self.target.*);1644 .size = math.cast(u32, val_ty.abiSize(self.target.*)) orelse {
1305 return self.allocMem(inst, abi_size, abi_align);1645 const mod = self.bin_file.options.module.?;
1646 return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(mod)});
1647 },
1648 .alignment = @max(ptr_ty.ptrAlignment(self.target.*), 1),
1649 }));
1306}1650}
13071651
1308fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {1652fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
...@@ -1320,42 +1664,35 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_...@@ -1320,42 +1664,35 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_
1320 };1664 };
13211665
1322 if (reg_ok) {1666 if (reg_ok) {
1323 switch (elem_ty.zigTypeTag()) {1667 // Make sure the type can fit in a register before we try to allocate one.
1324 .Vector => return self.fail("TODO allocRegOrMem for Vector type", .{}),1668 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
1325 .Float => {1669 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1326 if (intrinsicsAllowed(self.target.*, elem_ty)) {1670 if (abi_size <= ptr_bytes) {
1327 const ptr_bytes: u64 = 32;1671 if (self.register_manager.tryAllocReg(inst, try self.regClassForType(elem_ty))) |reg| {
1328 if (abi_size <= ptr_bytes) {1672 return MCValue{ .register = registerAlias(reg, abi_size) };
1329 if (self.register_manager.tryAllocReg(inst, sse)) |reg| {1673 }
1330 return MCValue{ .register = registerAlias(reg, abi_size) };
1331 }
1332 }
1333 }
1334
1335 return self.fail("TODO allocRegOrMem for Float type without SSE/AVX support", .{});
1336 },
1337 else => {
1338 // Make sure the type can fit in a register before we try to allocate one.
1339 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
1340 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1341 if (abi_size <= ptr_bytes) {
1342 if (self.register_manager.tryAllocReg(inst, gp)) |reg| {
1343 return MCValue{ .register = registerAlias(reg, abi_size) };
1344 }
1345 }
1346 },
1347 }1674 }
1348 }1675 }
13491676
1350 const abi_align = elem_ty.abiAlignment(self.target.*);1677 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(elem_ty, self.target.*));
1351 if (abi_align > self.stack_align)1678 return .{ .load_frame = .{ .index = frame_index } };
1352 self.stack_align = abi_align;1679}
1353 const stack_offset = try self.allocMem(inst, abi_size, abi_align);1680
1354 return MCValue{ .stack_offset = @intCast(i32, stack_offset) };1681fn regClassForType(self: *Self, ty: Type) !RegisterManager.RegisterBitSet {
1682 return switch (ty.zigTypeTag()) {
1683 .Vector => self.fail("TODO regClassForType for {}", .{ty.fmt(self.bin_file.options.module.?)}),
1684 .Float => switch (ty.floatBits(self.target.*)) {
1685 32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse)) sse else gp,
1686 64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2)) sse else gp,
1687 else => gp,
1688 },
1689 else => gp,
1690 };
1355}1691}
13561692
1357const State = struct {1693const State = struct {
1358 registers: RegisterManager.TrackedRegisters,1694 registers: RegisterManager.TrackedRegisters,
1695 reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
1359 free_registers: RegisterManager.RegisterBitSet,1696 free_registers: RegisterManager.RegisterBitSet,
1360 inst_tracking_len: u32,1697 inst_tracking_len: u32,
1361 scope_generation: u32,1698 scope_generation: u32,
...@@ -1370,8 +1707,14 @@ fn initRetroactiveState(self: *Self) State {...@@ -1370,8 +1707,14 @@ fn initRetroactiveState(self: *Self) State {
13701707
1371fn saveRetroactiveState(self: *Self, state: *State) !void {1708fn saveRetroactiveState(self: *Self, state: *State) !void {
1372 try self.spillEflagsIfOccupied();1709 try self.spillEflagsIfOccupied();
1373 state.registers = self.register_manager.registers;1710 const free_registers = self.register_manager.free_registers;
1374 state.free_registers = self.register_manager.free_registers;1711 var it = free_registers.iterator(.{ .kind = .unset });
1712 while (it.next()) |index| {
1713 const tracked_inst = self.register_manager.registers[index];
1714 state.registers[index] = tracked_inst;
1715 state.reg_tracking[index] = self.inst_tracking.get(tracked_inst).?;
1716 }
1717 state.free_registers = free_registers;
1375}1718}
13761719
1377fn saveState(self: *Self) !State {1720fn saveState(self: *Self) !State {
...@@ -1387,18 +1730,18 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -1387,18 +1730,18 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1387 close_scope: bool,1730 close_scope: bool,
1388}) !void {1731}) !void {
1389 if (opts.close_scope) {1732 if (opts.close_scope) {
1390 for (self.inst_tracking.values()[state.inst_tracking_len..]) |*tracking| tracking.die(self);1733 for (
1734 self.inst_tracking.keys()[state.inst_tracking_len..],
1735 self.inst_tracking.values()[state.inst_tracking_len..],
1736 ) |inst, *tracking| tracking.die(self, inst);
1391 self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);1737 self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
1392 }1738 }
13931739
1394 if (opts.resurrect) for (self.inst_tracking.values()[0..state.inst_tracking_len]) |*tracking|1740 if (opts.resurrect) for (
1395 tracking.resurrect(state.scope_generation);1741 self.inst_tracking.keys()[0..state.inst_tracking_len],
1396 const air_tags = self.air.instructions.items(.tag);1742 self.inst_tracking.values()[0..state.inst_tracking_len],
1397 for (deaths) |death| switch (air_tags[death]) {1743 ) |inst, *tracking| tracking.resurrect(inst, state.scope_generation);
1398 .constant => {},1744 for (deaths) |death| self.processDeath(death);
1399 .const_ty => unreachable,
1400 else => self.inst_tracking.getPtr(death).?.die(self),
1401 };
14021745
1403 for (0..state.registers.len) |index| {1746 for (0..state.registers.len) |index| {
1404 const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))1747 const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
...@@ -1411,26 +1754,34 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -1411,26 +1754,34 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1411 state.registers[index];1754 state.registers[index];
1412 if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst|1755 if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst|
1413 assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);1756 assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);
1414 if (current_maybe_inst == target_maybe_inst) continue;
1415 const reg = RegisterManager.regAtTrackedIndex(
1416 @intCast(RegisterManager.RegisterBitSet.ShiftInt, index),
1417 );
1418 if (opts.emit_instructions) {1757 if (opts.emit_instructions) {
1419 if (current_maybe_inst) |current_inst| {1758 if (current_maybe_inst) |current_inst| {
1420 try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);1759 try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);
1421 }1760 }
1422 if (target_maybe_inst) |target_inst| {1761 if (target_maybe_inst) |target_inst| {
1423 try self.inst_tracking.getPtr(target_inst).?.materialize(self, target_inst, reg);1762 try self.inst_tracking.getPtr(target_inst).?.materialize(
1763 self,
1764 target_inst,
1765 state.reg_tracking[index],
1766 );
1424 }1767 }
1425 }1768 }
1426 if (opts.update_tracking) {1769 if (opts.update_tracking) {
1427 if (current_maybe_inst) |current_inst| {1770 if (current_maybe_inst) |current_inst| {
1428 self.inst_tracking.getPtr(current_inst).?.trackSpill(self);1771 self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
1772 }
1773 {
1774 const reg = RegisterManager.regAtTrackedIndex(
1775 @intCast(RegisterManager.RegisterBitSet.ShiftInt, index),
1776 );
1777 self.register_manager.freeReg(reg);
1778 self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
1429 }1779 }
1430 self.register_manager.freeReg(reg);
1431 self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
1432 if (target_maybe_inst) |target_inst| {1780 if (target_maybe_inst) |target_inst| {
1433 self.inst_tracking.getPtr(target_inst).?.trackMaterialize(self, target_inst, reg);1781 self.inst_tracking.getPtr(target_inst).?.trackMaterialize(
1782 target_inst,
1783 state.reg_tracking[index],
1784 );
1434 }1785 }
1435 }1786 }
1436 }1787 }
...@@ -1438,7 +1789,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -1438,7 +1789,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1438 try self.inst_tracking.getPtr(inst).?.spill(self, inst);1789 try self.inst_tracking.getPtr(inst).?.spill(self, inst);
1439 if (opts.update_tracking) if (self.eflags_inst) |inst| {1790 if (opts.update_tracking) if (self.eflags_inst) |inst| {
1440 self.eflags_inst = null;1791 self.eflags_inst = null;
1441 self.inst_tracking.getPtr(inst).?.trackSpill(self);1792 self.inst_tracking.getPtr(inst).?.trackSpill(self, inst);
1442 };1793 };
14431794
1444 if (opts.update_tracking and std.debug.runtime_safety) {1795 if (opts.update_tracking and std.debug.runtime_safety) {
...@@ -1452,9 +1803,9 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -1452,9 +1803,9 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
14521803
1453pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {1804pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
1454 const tracking = self.inst_tracking.getPtr(inst).?;1805 const tracking = self.inst_tracking.getPtr(inst).?;
1455 assert(tracking.getReg().?.to64() == reg.to64());1806 assert(tracking.getReg().?.id() == reg.id());
1456 try tracking.spill(self, inst);1807 try tracking.spill(self, inst);
1457 tracking.trackSpill(self);1808 tracking.trackSpill(self, inst);
1458}1809}
14591810
1460pub fn spillEflagsIfOccupied(self: *Self) !void {1811pub fn spillEflagsIfOccupied(self: *Self) !void {
...@@ -1463,7 +1814,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {...@@ -1463,7 +1814,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
1463 const tracking = self.inst_tracking.getPtr(inst).?;1814 const tracking = self.inst_tracking.getPtr(inst).?;
1464 assert(tracking.getCondition() != null);1815 assert(tracking.getCondition() != null);
1465 try tracking.spill(self, inst);1816 try tracking.spill(self, inst);
1466 tracking.trackSpill(self);1817 tracking.trackSpill(self, inst);
1467 }1818 }
1468}1819}
14691820
...@@ -1477,15 +1828,8 @@ pub fn spillRegisters(self: *Self, registers: []const Register) !void {...@@ -1477,15 +1828,8 @@ pub fn spillRegisters(self: *Self, registers: []const Register) !void {
1477/// allocated. A second call to `copyToTmpRegister` may return the same register.1828/// allocated. A second call to `copyToTmpRegister` may return the same register.
1478/// This can have a side effect of spilling instructions to the stack to free up a register.1829/// This can have a side effect of spilling instructions to the stack to free up a register.
1479fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {1830fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
1480 const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {1831 const reg: Register = try self.register_manager.allocReg(null, try self.regClassForType(ty));
1481 .Float => blk: {1832 try self.genSetReg(reg, ty, mcv);
1482 if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;
1483 return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
1484 },
1485 else => gp,
1486 };
1487 const reg: Register = try self.register_manager.allocReg(null, reg_class);
1488 try self.genSetReg(ty, reg, mcv);
1489 return reg;1833 return reg;
1490}1834}
14911835
...@@ -1493,28 +1837,35 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {...@@ -1493,28 +1837,35 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
1493/// `reg_owner` is the instruction that gets associated with the register in the register table.1837/// `reg_owner` is the instruction that gets associated with the register in the register table.
1494/// This can have a side effect of spilling instructions to the stack to free up a register.1838/// This can have a side effect of spilling instructions to the stack to free up a register.
1495/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!1839/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
1496fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue {1840fn copyToRegisterWithInstTracking(
1497 const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {1841 self: *Self,
1498 .Float => blk: {1842 reg_owner: Air.Inst.Index,
1499 if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;1843 ty: Type,
1500 return self.fail("TODO copy {} to register", .{ty.fmtDebug()});1844 mcv: MCValue,
1501 },1845) !MCValue {
1502 else => gp,1846 const reg: Register = try self.register_manager.allocReg(reg_owner, try self.regClassForType(ty));
1503 };1847 try self.genSetReg(reg, ty, mcv);
1504 const reg: Register = try self.register_manager.allocReg(reg_owner, reg_class);
1505 try self.genSetReg(ty, reg, mcv);
1506 return MCValue{ .register = reg };1848 return MCValue{ .register = reg };
1507}1849}
15081850
1509fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {1851fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
1510 const stack_offset = try self.allocMemPtr(inst);1852 const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };
1511 const result = MCValue{ .ptr_stack_offset = @intCast(i32, stack_offset) };
1512 return self.finishAir(inst, result, .{ .none, .none, .none });1853 return self.finishAir(inst, result, .{ .none, .none, .none });
1513}1854}
15141855
1515fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {1856fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
1516 const stack_offset = try self.allocMemPtr(inst);1857 const result: MCValue = switch (self.ret_mcv.long) {
1517 const result = MCValue{ .ptr_stack_offset = @intCast(i32, stack_offset) };1858 else => unreachable,
1859 .none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },
1860 .load_frame => .{ .register_offset = .{
1861 .reg = (try self.copyToRegisterWithInstTracking(
1862 inst,
1863 self.air.typeOfIndex(inst),
1864 self.ret_mcv.long,
1865 )).register,
1866 .off = self.ret_mcv.short.indirect.off,
1867 } },
1868 };
1518 return self.finishAir(inst, result, .{ .none, .none, .none });1869 return self.finishAir(inst, result, .{ .none, .none, .none });
1519}1870}
15201871
...@@ -1577,12 +1928,14 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -1577,12 +1928,14 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1577 registerAlias(src_reg, min_abi_size),1928 registerAlias(src_reg, min_abi_size),
1578 );1929 );
1579 },1930 },
1580 .stack_offset => |src_off| {1931 .load_frame => |frame_addr| try self.asmRegisterMemory(
1581 try self.asmRegisterMemory(tag, dst_alias, Memory.sib(1932 tag,
1582 Memory.PtrSize.fromSize(min_abi_size),1933 dst_alias,
1583 .{ .base = .rbp, .disp = -src_off },1934 Memory.sib(Memory.PtrSize.fromSize(min_abi_size), .{
1584 ));1935 .base = .{ .frame = frame_addr.index },
1585 },1936 .disp = frame_addr.off,
1937 }),
1938 ),
1586 else => return self.fail("TODO airIntCast from {s} to {s}", .{1939 else => return self.fail("TODO airIntCast from {s} to {s}", .{
1587 @tagName(src_mcv),1940 @tagName(src_mcv),
1588 @tagName(dst_mcv),1941 @tagName(dst_mcv),
...@@ -1591,7 +1944,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -1591,7 +1944,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1591 if (self.regExtraBits(min_ty) > 0) try self.truncateRegister(min_ty, dst_reg);1944 if (self.regExtraBits(min_ty) > 0) try self.truncateRegister(min_ty, dst_reg);
1592 },1945 },
1593 else => {1946 else => {
1594 try self.setRegOrMem(min_ty, dst_mcv, src_mcv);1947 try self.genCopy(min_ty, dst_mcv, src_mcv);
1595 const extra = dst_abi_size * 8 - dst_int_info.bits;1948 const extra = dst_abi_size * 8 - dst_int_info.bits;
1596 if (extra > 0) {1949 if (extra > 0) {
1597 try self.genShiftBinOpMir(switch (signedness) {1950 try self.genShiftBinOpMir(switch (signedness) {
...@@ -1653,16 +2006,22 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {...@@ -1653,16 +2006,22 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1653 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;2006 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1654 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;2007 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
16552008
2009 const slice_ty = self.air.typeOfIndex(inst);
1656 const ptr = try self.resolveInst(bin_op.lhs);2010 const ptr = try self.resolveInst(bin_op.lhs);
1657 const ptr_ty = self.air.typeOf(bin_op.lhs);2011 const ptr_ty = self.air.typeOf(bin_op.lhs);
1658 const len = try self.resolveInst(bin_op.rhs);2012 const len = try self.resolveInst(bin_op.rhs);
1659 const len_ty = self.air.typeOf(bin_op.rhs);2013 const len_ty = self.air.typeOf(bin_op.rhs);
16602014
1661 const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));2015 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, self.target.*));
1662 try self.genSetStack(ptr_ty, stack_offset, ptr, .{});2016 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
1663 try self.genSetStack(len_ty, stack_offset - 8, len, .{});2017 try self.genSetMem(
1664 const result = MCValue{ .stack_offset = stack_offset };2018 .{ .frame = frame_index },
2019 @intCast(i32, ptr_ty.abiSize(self.target.*)),
2020 len_ty,
2021 len,
2022 );
16652023
2024 const result = MCValue{ .load_frame = .{ .index = frame_index } };
1666 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });2025 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1667}2026}
16682027
...@@ -1778,14 +2137,14 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -1778,14 +2137,14 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
17782137
1779 const reg_bits = self.regBitSize(ty);2138 const reg_bits = self.regBitSize(ty);
1780 const cc: Condition = if (ty.isSignedInt()) cc: {2139 const cc: Condition = if (ty.isSignedInt()) cc: {
1781 try self.genSetReg(ty, limit_reg, dst_mcv);2140 try self.genSetReg(limit_reg, ty, dst_mcv);
1782 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });2141 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1783 try self.genBinOpMir(.xor, ty, limit_mcv, .{2142 try self.genBinOpMir(.xor, ty, limit_mcv, .{
1784 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,2143 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1785 });2144 });
1786 break :cc .o;2145 break :cc .o;
1787 } else cc: {2146 } else cc: {
1788 try self.genSetReg(ty, limit_reg, .{2147 try self.genSetReg(limit_reg, ty, .{
1789 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),2148 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1790 });2149 });
1791 break :cc .c;2150 break :cc .c;
...@@ -1829,14 +2188,14 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -1829,14 +2188,14 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
18292188
1830 const reg_bits = self.regBitSize(ty);2189 const reg_bits = self.regBitSize(ty);
1831 const cc: Condition = if (ty.isSignedInt()) cc: {2190 const cc: Condition = if (ty.isSignedInt()) cc: {
1832 try self.genSetReg(ty, limit_reg, dst_mcv);2191 try self.genSetReg(limit_reg, ty, dst_mcv);
1833 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });2192 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1834 try self.genBinOpMir(.xor, ty, limit_mcv, .{2193 try self.genBinOpMir(.xor, ty, limit_mcv, .{
1835 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,2194 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1836 });2195 });
1837 break :cc .o;2196 break :cc .o;
1838 } else cc: {2197 } else cc: {
1839 try self.genSetReg(ty, limit_reg, .{ .immediate = 0 });2198 try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });
1840 break :cc .c;2199 break :cc .c;
1841 };2200 };
1842 try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);2201 try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
...@@ -1880,7 +2239,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -1880,7 +2239,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
18802239
1881 const reg_bits = self.regBitSize(ty);2240 const reg_bits = self.regBitSize(ty);
1882 const cc: Condition = if (ty.isSignedInt()) cc: {2241 const cc: Condition = if (ty.isSignedInt()) cc: {
1883 try self.genSetReg(ty, limit_reg, lhs_mcv);2242 try self.genSetReg(limit_reg, ty, lhs_mcv);
1884 try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);2243 try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);
1885 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });2244 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1886 try self.genBinOpMir(.xor, ty, limit_mcv, .{2245 try self.genBinOpMir(.xor, ty, limit_mcv, .{
...@@ -1888,7 +2247,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -1888,7 +2247,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1888 });2247 });
1889 break :cc .o;2248 break :cc .o;
1890 } else cc: {2249 } else cc: {
1891 try self.genSetReg(ty, limit_reg, .{2250 try self.genSetReg(limit_reg, ty, .{
1892 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),2251 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1893 });2252 });
1894 break :cc .c;2253 break :cc .c;
...@@ -1916,17 +2275,18 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1916,17 +2275,18 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1916 .Int => {2275 .Int => {
1917 try self.spillEflagsIfOccupied();2276 try self.spillEflagsIfOccupied();
19182277
1919 const partial_mcv = switch (tag) {2278 const partial_mcv = try self.genBinOp(null, switch (tag) {
1920 .add_with_overflow => try self.genBinOp(null, .add, bin_op.lhs, bin_op.rhs),2279 .add_with_overflow => .add,
1921 .sub_with_overflow => try self.genBinOp(null, .sub, bin_op.lhs, bin_op.rhs),2280 .sub_with_overflow => .sub,
1922 else => unreachable,2281 else => unreachable,
1923 };2282 }, bin_op.lhs, bin_op.rhs);
1924 const int_info = ty.intInfo(self.target.*);2283 const int_info = ty.intInfo(self.target.*);
1925 const cc: Condition = switch (int_info.signedness) {2284 const cc: Condition = switch (int_info.signedness) {
1926 .unsigned => .c,2285 .unsigned => .c,
1927 .signed => .o,2286 .signed => .o,
1928 };2287 };
19292288
2289 const tuple_ty = self.air.typeOfIndex(inst);
1930 if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {2290 if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
1931 switch (partial_mcv) {2291 switch (partial_mcv) {
1932 .register => |reg| {2292 .register => |reg| {
...@@ -1936,27 +2296,32 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1936,27 +2296,32 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1936 else => {},2296 else => {},
1937 }2297 }
19382298
1939 const abi_size = @intCast(i32, ty.abiSize(self.target.*));2299 const frame_index =
1940 const dst_mcv = try self.allocRegOrMem(inst, false);2300 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
1941 try self.genSetStack(2301 try self.genSetMem(
2302 .{ .frame = frame_index },
2303 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
1942 Type.u1,2304 Type.u1,
1943 dst_mcv.stack_offset - abi_size,
1944 .{ .eflags = cc },2305 .{ .eflags = cc },
1945 .{},
1946 );2306 );
1947 try self.genSetStack(ty, dst_mcv.stack_offset, partial_mcv, .{});2307 try self.genSetMem(
1948 break :result dst_mcv;2308 .{ .frame = frame_index },
2309 @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
2310 ty,
2311 partial_mcv,
2312 );
2313 break :result .{ .load_frame = .{ .index = frame_index } };
1949 }2314 }
19502315
1951 const tuple_ty = self.air.typeOfIndex(inst);2316 const frame_index =
1952 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));2317 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
1953 const tuple_align = tuple_ty.abiAlignment(self.target.*);2318 try self.genSetFrameTruncatedOverflowCompare(
1954 const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));2319 tuple_ty,
1955 const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));2320 frame_index,
19562321 partial_mcv.register,
1957 try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, partial_mcv.register, cc);2322 cc,
19582323 );
1959 break :result .{ .stack_offset = stack_offset };2324 break :result .{ .load_frame = .{ .index = frame_index } };
1960 },2325 },
1961 else => unreachable,2326 else => unreachable,
1962 }2327 }
...@@ -1998,6 +2363,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1998,6 +2363,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1998 try self.genBinOpMir(.cmp, lhs_ty, tmp_mcv, lhs);2363 try self.genBinOpMir(.cmp, lhs_ty, tmp_mcv, lhs);
1999 const cc = Condition.ne;2364 const cc = Condition.ne;
20002365
2366 const tuple_ty = self.air.typeOfIndex(inst);
2001 if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {2367 if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
2002 switch (partial_mcv) {2368 switch (partial_mcv) {
2003 .register => |reg| {2369 .register => |reg| {
...@@ -2007,27 +2373,32 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2007,27 +2373,32 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2007 else => {},2373 else => {},
2008 }2374 }
20092375
2010 const abi_size = @intCast(i32, lhs_ty.abiSize(self.target.*));2376 const frame_index =
2011 const dst_mcv = try self.allocRegOrMem(inst, false);2377 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2012 try self.genSetStack(2378 try self.genSetMem(
2013 Type.u1,2379 .{ .frame = frame_index },
2014 dst_mcv.stack_offset - abi_size,2380 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
2381 tuple_ty.structFieldType(1),
2015 .{ .eflags = cc },2382 .{ .eflags = cc },
2016 .{},
2017 );2383 );
2018 try self.genSetStack(lhs_ty, dst_mcv.stack_offset, partial_mcv, .{});2384 try self.genSetMem(
2019 break :result dst_mcv;2385 .{ .frame = frame_index },
2386 @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
2387 tuple_ty.structFieldType(0),
2388 partial_mcv,
2389 );
2390 break :result .{ .load_frame = .{ .index = frame_index } };
2020 }2391 }
20212392
2022 const tuple_ty = self.air.typeOfIndex(inst);2393 const frame_index =
2023 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));2394 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2024 const tuple_align = tuple_ty.abiAlignment(self.target.*);2395 try self.genSetFrameTruncatedOverflowCompare(
2025 const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));2396 tuple_ty,
2026 const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));2397 frame_index,
20272398 partial_mcv.register,
2028 try self.genSetStackTruncatedOverflowCompare(lhs_ty, stack_offset, overflow_bit_offset, partial_mcv.register, cc);2399 cc,
20292400 );
2030 break :result .{ .stack_offset = stack_offset };2401 break :result .{ .load_frame = .{ .index = frame_index } };
2031 },2402 },
2032 else => unreachable,2403 else => unreachable,
2033 }2404 }
...@@ -2035,17 +2406,17 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2035,17 +2406,17 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2035 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });2406 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2036}2407}
20372408
2038fn genSetStackTruncatedOverflowCompare(2409fn genSetFrameTruncatedOverflowCompare(
2039 self: *Self,2410 self: *Self,
2040 ty: Type,2411 tuple_ty: Type,
2041 stack_offset: i32,2412 frame_index: FrameIndex,
2042 overflow_bit_offset: i32,
2043 reg: Register,2413 reg: Register,
2044 cc: Condition,2414 cc: Condition,
2045) !void {2415) !void {
2046 const reg_lock = self.register_manager.lockReg(reg);2416 const reg_lock = self.register_manager.lockReg(reg);
2047 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);2417 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
20482418
2419 const ty = tuple_ty.structFieldType(0);
2049 const int_info = ty.intInfo(self.target.*);2420 const int_info = ty.intInfo(self.target.*);
2050 const extended_ty = switch (int_info.signedness) {2421 const extended_ty = switch (int_info.signedness) {
2051 .signed => Type.isize,2422 .signed => Type.isize,
...@@ -2062,7 +2433,7 @@ fn genSetStackTruncatedOverflowCompare(...@@ -2062,7 +2433,7 @@ fn genSetStackTruncatedOverflowCompare(
2062 try self.asmSetccRegister(overflow_reg.to8(), cc);2433 try self.asmSetccRegister(overflow_reg.to8(), cc);
20632434
2064 const scratch_reg = temp_regs[1];2435 const scratch_reg = temp_regs[1];
2065 try self.genSetReg(extended_ty, scratch_reg, .{ .register = reg });2436 try self.genSetReg(scratch_reg, extended_ty, .{ .register = reg });
2066 try self.truncateRegister(ty, scratch_reg);2437 try self.truncateRegister(ty, scratch_reg);
2067 try self.genBinOpMir(2438 try self.genBinOpMir(
2068 .cmp,2439 .cmp,
...@@ -2080,10 +2451,18 @@ fn genSetStackTruncatedOverflowCompare(...@@ -2080,10 +2451,18 @@ fn genSetStackTruncatedOverflowCompare(
2080 .{ .register = eq_reg },2451 .{ .register = eq_reg },
2081 );2452 );
20822453
2083 try self.genSetStack(ty, stack_offset, .{ .register = scratch_reg }, .{});2454 try self.genSetMem(
2084 try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{2455 .{ .frame = frame_index },
2085 .register = overflow_reg.to8(),2456 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
2086 }, .{});2457 tuple_ty.structFieldType(1),
2458 .{ .register = overflow_reg.to8() },
2459 );
2460 try self.genSetMem(
2461 .{ .frame = frame_index },
2462 @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
2463 ty,
2464 .{ .register = scratch_reg },
2465 );
2087}2466}
20882467
2089fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {2468fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
...@@ -2101,6 +2480,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2101,6 +2480,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2101 .unsigned => .c,2480 .unsigned => .c,
2102 .signed => .o,2481 .signed => .o,
2103 };2482 };
2483
2484 const tuple_ty = self.air.typeOfIndex(inst);
2104 if (dst_info.bits >= 8 and math.isPowerOfTwo(dst_info.bits)) {2485 if (dst_info.bits >= 8 and math.isPowerOfTwo(dst_info.bits)) {
2105 var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {2486 var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {
2106 .signed => .int_signed,2487 .signed => .int_signed,
...@@ -2127,16 +2508,21 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2127,16 +2508,21 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
21272508
2128 // For now, this is the only supported multiply that doesn't fit in a register.2509 // For now, this is the only supported multiply that doesn't fit in a register.
2129 assert(dst_info.bits == 128 and src_pl.data == 64);2510 assert(dst_info.bits == 128 and src_pl.data == 64);
2130 const dst_abi_size = @intCast(i32, dst_ty.abiSize(self.target.*));2511 const frame_index =
2131 const dst_mcv = try self.allocRegOrMem(inst, false);2512 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2132 try self.genSetStack(2513 try self.genSetMem(
2133 Type.u1,2514 .{ .frame = frame_index },
2134 dst_mcv.stack_offset - dst_abi_size,2515 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
2135 .{ .immediate = 0 }, // 64x64 -> 128 never overflows2516 tuple_ty.structFieldType(1),
2136 .{},2517 .{ .immediate = 0 }, // overflow is impossible for 64-bit*64-bit -> 128-bit
2137 );2518 );
2138 try self.genSetStack(dst_ty, dst_mcv.stack_offset, partial_mcv, .{});2519 try self.genSetMem(
2139 break :result dst_mcv;2520 .{ .frame = frame_index },
2521 @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
2522 tuple_ty.structFieldType(0),
2523 partial_mcv,
2524 );
2525 break :result .{ .load_frame = .{ .index = frame_index } };
2140 }2526 }
21412527
2142 const dst_reg: Register = dst_reg: {2528 const dst_reg: Register = dst_reg: {
...@@ -2184,15 +2570,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2184,15 +2570,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2184 }2570 }
2185 };2571 };
21862572
2187 const tuple_ty = self.air.typeOfIndex(inst);2573 const frame_index =
2188 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));2574 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2189 const tuple_align = tuple_ty.abiAlignment(self.target.*);2575 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, dst_reg, cc);
2190 const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));2576 break :result .{ .load_frame = .{ .index = frame_index } };
2191 const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));
2192
2193 try self.genSetStackTruncatedOverflowCompare(dst_ty, stack_offset, overflow_bit_offset, dst_reg, cc);
2194
2195 break :result .{ .stack_offset = stack_offset };
2196 },2577 },
2197 else => unreachable,2578 else => unreachable,
2198 }2579 }
...@@ -2215,31 +2596,40 @@ fn genIntMulDivOpMir(...@@ -2215,31 +2596,40 @@ fn genIntMulDivOpMir(
2215 return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});2596 return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
2216 }2597 }
22172598
2218 lhs: {2599 try self.genSetReg(.rax, ty, lhs);
2219 switch (lhs) {
2220 .register => |reg| if (reg.to64() == .rax) break :lhs,
2221 else => {},
2222 }
2223 try self.genSetReg(ty, .rax, lhs);
2224 }
2225
2226 switch (tag) {2600 switch (tag) {
2227 else => unreachable,2601 else => unreachable,
2228 .mul, .imul => {},2602 .mul, .imul => {},
2229 .div => try self.asmRegisterRegister(.xor, .edx, .edx),2603 .div => try self.asmRegisterRegister(.xor, .edx, .edx),
2230 .idiv => try self.asmOpOnly(.cqo),2604 .idiv => switch (self.regBitSize(ty)) {
2605 8 => try self.asmOpOnly(.cbw),
2606 16 => try self.asmOpOnly(.cwd),
2607 32 => try self.asmOpOnly(.cdq),
2608 64 => try self.asmOpOnly(.cqo),
2609 else => unreachable,
2610 },
2231 }2611 }
22322612
2233 const factor: MCValue = switch (rhs) {2613 const mat_rhs: MCValue = switch (rhs) {
2234 .register, .stack_offset => rhs,2614 .register, .indirect, .load_frame => rhs,
2235 else => .{ .register = try self.copyToTmpRegister(ty, rhs) },2615 else => .{ .register = try self.copyToTmpRegister(ty, rhs) },
2236 };2616 };
2237 switch (factor) {2617 switch (mat_rhs) {
2238 .register => |reg| try self.asmRegister(tag, reg),2618 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),
2239 .stack_offset => |off| try self.asmMemory(tag, Memory.sib(2619 .indirect, .load_frame => try self.asmMemory(
2240 Memory.PtrSize.fromSize(abi_size),2620 tag,
2241 .{ .base = .rbp, .disp = -off },2621 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (mat_rhs) {
2242 )),2622 .indirect => |reg_off| .{
2623 .base = .{ .reg = reg_off.reg },
2624 .disp = reg_off.off,
2625 },
2626 .load_frame => |frame_addr| .{
2627 .base = .{ .frame = frame_addr.index },
2628 .disp = frame_addr.off,
2629 },
2630 else => unreachable,
2631 }),
2632 ),
2243 else => unreachable,2633 else => unreachable,
2244 }2634 }
2245}2635}
...@@ -2247,7 +2637,8 @@ fn genIntMulDivOpMir(...@@ -2247,7 +2637,8 @@ fn genIntMulDivOpMir(
2247/// Always returns a register.2637/// Always returns a register.
2248/// Clobbers .rax and .rdx registers.2638/// Clobbers .rax and .rdx registers.
2249fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {2639fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
2250 const signedness = ty.intInfo(self.target.*).signedness;2640 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
2641 const int_info = ty.intInfo(self.target.*);
2251 const dividend: Register = switch (lhs) {2642 const dividend: Register = switch (lhs) {
2252 .register => |reg| reg,2643 .register => |reg| reg,
2253 else => try self.copyToTmpRegister(ty, lhs),2644 else => try self.copyToTmpRegister(ty, lhs),
...@@ -2262,16 +2653,32 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa...@@ -2262,16 +2653,32 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
2262 const divisor_lock = self.register_manager.lockReg(divisor);2653 const divisor_lock = self.register_manager.lockReg(divisor);
2263 defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock);2654 defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock);
22642655
2265 try self.genIntMulDivOpMir(switch (signedness) {2656 try self.genIntMulDivOpMir(switch (int_info.signedness) {
2266 .signed => .idiv,2657 .signed => .idiv,
2267 .unsigned => .div,2658 .unsigned => .div,
2268 }, Type.isize, .{ .register = dividend }, .{ .register = divisor });2659 }, ty, .{ .register = dividend }, .{ .register = divisor });
22692660
2270 try self.asmRegisterRegister(.xor, divisor.to64(), dividend.to64());2661 try self.asmRegisterRegister(
2271 try self.asmRegisterImmediate(.sar, divisor.to64(), Immediate.u(63));2662 .xor,
2272 try self.asmRegisterRegister(.@"test", .rdx, .rdx);2663 registerAlias(divisor, abi_size),
2273 try self.asmCmovccRegisterRegister(divisor.to64(), .rdx, .e);2664 registerAlias(dividend, abi_size),
2274 try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });2665 );
2666 try self.asmRegisterImmediate(
2667 .sar,
2668 registerAlias(divisor, abi_size),
2669 Immediate.u(int_info.bits - 1),
2670 );
2671 try self.asmRegisterRegister(
2672 .@"test",
2673 registerAlias(.rdx, abi_size),
2674 registerAlias(.rdx, abi_size),
2675 );
2676 try self.asmCmovccRegisterRegister(
2677 registerAlias(divisor, abi_size),
2678 registerAlias(.rdx, abi_size),
2679 .z,
2680 );
2681 try self.genBinOpMir(.add, ty, .{ .register = divisor }, .{ .register = .rax });
2275 return MCValue{ .register = divisor };2682 return MCValue{ .register = divisor };
2276}2683}
22772684
...@@ -2315,7 +2722,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -2315,7 +2722,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
2315 }2722 }
23162723
2317 const pl_mcv = try self.allocRegOrMem(inst, true);2724 const pl_mcv = try self.allocRegOrMem(inst, true);
2318 try self.setRegOrMem(pl_ty, pl_mcv, switch (opt_mcv) {2725 try self.genCopy(pl_ty, pl_mcv, switch (opt_mcv) {
2319 else => opt_mcv,2726 else => opt_mcv,
2320 .register_overflow => |ro| .{ .register = ro.reg },2727 .register_overflow => |ro| .{ .register = ro.reg },
2321 });2728 });
...@@ -2361,11 +2768,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {...@@ -2361,11 +2768,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
23612768
2362 const pl_ty = dst_ty.childType();2769 const pl_ty = dst_ty.childType();
2363 const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));2770 const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
2364 try self.asmMemoryImmediate(2771 try self.genSetMem(.{ .reg = dst_mcv.register }, pl_abi_size, Type.bool, .{ .immediate = 1 });
2365 .mov,
2366 Memory.sib(.byte, .{ .base = dst_mcv.register, .disp = pl_abi_size }),
2367 Immediate.u(1),
2368 );
2369 break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv;2772 break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv;
2370 };2773 };
2371 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });2774 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...@@ -2389,10 +2792,6 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2389,10 +2792,6 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
23892792
2390 const err_off = errUnionErrorOffset(payload_ty, self.target.*);2793 const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2391 switch (operand) {2794 switch (operand) {
2392 .stack_offset => |off| {
2393 const offset = off - @intCast(i32, err_off);
2394 break :result MCValue{ .stack_offset = offset };
2395 },
2396 .register => |reg| {2795 .register => |reg| {
2397 // TODO reuse operand2796 // TODO reuse operand
2398 const eu_lock = self.register_manager.lockReg(reg);2797 const eu_lock = self.register_manager.lockReg(reg);
...@@ -2407,6 +2806,10 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2407,6 +2806,10 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2407 }2806 }
2408 break :result result;2807 break :result result;
2409 },2808 },
2809 .load_frame => |frame_addr| break :result .{ .load_frame = .{
2810 .index = frame_addr.index,
2811 .off = frame_addr.off + @intCast(i32, err_off),
2812 } },
2410 else => return self.fail("TODO implement unwrap_err_err for {}", .{operand}),2813 else => return self.fail("TODO implement unwrap_err_err for {}", .{operand}),
2411 }2814 }
2412 };2815 };
...@@ -2434,10 +2837,10 @@ fn genUnwrapErrorUnionPayloadMir(...@@ -2434,10 +2837,10 @@ fn genUnwrapErrorUnionPayloadMir(
24342837
2435 const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);2838 const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2436 switch (err_union) {2839 switch (err_union) {
2437 .stack_offset => |off| {2840 .load_frame => |frame_addr| break :result .{ .load_frame = .{
2438 const offset = off - @intCast(i32, payload_off);2841 .index = frame_addr.index,
2439 break :result MCValue{ .stack_offset = offset };2842 .off = frame_addr.off + @intCast(i32, payload_off),
2440 },2843 } },
2441 .register => |reg| {2844 .register => |reg| {
2442 // TODO reuse operand2845 // TODO reuse operand
2443 const eu_lock = self.register_manager.lockReg(reg);2846 const eu_lock = self.register_manager.lockReg(reg);
...@@ -2488,7 +2891,10 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2488,7 +2891,10 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2488 try self.asmRegisterMemory(2891 try self.asmRegisterMemory(
2489 .mov,2892 .mov,
2490 registerAlias(dst_reg, err_abi_size),2893 registerAlias(dst_reg, err_abi_size),
2491 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),2894 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
2895 .base = .{ .reg = src_reg },
2896 .disp = err_off,
2897 }),
2492 );2898 );
24932899
2494 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });2900 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...@@ -2523,7 +2929,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2523,7 +2929,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
2523 try self.asmRegisterMemory(2929 try self.asmRegisterMemory(
2524 .lea,2930 .lea,
2525 registerAlias(dst_reg, dst_abi_size),2931 registerAlias(dst_reg, dst_abi_size),
2526 Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),2932 Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
2527 );2933 );
25282934
2529 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });2935 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...@@ -2548,7 +2954,10 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {...@@ -2548,7 +2954,10 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2548 const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));2954 const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
2549 try self.asmMemoryImmediate(2955 try self.asmMemoryImmediate(
2550 .mov,2956 .mov,
2551 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),2957 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
2958 .base = .{ .reg = src_reg },
2959 .disp = err_off,
2960 }),
2552 Immediate.u(0),2961 Immediate.u(0),
2553 );2962 );
25542963
...@@ -2567,7 +2976,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {...@@ -2567,7 +2976,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2567 try self.asmRegisterMemory(2976 try self.asmRegisterMemory(
2568 .lea,2977 .lea,
2569 registerAlias(dst_reg, dst_abi_size),2978 registerAlias(dst_reg, dst_abi_size),
2570 Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),2979 Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
2571 );2980 );
2572 break :result .{ .register = dst_reg };2981 break :result .{ .register = dst_reg };
2573 };2982 };
...@@ -2608,7 +3017,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {...@@ -2608,7 +3017,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2608 defer if (pl_lock) |lock| self.register_manager.unlockReg(lock);3017 defer if (pl_lock) |lock| self.register_manager.unlockReg(lock);
26093018
2610 const opt_mcv = try self.allocRegOrMem(inst, true);3019 const opt_mcv = try self.allocRegOrMem(inst, true);
2611 try self.setRegOrMem(pl_ty, opt_mcv, pl_mcv);3020 try self.genCopy(pl_ty, opt_mcv, pl_mcv);
26123021
2613 if (!same_repr) {3022 if (!same_repr) {
2614 const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));3023 const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
...@@ -2621,9 +3030,12 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {...@@ -2621,9 +3030,12 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2621 Immediate.u(@intCast(u6, pl_abi_size * 8)),3030 Immediate.u(@intCast(u6, pl_abi_size * 8)),
2622 ),3031 ),
26233032
2624 .stack_offset => |off| try self.asmMemoryImmediate(3033 .load_frame => |frame_addr| try self.asmMemoryImmediate(
2625 .mov,3034 .mov,
2626 Memory.sib(.byte, .{ .base = .rbp, .disp = pl_abi_size - off }),3035 Memory.sib(.byte, .{
3036 .base = .{ .frame = frame_addr.index },
3037 .disp = frame_addr.off + pl_abi_size,
3038 }),
2627 Immediate.u(1),3039 Immediate.u(1),
2628 ),3040 ),
2629 }3041 }
...@@ -2637,26 +3049,21 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {...@@ -2637,26 +3049,21 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2637fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {3049fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2638 const ty_op = self.air.instructions.items(.data)[inst].ty_op;3050 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
26393051
2640 const error_union_ty = self.air.getRefType(ty_op.ty);3052 const eu_ty = self.air.getRefType(ty_op.ty);
2641 const payload_ty = error_union_ty.errorUnionPayload();3053 const pl_ty = eu_ty.errorUnionPayload();
3054 const err_ty = eu_ty.errorUnionSet();
2642 const operand = try self.resolveInst(ty_op.operand);3055 const operand = try self.resolveInst(ty_op.operand);
26433056
2644 const result: MCValue = result: {3057 const result: MCValue = result: {
2645 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {3058 if (!pl_ty.hasRuntimeBitsIgnoreComptime()) break :result .{ .immediate = 0 };
2646 break :result operand;
2647 }
2648
2649 const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
2650 const abi_align = error_union_ty.abiAlignment(self.target.*);
2651 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
2652 const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2653 const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2654 try self.genSetStack(payload_ty, stack_offset - @intCast(i32, payload_off), operand, .{});
2655 try self.genSetStack(Type.anyerror, stack_offset - @intCast(i32, err_off), .{ .immediate = 0 }, .{});
26563059
2657 break :result MCValue{ .stack_offset = stack_offset };3060 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, self.target.*));
3061 const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3062 const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3063 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
3064 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 });
3065 break :result .{ .load_frame = .{ .index = frame_index } };
2658 };3066 };
2659
2660 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3067 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2661}3068}
26623069
...@@ -2664,26 +3071,21 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -2664,26 +3071,21 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2664fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {3071fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2665 const ty_op = self.air.instructions.items(.data)[inst].ty_op;3072 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
26663073
2667 const error_union_ty = self.air.getRefType(ty_op.ty);3074 const eu_ty = self.air.getRefType(ty_op.ty);
2668 const payload_ty = error_union_ty.errorUnionPayload();3075 const pl_ty = eu_ty.errorUnionPayload();
2669 const operand = try self.resolveInst(ty_op.operand);3076 const err_ty = eu_ty.errorUnionSet();
26703077
2671 const result: MCValue = result: {3078 const result: MCValue = result: {
2672 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {3079 if (!pl_ty.hasRuntimeBitsIgnoreComptime()) break :result try self.resolveInst(ty_op.operand);
2673 break :result operand;
2674 }
2675
2676 const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
2677 const abi_align = error_union_ty.abiAlignment(self.target.*);
2678 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
2679 const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2680 const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2681 try self.genSetStack(Type.anyerror, stack_offset - @intCast(i32, err_off), operand, .{});
2682 try self.genSetStack(payload_ty, stack_offset - @intCast(i32, payload_off), .undef, .{});
26833080
2684 break :result MCValue{ .stack_offset = stack_offset };3081 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, self.target.*));
3082 const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3083 const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3084 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);
3085 const operand = try self.resolveInst(ty_op.operand);
3086 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand);
3087 break :result .{ .load_frame = .{ .index = frame_index } };
2685 };3088 };
2686
2687 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3089 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2688}3090}
26893091
...@@ -2695,7 +3097,7 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2695,7 +3097,7 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
26953097
2696 const dst_mcv = try self.allocRegOrMem(inst, true);3098 const dst_mcv = try self.allocRegOrMem(inst, true);
2697 const dst_ty = self.air.typeOfIndex(inst);3099 const dst_ty = self.air.typeOfIndex(inst);
2698 try self.setRegOrMem(dst_ty, dst_mcv, src_mcv);3100 try self.genCopy(dst_ty, dst_mcv, src_mcv);
2699 break :result dst_mcv;3101 break :result dst_mcv;
2700 };3102 };
2701 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3103 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...@@ -2707,9 +3109,10 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {...@@ -2707,9 +3109,10 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2707 const operand = try self.resolveInst(ty_op.operand);3109 const operand = try self.resolveInst(ty_op.operand);
2708 const dst_mcv: MCValue = blk: {3110 const dst_mcv: MCValue = blk: {
2709 switch (operand) {3111 switch (operand) {
2710 .stack_offset => |off| {3112 .load_frame => |frame_addr| break :blk .{ .load_frame = .{
2711 break :blk MCValue{ .stack_offset = off - 8 };3113 .index = frame_addr.index,
2712 },3114 .off = frame_addr.off + 8,
3115 } },
2713 else => return self.fail("TODO implement slice_len for {}", .{operand}),3116 else => return self.fail("TODO implement slice_len for {}", .{operand}),
2714 }3117 }
2715 };3118 };
...@@ -2743,7 +3146,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2743,7 +3146,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
2743 .lea,3146 .lea,
2744 registerAlias(dst_reg, dst_abi_size),3147 registerAlias(dst_reg, dst_abi_size),
2745 Memory.sib(.qword, .{3148 Memory.sib(.qword, .{
2746 .base = src_reg,3149 .base = .{ .reg = src_reg },
2747 .disp = @divExact(self.target.cpu.arch.ptrBitWidth(), 8),3150 .disp = @divExact(self.target.cpu.arch.ptrBitWidth(), 8),
2748 }),3151 }),
2749 );3152 );
...@@ -2771,7 +3174,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi...@@ -2771,7 +3174,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
2771 // Optimisation: if index MCValue is an immediate, we can multiply in `comptime`3174 // Optimisation: if index MCValue is an immediate, we can multiply in `comptime`
2772 // and set the register directly to the scaled offset as an immediate.3175 // and set the register directly to the scaled offset as an immediate.
2773 const reg = try self.register_manager.allocReg(null, gp);3176 const reg = try self.register_manager.allocReg(null, gp);
2774 try self.genSetReg(index_ty, reg, .{ .immediate = imm * elem_size });3177 try self.genSetReg(reg, index_ty, .{ .immediate = imm * elem_size });
2775 break :blk reg;3178 break :blk reg;
2776 },3179 },
2777 else => {3180 else => {
...@@ -2812,10 +3215,14 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {...@@ -2812,10 +3215,14 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
28123215
2813 const addr_reg = try self.register_manager.allocReg(null, gp);3216 const addr_reg = try self.register_manager.allocReg(null, gp);
2814 switch (slice_mcv) {3217 switch (slice_mcv) {
2815 .stack_offset => |off| try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{3218 .load_frame => |frame_addr| try self.asmRegisterMemory(
2816 .base = .rbp,3219 .mov,
2817 .disp = -off,3220 addr_reg.to64(),
2818 })),3221 Memory.sib(.qword, .{
3222 .base = .{ .frame = frame_addr.index },
3223 .disp = frame_addr.off,
3224 }),
3225 ),
2819 else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),3226 else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),
2820 }3227 }
2821 // TODO we could allocate register here, but need to expect addr register and potentially3228 // TODO we could allocate register here, but need to expect addr register and potentially
...@@ -2834,7 +3241,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2834,7 +3241,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2834 const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);3241 const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
2835 const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);3242 const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
2836 const dst_mcv = try self.allocRegOrMem(inst, false);3243 const dst_mcv = try self.allocRegOrMem(inst, false);
2837 try self.load(dst_mcv, elem_ptr, slice_ptr_field_type);3244 try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
28383245
2839 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });3246 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2840}3247}
...@@ -2872,33 +3279,27 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2872,33 +3279,27 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2872 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);3279 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2873 defer self.register_manager.unlockReg(offset_reg_lock);3280 defer self.register_manager.unlockReg(offset_reg_lock);
28743281
2875 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();3282 const addr_reg = try self.register_manager.allocReg(null, gp);
2876 switch (array) {3283 switch (array) {
2877 .register => {3284 .register => {
2878 const off = @intCast(i32, try self.allocMem(3285 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, self.target.*));
2879 inst,3286 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array);
2880 @intCast(u32, array_ty.abiSize(self.target.*)),3287 try self.asmRegisterMemory(
2881 array_ty.abiAlignment(self.target.*),3288 .lea,
2882 ));3289 addr_reg,
2883 try self.genSetStack(array_ty, off, array, .{});3290 Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
2884 try self.asmRegisterMemory(.lea, addr_reg, Memory.sib(.qword, .{3291 );
2885 .base = .rbp,
2886 .disp = -off,
2887 }));
2888 },
2889 .stack_offset => |off| {
2890 try self.asmRegisterMemory(.lea, addr_reg, Memory.sib(.qword, .{
2891 .base = .rbp,
2892 .disp = -off,
2893 }));
2894 },3292 },
2895 .load_got => try self.genSetReg(array_ty, addr_reg, array),3293 .load_frame => |frame_addr| try self.asmRegisterMemory(
2896 .memory, .load_direct, .load_tlv => try self.genSetReg(Type.usize, addr_reg, switch (array) {3294 .lea,
2897 .memory => |addr| .{ .immediate = addr },3295 addr_reg,
2898 .load_direct => |sym_index| .{ .lea_direct = sym_index },3296 Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),
2899 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },3297 ),
2900 else => unreachable,3298 .memory,
2901 }),3299 .load_direct,
3300 .load_got,
3301 .load_tlv,
3302 => try self.genSetReg(addr_reg, Type.usize, array.address()),
2902 .lea_direct, .lea_tlv => unreachable,3303 .lea_direct, .lea_tlv => unreachable,
2903 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),3304 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
2904 }3305 }
...@@ -2907,7 +3308,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2907,7 +3308,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2907 // offset register.3308 // offset register.
2908 const dst_mcv = try self.allocRegOrMem(inst, false);3309 const dst_mcv = try self.allocRegOrMem(inst, false);
2909 try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });3310 try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
2910 try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty);3311 try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
29113312
2912 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });3313 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2913}3314}
...@@ -2948,7 +3349,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2948,7 +3349,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2948 else => null,3349 else => null,
2949 };3350 };
2950 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);3351 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
2951 try self.load(dst_mcv, .{ .register = elem_ptr_reg }, ptr_ty);3352 try self.load(dst_mcv, ptr_ty, .{ .register = elem_ptr_reg });
29523353
2953 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });3354 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2954}3355}
...@@ -2987,8 +3388,8 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2987,8 +3388,8 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
29873388
2988fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {3389fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2989 const bin_op = self.air.instructions.items(.data)[inst].bin_op;3390 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2990 const ptr_ty = self.air.typeOf(bin_op.lhs);3391 const ptr_union_ty = self.air.typeOf(bin_op.lhs);
2991 const union_ty = ptr_ty.childType();3392 const union_ty = ptr_union_ty.childType();
2992 const tag_ty = self.air.typeOf(bin_op.rhs);3393 const tag_ty = self.air.typeOf(bin_op.rhs);
2993 const layout = union_ty.unionGetLayout(self.target.*);3394 const layout = union_ty.unionGetLayout(self.target.*);
29943395
...@@ -3012,12 +3413,15 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {...@@ -3012,12 +3413,15 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
30123413
3013 const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {3414 const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
3014 // TODO reusing the operand3415 // TODO reusing the operand
3015 const reg = try self.copyToTmpRegister(ptr_ty, ptr);3416 const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);
3016 try self.genBinOpMir(.add, ptr_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });3417 try self.genBinOpMir(.add, ptr_union_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });
3017 break :blk MCValue{ .register = reg };3418 break :blk MCValue{ .register = reg };
3018 } else ptr;3419 } else ptr;
30193420
3020 try self.store(adjusted_ptr, tag, ptr_ty, tag_ty);3421 var ptr_tag_pl = ptr_union_ty.ptrInfo();
3422 ptr_tag_pl.data.pointee_type = tag_ty;
3423 const ptr_tag_ty = Type.initPayload(&ptr_tag_pl.base);
3424 try self.store(ptr_tag_ty, adjusted_ptr, tag);
30213425
3022 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });3426 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
3023}3427}
...@@ -3044,11 +3448,14 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {...@@ -3044,11 +3448,14 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3044 const tag_abi_size = tag_ty.abiSize(self.target.*);3448 const tag_abi_size = tag_ty.abiSize(self.target.*);
3045 const dst_mcv: MCValue = blk: {3449 const dst_mcv: MCValue = blk: {
3046 switch (operand) {3450 switch (operand) {
3047 .stack_offset => |off| {3451 .load_frame => |frame_addr| {
3048 if (tag_abi_size <= 8) {3452 if (tag_abi_size <= 8) {
3049 const offset: i32 = if (layout.tag_align < layout.payload_align) @intCast(i32, layout.payload_size) else 0;3453 const off: i32 = if (layout.tag_align < layout.payload_align)
3454 @intCast(i32, layout.payload_size)
3455 else
3456 0;
3050 break :blk try self.copyToRegisterWithInstTracking(inst, tag_ty, .{3457 break :blk try self.copyToRegisterWithInstTracking(inst, tag_ty, .{
3051 .stack_offset = off - offset,3458 .load_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
3052 });3459 });
3053 }3460 }
30543461
...@@ -3131,7 +3538,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -3131,7 +3538,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3131 .nz,3538 .nz,
3132 );3539 );
31333540
3134 try self.genSetReg(dst_ty, dst_reg, .{ .immediate = src_bits - 1 });3541 try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });
3135 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });3542 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });
3136 }3543 }
3137 break :result dst_mcv;3544 break :result dst_mcv;
...@@ -3171,7 +3578,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -3171,7 +3578,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
3171 };3578 };
3172 const tmp_mcv = tmp: {3579 const tmp_mcv = tmp: {
3173 if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) break :tmp src_mcv;3580 if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) break :tmp src_mcv;
3174 try self.genSetReg(src_ty, dst_reg, src_mcv);3581 try self.genSetReg(dst_reg, src_ty, src_mcv);
3175 break :tmp dst_mcv;3582 break :tmp dst_mcv;
3176 };3583 };
3177 try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);3584 try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);
...@@ -3345,7 +3752,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m...@@ -3345,7 +3752,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
3345 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);3752 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);
3346 defer self.register_manager.unlockReg(dst_lock);3753 defer self.register_manager.unlockReg(dst_lock);
33473754
3348 try self.genSetReg(src_ty, dst_mcv.register, src_mcv);3755 try self.genSetReg(dst_mcv.register, src_ty, src_mcv);
3349 switch (src_bits) {3756 switch (src_bits) {
3350 else => unreachable,3757 else => unreachable,
3351 16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),3758 16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),
...@@ -3450,7 +3857,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {...@@ -3450,7 +3857,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
3450 .lea,3857 .lea,
3451 if (src_abi_size > 4) tmp.to64() else tmp.to32(),3858 if (src_abi_size > 4) tmp.to64() else tmp.to32(),
3452 Memory.sib(.qword, .{3859 Memory.sib(.qword, .{
3453 .base = dst.to64(),3860 .base = .{ .reg = dst.to64() },
3454 .scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 },3861 .scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 },
3455 }),3862 }),
3456 );3863 );
...@@ -3473,7 +3880,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {...@@ -3473,7 +3880,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
3473 .lea,3880 .lea,
3474 if (src_abi_size > 4) dst.to64() else dst.to32(),3881 if (src_abi_size > 4) dst.to64() else dst.to32(),
3475 Memory.sib(.qword, .{3882 Memory.sib(.qword, .{
3476 .base = tmp.to64(),3883 .base = .{ .reg = tmp.to64() },
3477 .scale_index = .{ .index = dst.to64(), .scale = 1 << 1 },3884 .scale_index = .{ .index = dst.to64(), .scale = 1 << 1 },
3478 }),3885 }),
3479 );3886 );
...@@ -3506,6 +3913,17 @@ fn reuseOperand(...@@ -3506,6 +3913,17 @@ fn reuseOperand(
3506 operand: Air.Inst.Ref,3913 operand: Air.Inst.Ref,
3507 op_index: Liveness.OperandInt,3914 op_index: Liveness.OperandInt,
3508 mcv: MCValue,3915 mcv: MCValue,
3916) bool {
3917 return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
3918}
3919
3920fn reuseOperandAdvanced(
3921 self: *Self,
3922 inst: Air.Inst.Index,
3923 operand: Air.Inst.Ref,
3924 op_index: Liveness.OperandInt,
3925 mcv: MCValue,
3926 tracked_inst: Air.Inst.Index,
3509) bool {3927) bool {
3510 if (!self.liveness.operandDies(inst, op_index))3928 if (!self.liveness.operandDies(inst, op_index))
3511 return false;3929 return false;
...@@ -3516,25 +3934,23 @@ fn reuseOperand(...@@ -3516,25 +3934,23 @@ fn reuseOperand(
3516 // new instruction.3934 // new instruction.
3517 if (!self.register_manager.isRegFree(reg)) {3935 if (!self.register_manager.isRegFree(reg)) {
3518 if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {3936 if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {
3519 self.register_manager.registers[index] = inst;3937 self.register_manager.registers[index] = tracked_inst;
3520 }3938 }
3521 }3939 }
3522 log.debug("%{d} => {} (reused)", .{ inst, reg });
3523 },
3524 .stack_offset => |off| {
3525 log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
3526 },3940 },
3941 .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,
3527 else => return false,3942 else => return false,
3528 }3943 }
35293944
3530 // Prevent the operand deaths processing code from deallocating it.3945 // Prevent the operand deaths processing code from deallocating it.
3531 self.liveness.clearOperandDeath(inst, op_index);3946 self.liveness.clearOperandDeath(inst, op_index);
3532 self.getResolvedInstValue(Air.refToIndex(operand).?).reuse(self);3947 const op_inst = Air.refToIndex(operand).?;
3948 self.getResolvedInstValue(op_inst).reuse(self, tracked_inst, op_inst);
35333949
3534 return true;3950 return true;
3535}3951}
35363952
3537fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void {3953fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
3538 const ptr_info = ptr_ty.ptrInfo().data;3954 const ptr_info = ptr_ty.ptrInfo().data;
35393955
3540 const val_ty = ptr_info.pointee_type;3956 const val_ty = ptr_info.pointee_type;
...@@ -3564,10 +3980,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn...@@ -3564,10 +3980,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn
3564 if (val_bit_off < val_extra_bits) val_abi_size else val_abi_size * 2;3980 if (val_bit_off < val_extra_bits) val_abi_size else val_abi_size * 2;
3565 if (load_abi_size <= 8) {3981 if (load_abi_size <= 8) {
3566 const load_reg = registerAlias(dst_reg, load_abi_size);3982 const load_reg = registerAlias(dst_reg, load_abi_size);
3567 try self.asmRegisterMemory(.mov, load_reg, Memory.sib(3983 try self.asmRegisterMemory(
3568 Memory.PtrSize.fromSize(load_abi_size),3984 .mov,
3569 .{ .base = ptr_reg, .disp = val_byte_off },3985 load_reg,
3570 ));3986 Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
3987 .base = .{ .reg = ptr_reg },
3988 .disp = val_byte_off,
3989 }),
3990 );
3571 try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));3991 try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));
3572 } else {3992 } else {
3573 const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);3993 const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);
...@@ -3575,85 +3995,60 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn...@@ -3575,85 +3995,60 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn
3575 defer self.register_manager.unlockReg(tmp_lock);3995 defer self.register_manager.unlockReg(tmp_lock);
35763996
3577 const dst_alias = registerAlias(dst_reg, val_abi_size);3997 const dst_alias = registerAlias(dst_reg, val_abi_size);
3578 try self.asmRegisterMemory(.mov, dst_alias, Memory.sib(3998 try self.asmRegisterMemory(
3579 Memory.PtrSize.fromSize(val_abi_size),3999 .mov,
3580 .{ .base = ptr_reg, .disp = val_byte_off },4000 dst_alias,
3581 ));4001 Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
3582 try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(4002 .base = .{ .reg = ptr_reg },
3583 Memory.PtrSize.fromSize(val_abi_size),4003 .disp = val_byte_off,
3584 .{ .base = ptr_reg, .disp = val_byte_off + 1 },4004 }),
3585 ));4005 );
4006 try self.asmRegisterMemory(
4007 .mov,
4008 tmp_reg,
4009 Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
4010 .base = .{ .reg = ptr_reg },
4011 .disp = val_byte_off + 1,
4012 }),
4013 );
3586 try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));4014 try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));
3587 }4015 }
35884016
3589 if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);4017 if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
3590 try self.setRegOrMem(val_ty, dst_mcv, .{ .register = dst_reg });4018 try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg });
3591}4019}
3592
3593fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
3594 const elem_ty = ptr_ty.elemType();
3595 const abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
3596 switch (ptr) {
3597 .none => unreachable,
3598 .undef => unreachable,
3599 .unreach => unreachable,
3600 .dead => unreachable,
3601 .eflags => unreachable,
3602 .register_overflow => unreachable,
3603 .immediate => |imm| {
3604 try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm });
3605 },
3606 .stack_offset => {
3607 const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3608 try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
3609 },
3610 .ptr_stack_offset => |off| {
3611 try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off });
3612 },
3613 .register => |reg| {
3614 const reg_lock = self.register_manager.lockReg(reg);
3615 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
36164020
3617 switch (dst_mcv) {4021fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
3618 .dead => unreachable,4022 const dst_ty = ptr_ty.childType();
3619 .undef => unreachable,4023 switch (ptr_mcv) {
3620 .eflags => unreachable,4024 .none,
3621 .register => |dst_reg| {4025 .unreach,
3622 try self.asmRegisterMemory(4026 .dead,
3623 .mov,4027 .undef,
3624 registerAlias(dst_reg, abi_size),4028 .eflags,
3625 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg }),4029 .register_overflow,
3626 );4030 .reserved_frame,
3627 },4031 => unreachable, // not a valid pointer
3628 .stack_offset => |off| {4032 .immediate,
3629 if (abi_size <= 8) {4033 .register,
3630 const tmp_reg = try self.register_manager.allocReg(null, gp);4034 .register_offset,
3631 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);4035 .lea_direct,
3632 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});4036 .lea_got,
3633 }4037 .lea_tlv,
4038 .lea_frame,
4039 => try self.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()),
4040 .memory,
4041 .indirect,
4042 .load_direct,
4043 .load_got,
4044 .load_tlv,
4045 .load_frame,
4046 => {
4047 const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
4048 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4049 defer self.register_manager.unlockReg(addr_lock);
36344050
3635 try self.genInlineMemcpy(4051 try self.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
3636 .{ .ptr_stack_offset = off },
3637 ptr,
3638 .{ .immediate = abi_size },
3639 .{},
3640 );
3641 },
3642 else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
3643 }
3644 },
3645 .load_direct => |sym_index| {
3646 const addr_reg = try self.copyToTmpRegister(Type.usize, .{ .lea_direct = sym_index });
3647 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3648 defer self.register_manager.unlockReg(addr_reg_lock);
3649 // Load the pointer, which is stored in memory
3650 try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{ .base = addr_reg }));
3651 try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty);
3652 },
3653 .load_tlv => |sym_index| try self.load(dst_mcv, .{ .lea_tlv = sym_index }, ptr_ty),
3654 .memory, .load_got, .lea_direct, .lea_tlv => {
3655 const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3656 try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
3657 },4052 },
3658 }4053 }
3659}4054}
...@@ -3669,53 +4064,48 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -3669,53 +4064,48 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
3669 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });4064 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
3670 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);4065 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
36714066
3672 const ptr = try self.resolveInst(ty_op.operand);4067 const ptr_mcv = try self.resolveInst(ty_op.operand);
3673 const dst_mcv: MCValue = if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr))4068 const dst_mcv = if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr_mcv))
3674 // The MCValue that holds the pointer can be re-used as the value.4069 // The MCValue that holds the pointer can be re-used as the value.
3675 ptr4070 ptr_mcv
3676 else4071 else
3677 try self.allocRegOrMem(inst, true);4072 try self.allocRegOrMem(inst, true);
36784073
3679 const ptr_ty = self.air.typeOf(ty_op.operand);4074 const ptr_ty = self.air.typeOf(ty_op.operand);
3680 if (ptr_ty.ptrInfo().data.host_size > 0) {4075 if (ptr_ty.ptrInfo().data.host_size > 0) {
3681 try self.packedLoad(dst_mcv, ptr, ptr_ty);4076 try self.packedLoad(dst_mcv, ptr_ty, ptr_mcv);
3682 } else {4077 } else {
3683 try self.load(dst_mcv, ptr, ptr_ty);4078 try self.load(dst_mcv, ptr_ty, ptr_mcv);
3684 }4079 }
3685 break :result dst_mcv;4080 break :result dst_mcv;
3686 };4081 };
3687 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });4082 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3688}4083}
36894084
3690fn packedStore(4085fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
3691 self: *Self,
3692 ptr_mcv: MCValue,
3693 val_mcv: MCValue,
3694 ptr_ty: Type,
3695 val_ty: Type,
3696) InnerError!void {
3697 const ptr_info = ptr_ty.ptrInfo().data;4086 const ptr_info = ptr_ty.ptrInfo().data;
4087 const src_ty = ptr_ty.childType();
36984088
3699 const limb_abi_size = @min(ptr_info.host_size, 8);4089 const limb_abi_size = @min(ptr_info.host_size, 8);
3700 const limb_abi_bits = limb_abi_size * 8;4090 const limb_abi_bits = limb_abi_size * 8;
37014091
3702 const val_bit_size = val_ty.bitSize(self.target.*);4092 const src_bit_size = src_ty.bitSize(self.target.*);
3703 const val_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size);4093 const src_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size);
3704 const val_bit_off = ptr_info.bit_offset % limb_abi_bits;4094 const src_bit_off = ptr_info.bit_offset % limb_abi_bits;
37054095
3706 const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);4096 const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
3707 const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);4097 const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);
3708 defer self.register_manager.unlockReg(ptr_lock);4098 defer self.register_manager.unlockReg(ptr_lock);
37094099
3710 var limb_i: u16 = 0;4100 var limb_i: u16 = 0;
3711 while (limb_i * limb_abi_bits < val_bit_off + val_bit_size) : (limb_i += 1) {4101 while (limb_i * limb_abi_bits < src_bit_off + src_bit_size) : (limb_i += 1) {
3712 const part_bit_off = if (limb_i == 0) val_bit_off else 0;4102 const part_bit_off = if (limb_i == 0) src_bit_off else 0;
3713 const part_bit_size =4103 const part_bit_size =
3714 @min(val_bit_off + val_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off;4104 @min(src_bit_off + src_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off;
3715 const limb_mem = Memory.sib(4105 const limb_mem = Memory.sib(Memory.PtrSize.fromSize(limb_abi_size), .{
3716 Memory.PtrSize.fromSize(limb_abi_size),4106 .base = .{ .reg = ptr_reg },
3717 .{ .base = ptr_reg, .disp = val_byte_off + limb_i * limb_abi_bits },4107 .disp = src_byte_off + limb_i * limb_abi_bits,
3718 );4108 });
37194109
3720 const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - part_bit_size)) <<4110 const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - part_bit_size)) <<
3721 @intCast(u6, part_bit_off);4111 @intCast(u6, part_bit_off);
...@@ -3731,211 +4121,62 @@ fn packedStore(...@@ -3731,211 +4121,62 @@ fn packedStore(
3731 try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);4121 try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);
3732 }4122 }
37334123
3734 if (val_bit_size <= 64) {4124 if (src_bit_size <= 64) {
3735 const tmp_reg = try self.register_manager.allocReg(null, gp);4125 const tmp_reg = try self.register_manager.allocReg(null, gp);
3736 const tmp_mcv = MCValue{ .register = tmp_reg };4126 const tmp_mcv = MCValue{ .register = tmp_reg };
3737 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);4127 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3738 defer self.register_manager.unlockReg(tmp_lock);4128 defer self.register_manager.unlockReg(tmp_lock);
37394129
3740 try self.genSetReg(val_ty, tmp_reg, val_mcv);4130 try self.genSetReg(tmp_reg, src_ty, src_mcv);
3741 switch (limb_i) {4131 switch (limb_i) {
3742 0 => try self.genShiftBinOpMir(.shl, val_ty, tmp_mcv, .{ .immediate = val_bit_off }),4132 0 => try self.genShiftBinOpMir(.shl, src_ty, tmp_mcv, .{ .immediate = src_bit_off }),
3743 1 => try self.genShiftBinOpMir(.shr, val_ty, tmp_mcv, .{4133 1 => try self.genShiftBinOpMir(.shr, src_ty, tmp_mcv, .{
3744 .immediate = limb_abi_bits - val_bit_off,4134 .immediate = limb_abi_bits - src_bit_off,
3745 }),4135 }),
3746 else => unreachable,4136 else => unreachable,
3747 }4137 }
3748 try self.genBinOpMir(.@"and", val_ty, tmp_mcv, .{ .immediate = part_mask });4138 try self.genBinOpMir(.@"and", src_ty, tmp_mcv, .{ .immediate = part_mask });
3749 try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));4139 try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));
3750 } else return self.fail("TODO: implement packed store of {}", .{4140 } else return self.fail("TODO: implement packed store of {}", .{
3751 val_ty.fmt(self.bin_file.options.module.?),4141 src_ty.fmt(self.bin_file.options.module.?),
3752 });4142 });
3753 }4143 }
3754}4144}
37554145
3756fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {4146fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
3757 const abi_size = @intCast(u32, value_ty.abiSize(self.target.*));4147 const src_ty = ptr_ty.childType();
3758 switch (ptr) {4148 switch (ptr_mcv) {
3759 .none => unreachable,4149 .none,
3760 .undef => unreachable,4150 .unreach,
3761 .unreach => unreachable,4151 .dead,
3762 .dead => unreachable,4152 .undef,
3763 .eflags => unreachable,4153 .eflags,
3764 .register_overflow => unreachable,4154 .register_overflow,
3765 .immediate, .stack_offset => {4155 .reserved_frame,
3766 const reg = try self.copyToTmpRegister(ptr_ty, ptr);4156 => unreachable, // not a valid pointer
3767 try self.store(.{ .register = reg }, value, ptr_ty, value_ty);4157 .immediate,
3768 },4158 .register,
3769 .ptr_stack_offset => |off| {4159 .register_offset,
3770 try self.genSetStack(value_ty, off, value, .{});4160 .lea_direct,
4161 .lea_got,
4162 .lea_tlv,
4163 .lea_frame,
4164 => try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv),
4165 .memory,
4166 .indirect,
4167 .load_direct,
4168 .load_got,
4169 .load_tlv,
4170 .load_frame,
4171 => {
4172 const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
4173 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4174 defer self.register_manager.unlockReg(addr_lock);
4175
4176 try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
3771 },4177 },
3772 .register => |reg| {4178 }
3773 const reg_lock = self.register_manager.lockReg(reg);4179}
3774 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
3775
3776 switch (value) {
3777 .none => unreachable,
3778 .dead => unreachable,
3779 .unreach => unreachable,
3780 .eflags => |cc| try self.asmSetccMemory(
3781 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3782 cc,
3783 ),
3784 .undef => if (self.wantSafety()) switch (abi_size) {
3785 1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty),
3786 2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty),
3787 4 => try self.store(ptr, .{ .immediate = 0xaaaaaaaa }, ptr_ty, value_ty),
3788 8 => try self.store(ptr, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, ptr_ty, value_ty),
3789 else => try self.genInlineMemset(
3790 ptr,
3791 .{ .immediate = 0xaa },
3792 .{ .immediate = abi_size },
3793 .{},
3794 ),
3795 },
3796 .immediate => |imm| switch (self.regBitSize(value_ty)) {
3797 8 => try self.asmMemoryImmediate(
3798 .mov,
3799 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3800 if (math.cast(i8, @bitCast(i64, imm))) |small|
3801 Immediate.s(small)
3802 else
3803 Immediate.u(@intCast(u8, imm)),
3804 ),
3805 16 => try self.asmMemoryImmediate(
3806 .mov,
3807 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3808 if (math.cast(i16, @bitCast(i64, imm))) |small|
3809 Immediate.s(small)
3810 else
3811 Immediate.u(@intCast(u16, imm)),
3812 ),
3813 32 => try self.asmMemoryImmediate(
3814 .mov,
3815 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3816 if (math.cast(i32, @bitCast(i64, imm))) |small|
3817 Immediate.s(small)
3818 else
3819 Immediate.u(@intCast(u32, imm)),
3820 ),
3821 64 => if (math.cast(i32, @bitCast(i64, imm))) |small|
3822 try self.asmMemoryImmediate(
3823 .mov,
3824 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3825 Immediate.s(small),
3826 )
3827 else
3828 try self.asmMemoryRegister(
3829 .mov,
3830 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3831 registerAlias(try self.copyToTmpRegister(value_ty, value), abi_size),
3832 ),
3833 else => unreachable,
3834 },
3835 .register => |src_reg| try self.genInlineMemcpyRegisterRegister(
3836 value_ty,
3837 reg,
3838 src_reg,
3839 0,
3840 ),
3841 .register_overflow => |ro| {
3842 const ro_reg_lock = self.register_manager.lockReg(ro.reg);
3843 defer if (ro_reg_lock) |lock| self.register_manager.unlockReg(lock);
3844
3845 const wrapped_ty = value_ty.structFieldType(0);
3846 try self.genInlineMemcpyRegisterRegister(wrapped_ty, reg, ro.reg, 0);
3847
3848 const overflow_bit_ty = value_ty.structFieldType(1);
3849 const overflow_bit_offset = value_ty.structFieldOffset(1, self.target.*);
3850 const tmp_reg = try self.register_manager.allocReg(null, gp);
3851 try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);
3852 try self.genInlineMemcpyRegisterRegister(
3853 overflow_bit_ty,
3854 reg,
3855 tmp_reg,
3856 -@intCast(i32, overflow_bit_offset),
3857 );
3858 },
3859 .memory, .load_tlv, .load_direct => if (abi_size <= 8) {
3860 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3861 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3862 defer self.register_manager.unlockReg(tmp_lock);
3863
3864 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3865 } else {
3866 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
3867 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3868 defer self.register_manager.unlockReg(addr_lock);
3869
3870 try self.genSetReg(Type.usize, addr_reg, switch (value) {
3871 .memory => |addr| .{ .immediate = addr },
3872 .load_direct => |sym_index| .{ .lea_direct = sym_index },
3873 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
3874 else => unreachable,
3875 });
3876 try self.genInlineMemcpy(
3877 ptr,
3878 .{ .register = addr_reg },
3879 .{ .immediate = abi_size },
3880 .{},
3881 );
3882 },
3883 .stack_offset => |off| if (abi_size <= 8) {
3884 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3885 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3886 defer self.register_manager.unlockReg(tmp_lock);
3887
3888 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3889 } else try self.genInlineMemcpy(
3890 ptr,
3891 .{ .ptr_stack_offset = off },
3892 .{ .immediate = abi_size },
3893 .{},
3894 ),
3895 .ptr_stack_offset, .load_got, .lea_direct, .lea_tlv => {
3896 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3897 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3898 defer self.register_manager.unlockReg(tmp_lock);
3899
3900 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3901 },
3902 }
3903 },
3904 .memory, .load_direct, .load_tlv => {
3905 const value_lock: ?RegisterLock = switch (value) {
3906 .register => |reg| self.register_manager.lockReg(reg),
3907 else => null,
3908 };
3909 defer if (value_lock) |lock| self.register_manager.unlockReg(lock);
3910
3911 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
3912 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3913 defer self.register_manager.unlockReg(addr_reg_lock);
3914
3915 switch (ptr) {
3916 .memory => |addr| try self.genSetReg(ptr_ty, addr_reg, .{ .immediate = addr }),
3917 .load_direct => |sym_index| try self.genSetReg(ptr_ty, addr_reg, .{ .lea_direct = sym_index }),
3918 .load_tlv => |sym_index| try self.genSetReg(ptr_ty, addr_reg, .{ .lea_tlv = sym_index }),
3919 else => unreachable,
3920 }
3921 if (ptr != .load_tlv) {
3922 // Load the pointer, which is stored in memory
3923 try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{ .base = addr_reg }));
3924 }
3925
3926 const new_ptr = MCValue{ .register = addr_reg };
3927 try self.store(new_ptr, value, ptr_ty, value_ty);
3928 },
3929 .load_got, .lea_direct, .lea_tlv => {
3930 const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
3931 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3932 defer self.register_manager.unlockReg(addr_reg_lock);
3933
3934 const new_ptr = MCValue{ .register = addr_reg };
3935 try self.store(new_ptr, value, ptr_ty, value_ty);
3936 },
3937 }
3938}
39394180
3940fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {4181fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
3941 if (safety) {4182 if (safety) {
...@@ -3944,15 +4185,13 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -3944,15 +4185,13 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
3944 // TODO if the value is undef, don't lower this instruction4185 // TODO if the value is undef, don't lower this instruction
3945 }4186 }
3946 const bin_op = self.air.instructions.items(.data)[inst].bin_op;4187 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
3947 const ptr = try self.resolveInst(bin_op.lhs);4188 const ptr_mcv = try self.resolveInst(bin_op.lhs);
3948 const ptr_ty = self.air.typeOf(bin_op.lhs);4189 const ptr_ty = self.air.typeOf(bin_op.lhs);
3949 const value = try self.resolveInst(bin_op.rhs);4190 const src_mcv = try self.resolveInst(bin_op.rhs);
3950 const value_ty = self.air.typeOf(bin_op.rhs);
3951 log.debug("airStore(%{d}): {} <- {}", .{ inst, ptr, value });
3952 if (ptr_ty.ptrInfo().data.host_size > 0) {4191 if (ptr_ty.ptrInfo().data.host_size > 0) {
3953 try self.packedStore(ptr, value, ptr_ty, value_ty);4192 try self.packedStore(ptr_ty, ptr_mcv, src_mcv);
3954 } else {4193 } else {
3955 try self.store(ptr, value, ptr_ty, value_ty);4194 try self.store(ptr_ty, ptr_mcv, src_mcv);
3956 }4195 }
3957 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });4196 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
3958}4197}
...@@ -3971,67 +4210,65 @@ fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {...@@ -3971,67 +4210,65 @@ fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
3971}4210}
39724211
3973fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {4212fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
4213 const ptr_field_ty = self.air.typeOfIndex(inst);
3974 const mcv = try self.resolveInst(operand);4214 const mcv = try self.resolveInst(operand);
3975 const ptr_ty = self.air.typeOf(operand);4215 const ptr_container_ty = self.air.typeOf(operand);
3976 const container_ty = ptr_ty.childType();4216 const container_ty = ptr_container_ty.childType();
3977 const field_offset = switch (container_ty.containerLayout()) {4217 const field_offset = switch (container_ty.containerLayout()) {
3978 .Auto, .Extern => @intCast(u32, container_ty.structFieldOffset(index, self.target.*)),4218 .Auto, .Extern => @intCast(u32, container_ty.structFieldOffset(index, self.target.*)),
3979 .Packed => if (container_ty.zigTypeTag() == .Struct and ptr_ty.ptrInfo().data.host_size == 0)4219 .Packed => if (container_ty.zigTypeTag() == .Struct and
4220 ptr_field_ty.ptrInfo().data.host_size == 0)
3980 container_ty.packedStructFieldByteOffset(index, self.target.*)4221 container_ty.packedStructFieldByteOffset(index, self.target.*)
3981 else4222 else
3982 0,4223 0,
3983 };4224 };
39844225
3985 const dst_mcv: MCValue = result: {4226 const result: MCValue = result: {
3986 switch (mcv) {4227 switch (mcv) {
3987 .stack_offset, .lea_tlv, .load_tlv => {4228 .load_frame, .lea_tlv, .load_tlv => {
3988 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{4229 const offset_reg = try self.copyToTmpRegister(Type.usize, .{
3989 .immediate = field_offset,4230 .immediate = field_offset,
3990 });4231 });
3991 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);4232 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
3992 defer self.register_manager.unlockReg(offset_reg_lock);4233 defer self.register_manager.unlockReg(offset_reg_lock);
39934234
3994 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, switch (mcv) {4235 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, Type.usize, switch (mcv) {
3995 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },4236 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
3996 else => mcv,4237 else => mcv,
3997 });4238 });
3998 try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });4239 try self.genBinOpMir(.add, Type.usize, dst_mcv, .{ .register = offset_reg });
3999 break :result dst_mcv;4240 break :result dst_mcv;
4000 },4241 },
4001 .ptr_stack_offset => |off| {4242 .indirect => |reg_off| break :result .{ .indirect = .{
4002 const ptr_stack_offset = off - @intCast(i32, field_offset);4243 .reg = reg_off.reg,
4003 break :result MCValue{ .ptr_stack_offset = ptr_stack_offset };4244 .off = reg_off.off + @intCast(i32, field_offset),
4004 },4245 } },
4005 .register => |reg| {4246 .lea_frame => |frame_addr| break :result .{ .lea_frame = .{
4006 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);4247 .index = frame_addr.index,
4007 defer self.register_manager.unlockReg(reg_lock);4248 .off = frame_addr.off + @intCast(i32, field_offset),
40084249 } },
4009 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{4250 .register, .register_offset => {
4010 .immediate = field_offset,4251 const src_reg = mcv.getReg().?;
4011 });4252 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
4012 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);4253 defer self.register_manager.unlockReg(src_lock);
4013 defer self.register_manager.unlockReg(offset_reg_lock);
4014
4015 const can_reuse_operand = self.reuseOperand(inst, operand, 0, mcv);
4016 const result_reg: Register = blk: {
4017 if (can_reuse_operand) {
4018 break :blk reg;
4019 } else {
4020 const result_reg = try self.register_manager.allocReg(inst, gp);
4021 try self.genSetReg(ptr_ty, result_reg, mcv);
4022 break :blk result_reg;
4023 }
4024 };
4025 const result_reg_lock = self.register_manager.lockReg(result_reg);
4026 defer if (result_reg_lock) |lock| self.register_manager.unlockReg(lock);
40274254
4028 try self.genBinOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg });4255 const dst_mcv: MCValue = if (self.reuseOperand(inst, operand, 0, mcv))
4029 break :result MCValue{ .register = result_reg };4256 mcv
4257 else
4258 .{ .register = try self.copyToTmpRegister(ptr_field_ty, mcv) };
4259 break :result .{ .register_offset = .{
4260 .reg = dst_mcv.getReg().?,
4261 .off = switch (dst_mcv) {
4262 .register => 0,
4263 .register_offset => |reg_off| reg_off.off,
4264 else => unreachable,
4265 } + @intCast(i32, field_offset),
4266 } };
4030 },4267 },
4031 else => return self.fail("TODO implement fieldPtr for {}", .{mcv}),4268 else => return self.fail("TODO implement fieldPtr for {}", .{mcv}),
4032 }4269 }
4033 };4270 };
4034 return dst_mcv;4271 return result;
4035}4272}
40364273
4037fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {4274fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
...@@ -4055,7 +4292,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4055,7 +4292,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4055 };4292 };
40564293
4057 switch (src_mcv) {4294 switch (src_mcv) {
4058 .stack_offset => |src_off| {4295 .load_frame => |frame_addr| {
4059 const field_abi_size = @intCast(u32, field_ty.abiSize(self.target.*));4296 const field_abi_size = @intCast(u32, field_ty.abiSize(self.target.*));
4060 const limb_abi_size = @min(field_abi_size, 8);4297 const limb_abi_size = @min(field_abi_size, 8);
4061 const limb_abi_bits = limb_abi_size * 8;4298 const limb_abi_bits = limb_abi_size * 8;
...@@ -4063,11 +4300,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4063,11 +4300,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4063 const field_bit_off = field_off % limb_abi_bits;4300 const field_bit_off = field_off % limb_abi_bits;
40644301
4065 if (field_bit_off == 0) {4302 if (field_bit_off == 0) {
4066 const off_mcv = MCValue{ .stack_offset = src_off - field_byte_off };4303 const off_mcv = MCValue{ .load_frame = .{
4304 .index = frame_addr.index,
4305 .off = frame_addr.off + field_byte_off,
4306 } };
4067 if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result off_mcv;4307 if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result off_mcv;
40684308
4069 const dst_mcv = try self.allocRegOrMem(inst, true);4309 const dst_mcv = try self.allocRegOrMem(inst, true);
4070 try self.setRegOrMem(field_ty, dst_mcv, off_mcv);4310 try self.genCopy(field_ty, dst_mcv, off_mcv);
4071 break :result dst_mcv;4311 break :result dst_mcv;
4072 }4312 }
40734313
...@@ -4081,10 +4321,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4081,10 +4321,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4081 if (field_bit_off < field_extra_bits) field_abi_size else field_abi_size * 2;4321 if (field_bit_off < field_extra_bits) field_abi_size else field_abi_size * 2;
4082 if (load_abi_size <= 8) {4322 if (load_abi_size <= 8) {
4083 const load_reg = registerAlias(dst_reg, load_abi_size);4323 const load_reg = registerAlias(dst_reg, load_abi_size);
4084 try self.asmRegisterMemory(.mov, load_reg, Memory.sib(4324 try self.asmRegisterMemory(
4085 Memory.PtrSize.fromSize(load_abi_size),4325 .mov,
4086 .{ .base = .rbp, .disp = field_byte_off - src_off },4326 load_reg,
4087 ));4327 Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
4328 .base = .{ .frame = frame_addr.index },
4329 .disp = frame_addr.off + field_byte_off,
4330 }),
4331 );
4088 try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(field_bit_off));4332 try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(field_bit_off));
4089 } else {4333 } else {
4090 const tmp_reg = registerAlias(4334 const tmp_reg = registerAlias(
...@@ -4095,14 +4339,22 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4095,14 +4339,22 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4095 defer self.register_manager.unlockReg(tmp_lock);4339 defer self.register_manager.unlockReg(tmp_lock);
40964340
4097 const dst_alias = registerAlias(dst_reg, field_abi_size);4341 const dst_alias = registerAlias(dst_reg, field_abi_size);
4098 try self.asmRegisterMemory(.mov, dst_alias, Memory.sib(4342 try self.asmRegisterMemory(
4099 Memory.PtrSize.fromSize(field_abi_size),4343 .mov,
4100 .{ .base = .rbp, .disp = field_byte_off - src_off },4344 dst_alias,
4101 ));4345 Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
4102 try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(4346 .base = .{ .frame = frame_addr.index },
4103 Memory.PtrSize.fromSize(field_abi_size),4347 .disp = frame_addr.off + field_byte_off,
4104 .{ .base = .rbp, .disp = field_byte_off + 1 - src_off },4348 }),
4105 ));4349 );
4350 try self.asmRegisterMemory(
4351 .mov,
4352 tmp_reg,
4353 Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
4354 .base = .{ .frame = frame_addr.index },
4355 .disp = frame_addr.off + field_byte_off + @intCast(i32, limb_abi_size),
4356 }),
4357 );
4106 try self.asmRegisterRegisterImmediate(4358 try self.asmRegisterRegisterImmediate(
4107 .shrd,4359 .shrd,
4108 dst_alias,4360 dst_alias,
...@@ -4249,41 +4501,51 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:...@@ -4249,41 +4501,51 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
42494501
4250fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {4502fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {
4251 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));4503 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
4504 if (abi_size > 8) return self.fail("TODO implement {} for {}", .{
4505 mir_tag,
4506 dst_ty.fmt(self.bin_file.options.module.?),
4507 });
4252 switch (dst_mcv) {4508 switch (dst_mcv) {
4253 .none => unreachable,4509 .none,
4254 .undef => unreachable,4510 .unreach,
4255 .dead, .unreach, .immediate => unreachable,4511 .dead,
4256 .eflags => unreachable,4512 .undef,
4257 .register_overflow => unreachable,4513 .immediate,
4514 .register_offset,
4515 .eflags,
4516 .register_overflow,
4517 .lea_direct,
4518 .lea_got,
4519 .lea_tlv,
4520 .lea_frame,
4521 .reserved_frame,
4522 => unreachable, // unmodifiable destination
4258 .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),4523 .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),
4259 .stack_offset => |off| {4524 .memory, .load_got, .load_direct, .load_tlv => {
4260 if (abi_size > 8) {4525 const addr_reg = try self.register_manager.allocReg(null, gp);
4261 return self.fail("TODO implement {} for stack dst with large ABI", .{mir_tag});
4262 }
4263
4264 try self.asmMemory(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
4265 .base = .rbp,
4266 .disp = -off,
4267 }));
4268 },
4269 .ptr_stack_offset => unreachable,
4270 .load_got, .lea_direct, .lea_tlv => unreachable,
4271 .memory, .load_direct, .load_tlv => {
4272 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
4273 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);4526 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4274 defer self.register_manager.unlockReg(addr_reg_lock);4527 defer self.register_manager.unlockReg(addr_reg_lock);
42754528
4276 try self.genSetReg(Type.usize, addr_reg, switch (dst_mcv) {4529 try self.genSetReg(addr_reg, Type.usize, dst_mcv.address());
4277 .memory => |addr| .{ .immediate = addr },
4278 .load_direct => |sym_index| .{ .lea_direct = sym_index },
4279 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
4280 else => unreachable,
4281 });
4282 try self.asmMemory(4530 try self.asmMemory(
4283 mir_tag,4531 mir_tag,
4284 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),4532 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
4285 );4533 );
4286 },4534 },
4535 .indirect, .load_frame => try self.asmMemory(
4536 mir_tag,
4537 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (dst_mcv) {
4538 .indirect => |reg_off| .{
4539 .base = .{ .reg = reg_off.reg },
4540 .disp = reg_off.off,
4541 },
4542 .load_frame => |frame_addr| .{
4543 .base = .{ .frame = frame_addr.index },
4544 .disp = frame_addr.off,
4545 },
4546 else => unreachable,
4547 }),
4548 ),
4287 }4549 }
4288}4550}
42894551
...@@ -4305,7 +4567,7 @@ fn genShiftBinOpMir(...@@ -4305,7 +4567,7 @@ fn genShiftBinOpMir(
4305 else => {},4567 else => {},
4306 }4568 }
4307 self.register_manager.getRegAssumeFree(.rcx, null);4569 self.register_manager.getRegAssumeFree(.rcx, null);
4308 try self.genSetReg(Type.u8, .rcx, shift_mcv);4570 try self.genSetReg(.cl, Type.u8, shift_mcv);
4309 break :rhs .{ .register = .rcx };4571 break :rhs .{ .register = .rcx };
4310 };4572 };
43114573
...@@ -4328,21 +4590,42 @@ fn genShiftBinOpMir(...@@ -4328,21 +4590,42 @@ fn genShiftBinOpMir(
4328 @tagName(rhs_mcv),4590 @tagName(rhs_mcv),
4329 }),4591 }),
4330 },4592 },
4331 .stack_offset => |lhs_off| switch (rhs_mcv) {4593 .memory, .indirect, .load_frame => {
4332 .immediate => |rhs_imm| try self.asmMemoryImmediate(4594 const lhs_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (lhs_mcv) {
4333 tag,4595 .memory => |addr| .{
4334 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),4596 .base = .{ .reg = .ds },
4335 Immediate.u(rhs_imm),4597 .disp = math.cast(i32, @bitCast(i64, addr)) orelse
4336 ),4598 return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
4337 .register => |rhs_reg| try self.asmMemoryRegister(4599 @tagName(lhs_mcv),
4338 tag,4600 @tagName(rhs_mcv),
4339 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),4601 }),
4340 registerAlias(rhs_reg, 1),4602 },
4341 ),4603 .indirect => |reg_off| .{
4342 else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{4604 .base = .{ .reg = reg_off.reg },
4343 @tagName(lhs_mcv),4605 .disp = reg_off.off,
4344 @tagName(rhs_mcv),4606 },
4345 }),4607 .load_frame => |frame_addr| .{
4608 .base = .{ .frame = frame_addr.index },
4609 .disp = frame_addr.off,
4610 },
4611 else => unreachable,
4612 });
4613 switch (rhs_mcv) {
4614 .immediate => |rhs_imm| try self.asmMemoryImmediate(
4615 tag,
4616 lhs_mem,
4617 Immediate.u(rhs_imm),
4618 ),
4619 .register => |rhs_reg| try self.asmMemoryRegister(
4620 tag,
4621 lhs_mem,
4622 registerAlias(rhs_reg, 1),
4623 ),
4624 else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
4625 @tagName(lhs_mcv),
4626 @tagName(rhs_mcv),
4627 }),
4628 }
4346 },4629 },
4347 else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{4630 else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
4348 @tagName(lhs_mcv),4631 @tagName(lhs_mcv),
...@@ -4360,22 +4643,31 @@ fn genShiftBinOpMir(...@@ -4360,22 +4643,31 @@ fn genShiftBinOpMir(
4360 else => unreachable,4643 else => unreachable,
4361 };4644 };
4362 switch (lhs_mcv) {4645 switch (lhs_mcv) {
4363 .stack_offset => |dst_off| switch (rhs_mcv) {4646 .load_frame => |dst_frame_addr| switch (rhs_mcv) {
4364 .immediate => |rhs_imm| if (rhs_imm == 0) {} else if (rhs_imm < 64) {4647 .immediate => |rhs_imm| if (rhs_imm == 0) {} else if (rhs_imm < 64) {
4365 try self.asmRegisterMemory(4648 try self.asmRegisterMemory(
4366 .mov,4649 .mov,
4367 tmp_reg,4650 tmp_reg,
4368 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4651 Memory.sib(.qword, .{
4652 .base = .{ .frame = dst_frame_addr.index },
4653 .disp = dst_frame_addr.off + info.offsets[0],
4654 }),
4369 );4655 );
4370 try self.asmMemoryRegisterImmediate(4656 try self.asmMemoryRegisterImmediate(
4371 info.double_tag,4657 info.double_tag,
4372 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),4658 Memory.sib(.qword, .{
4659 .base = .{ .frame = dst_frame_addr.index },
4660 .disp = dst_frame_addr.off + info.offsets[1],
4661 }),
4373 tmp_reg,4662 tmp_reg,
4374 Immediate.u(rhs_imm),4663 Immediate.u(rhs_imm),
4375 );4664 );
4376 try self.asmMemoryImmediate(4665 try self.asmMemoryImmediate(
4377 tag,4666 tag,
4378 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4667 Memory.sib(.qword, .{
4668 .base = .{ .frame = dst_frame_addr.index },
4669 .disp = dst_frame_addr.off + info.offsets[0],
4670 }),
4379 Immediate.u(rhs_imm),4671 Immediate.u(rhs_imm),
4380 );4672 );
4381 } else {4673 } else {
...@@ -4383,14 +4675,20 @@ fn genShiftBinOpMir(...@@ -4383,14 +4675,20 @@ fn genShiftBinOpMir(
4383 try self.asmRegisterMemory(4675 try self.asmRegisterMemory(
4384 .mov,4676 .mov,
4385 tmp_reg,4677 tmp_reg,
4386 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4678 Memory.sib(.qword, .{
4679 .base = .{ .frame = dst_frame_addr.index },
4680 .disp = dst_frame_addr.off + info.offsets[0],
4681 }),
4387 );4682 );
4388 if (rhs_imm > 64) {4683 if (rhs_imm > 64) {
4389 try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));4684 try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));
4390 }4685 }
4391 try self.asmMemoryRegister(4686 try self.asmMemoryRegister(
4392 .mov,4687 .mov,
4393 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),4688 Memory.sib(.qword, .{
4689 .base = .{ .frame = dst_frame_addr.index },
4690 .disp = dst_frame_addr.off + info.offsets[1],
4691 }),
4394 tmp_reg,4692 tmp_reg,
4395 );4693 );
4396 switch (tag) {4694 switch (tag) {
...@@ -4398,13 +4696,19 @@ fn genShiftBinOpMir(...@@ -4398,13 +4696,19 @@ fn genShiftBinOpMir(
4398 try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());4696 try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());
4399 try self.asmMemoryRegister(4697 try self.asmMemoryRegister(
4400 .mov,4698 .mov,
4401 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4699 Memory.sib(.qword, .{
4700 .base = .{ .frame = dst_frame_addr.index },
4701 .disp = dst_frame_addr.off + info.offsets[0],
4702 }),
4402 tmp_reg,4703 tmp_reg,
4403 );4704 );
4404 },4705 },
4405 .sar => try self.asmMemoryImmediate(4706 .sar => try self.asmMemoryImmediate(
4406 tag,4707 tag,
4407 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4708 Memory.sib(.qword, .{
4709 .base = .{ .frame = dst_frame_addr.index },
4710 .disp = dst_frame_addr.off + info.offsets[0],
4711 }),
4408 Immediate.u(63),4712 Immediate.u(63),
4409 ),4713 ),
4410 else => unreachable,4714 else => unreachable,
...@@ -4419,16 +4723,22 @@ fn genShiftBinOpMir(...@@ -4419,16 +4723,22 @@ fn genShiftBinOpMir(
4419 const second_lock = self.register_manager.lockRegAssumeUnused(second_reg);4723 const second_lock = self.register_manager.lockRegAssumeUnused(second_reg);
4420 defer self.register_manager.unlockReg(second_lock);4724 defer self.register_manager.unlockReg(second_lock);
44214725
4422 try self.genSetReg(Type.u8, .cl, rhs_mcv);4726 try self.genSetReg(.cl, Type.u8, rhs_mcv);
4423 try self.asmRegisterMemory(4727 try self.asmRegisterMemory(
4424 .mov,4728 .mov,
4425 first_reg,4729 first_reg,
4426 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4730 Memory.sib(.qword, .{
4731 .base = .{ .frame = dst_frame_addr.index },
4732 .disp = dst_frame_addr.off + info.offsets[0],
4733 }),
4427 );4734 );
4428 try self.asmRegisterMemory(4735 try self.asmRegisterMemory(
4429 .mov,4736 .mov,
4430 second_reg,4737 second_reg,
4431 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),4738 Memory.sib(.qword, .{
4739 .base = .{ .frame = dst_frame_addr.index },
4740 .disp = dst_frame_addr.off + info.offsets[1],
4741 }),
4432 );4742 );
4433 switch (tag) {4743 switch (tag) {
4434 .shl, .sal, .shr => try self.asmRegisterRegister(4744 .shl, .sal, .shr => try self.asmRegisterRegister(
...@@ -4449,12 +4759,18 @@ fn genShiftBinOpMir(...@@ -4449,12 +4759,18 @@ fn genShiftBinOpMir(
4449 try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);4759 try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
4450 try self.asmMemoryRegister(4760 try self.asmMemoryRegister(
4451 .mov,4761 .mov,
4452 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),4762 Memory.sib(.qword, .{
4763 .base = .{ .frame = dst_frame_addr.index },
4764 .disp = dst_frame_addr.off + info.offsets[1],
4765 }),
4453 second_reg,4766 second_reg,
4454 );4767 );
4455 try self.asmMemoryRegister(4768 try self.asmMemoryRegister(
4456 .mov,4769 .mov,
4457 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4770 Memory.sib(.qword, .{
4771 .base = .{ .frame = dst_frame_addr.index },
4772 .disp = dst_frame_addr.off + info.offsets[0],
4773 }),
4458 first_reg,4774 first_reg,
4459 );4775 );
4460 },4776 },
...@@ -4515,7 +4831,7 @@ fn genShiftBinOp(...@@ -4515,7 +4831,7 @@ fn genShiftBinOp(
4515 if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;4831 if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;
4516 }4832 }
4517 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);4833 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
4518 try self.setRegOrMem(lhs_ty, dst_mcv, lhs_mcv);4834 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
4519 break :dst dst_mcv;4835 break :dst dst_mcv;
4520 };4836 };
45214837
...@@ -4610,14 +4926,22 @@ fn genMulDivBinOp(...@@ -4610,14 +4926,22 @@ fn genMulDivBinOp(
4610 }, dst_abi_size) };4926 }, dst_abi_size) };
46114927
4612 const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);4928 const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);
4613 try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{4929 try self.asmMemoryRegister(
4614 .base = .rbp,4930 .mov,
4615 .disp = 0 - dst_mcv.stack_offset,4931 Memory.sib(.qword, .{
4616 }), .rax);4932 .base = .{ .frame = dst_mcv.load_frame.index },
4617 try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{4933 .disp = dst_mcv.load_frame.off,
4618 .base = .rbp,4934 }),
4619 .disp = 8 - dst_mcv.stack_offset,4935 .rax,
4620 }), .rdx);4936 );
4937 try self.asmMemoryRegister(
4938 .mov,
4939 Memory.sib(.qword, .{
4940 .base = .{ .frame = dst_mcv.load_frame.index },
4941 .disp = dst_mcv.load_frame.off + 8,
4942 }),
4943 .rdx,
4944 );
4621 return dst_mcv;4945 return dst_mcv;
4622 },4946 },
46234947
...@@ -4627,8 +4951,24 @@ fn genMulDivBinOp(...@@ -4627,8 +4951,24 @@ fn genMulDivBinOp(
46274951
4628 switch (signedness) {4952 switch (signedness) {
4629 .signed => {4953 .signed => {
4954 const lhs_lock = switch (lhs) {
4955 .register => |reg| self.register_manager.lockReg(reg),
4956 else => null,
4957 };
4958 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
4959 const rhs_lock = switch (rhs) {
4960 .register => |reg| self.register_manager.lockReg(reg),
4961 else => null,
4962 };
4963 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
4964
4965 // hack around hazard between rhs and div_floor by copying rhs to another register
4966 const rhs_copy = try self.copyToTmpRegister(ty, rhs);
4967 const rhs_copy_lock = self.register_manager.lockRegAssumeUnused(rhs_copy);
4968 defer self.register_manager.unlockReg(rhs_copy_lock);
4969
4630 const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs);4970 const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs);
4631 try self.genIntMulComplexOpMir(ty, div_floor, rhs);4971 try self.genIntMulComplexOpMir(ty, div_floor, .{ .register = rhs_copy });
4632 const div_floor_lock = self.register_manager.lockReg(div_floor.register);4972 const div_floor_lock = self.register_manager.lockReg(div_floor.register);
4633 defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock);4973 defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock);
46344974
...@@ -4775,18 +5115,14 @@ fn genBinOp(...@@ -4775,18 +5115,14 @@ fn genBinOp(
4775 }5115 }
4776 }5116 }
4777 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);5117 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
4778 try self.setRegOrMem(lhs_ty, dst_mcv, switch (lhs) {5118 try self.genCopy(lhs_ty, dst_mcv, lhs);
4779 .load_direct => |sym_index| .{ .lea_direct = sym_index },
4780 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
4781 else => lhs,
4782 });
4783 break :dst dst_mcv;5119 break :dst dst_mcv;
4784 };5120 };
4785 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {5121 const dst_lock: ?RegisterLock = switch (dst_mcv) {
4786 .register => |reg| self.register_manager.lockReg(reg),5122 .register => |reg| self.register_manager.lockReg(reg),
4787 else => null,5123 else => null,
4788 };5124 };
4789 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);5125 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
47905126
4791 const src_mcv = if (flipped) lhs else rhs;5127 const src_mcv = if (flipped) lhs else rhs;
4792 switch (tag) {5128 switch (tag) {
...@@ -4878,10 +5214,21 @@ fn genBinOp(...@@ -4878,10 +5214,21 @@ fn genBinOp(
4878 .max,5214 .max,
4879 => switch (lhs_ty.zigTypeTag()) {5215 => switch (lhs_ty.zigTypeTag()) {
4880 .Int => {5216 .Int => {
4881 const mat_src_mcv = switch (src_mcv) {5217 const mat_src_mcv: MCValue = if (switch (src_mcv) {
4882 .immediate => MCValue{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) },5218 .immediate,
4883 else => src_mcv,5219 .eflags,
4884 };5220 .register_offset,
5221 .load_direct,
5222 .lea_direct,
5223 .load_got,
5224 .lea_got,
5225 .load_tlv,
5226 .lea_tlv,
5227 .lea_frame,
5228 => true,
5229 .memory => |addr| math.cast(i32, @bitCast(i64, addr)) == null,
5230 else => false,
5231 }) .{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) } else src_mcv;
4885 const mat_mcv_lock = switch (mat_src_mcv) {5232 const mat_mcv_lock = switch (mat_src_mcv) {
4886 .register => |reg| self.register_manager.lockReg(reg),5233 .register => |reg| self.register_manager.lockReg(reg),
4887 else => null,5234 else => null,
...@@ -4913,57 +5260,56 @@ fn genBinOp(...@@ -4913,57 +5260,56 @@ fn genBinOp(
4913 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);5260 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
4914 switch (mat_src_mcv) {5261 switch (mat_src_mcv) {
4915 .none,5262 .none,
4916 .undef,
4917 .dead,
4918 .unreach,5263 .unreach,
5264 .dead,
5265 .undef,
4919 .immediate,5266 .immediate,
4920 .eflags,5267 .eflags,
5268 .register_offset,
4921 .register_overflow,5269 .register_overflow,
4922 .ptr_stack_offset,5270 .load_direct,
4923 .load_got,
4924 .lea_direct,5271 .lea_direct,
5272 .load_got,
5273 .lea_got,
5274 .load_tlv,
4925 .lea_tlv,5275 .lea_tlv,
5276 .lea_frame,
5277 .reserved_frame,
4926 => unreachable,5278 => unreachable,
4927 .register => |src_reg| try self.asmCmovccRegisterRegister(5279 .register => |src_reg| try self.asmCmovccRegisterRegister(
4928 registerAlias(tmp_reg, cmov_abi_size),5280 registerAlias(tmp_reg, cmov_abi_size),
4929 registerAlias(src_reg, cmov_abi_size),5281 registerAlias(src_reg, cmov_abi_size),
4930 cc,5282 cc,
4931 ),5283 ),
4932 .stack_offset => |off| try self.asmCmovccRegisterMemory(5284 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
4933 registerAlias(tmp_reg, cmov_abi_size),5285 registerAlias(tmp_reg, cmov_abi_size),
4934 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{5286 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), switch (mat_src_mcv) {
4935 .base = .rbp,5287 .memory => |addr| .{
4936 .disp = -off,5288 .base = .{ .reg = .ds },
5289 .disp = @intCast(i32, @bitCast(i64, addr)),
5290 },
5291 .indirect => |reg_off| .{
5292 .base = .{ .reg = reg_off.reg },
5293 .disp = reg_off.off,
5294 },
5295 .load_frame => |frame_addr| .{
5296 .base = .{ .frame = frame_addr.index },
5297 .disp = frame_addr.off,
5298 },
5299 else => unreachable,
4937 }),5300 }),
4938 cc,5301 cc,
4939 ),5302 ),
4940 .memory, .load_direct, .load_tlv => {
4941 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
4942 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4943 defer self.register_manager.unlockReg(addr_reg_lock);
4944
4945 try self.genSetReg(Type.usize, addr_reg, switch (mat_src_mcv) {
4946 .memory => |addr| .{ .immediate = addr },
4947 .load_direct => |sym_index| .{ .lea_direct = sym_index },
4948 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
4949 else => unreachable,
4950 });
4951 try self.asmCmovccRegisterMemory(
4952 registerAlias(tmp_reg, cmov_abi_size),
4953 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{ .base = addr_reg }),
4954 cc,
4955 );
4956 },
4957 }5303 }
4958 try self.setRegOrMem(lhs_ty, dst_mcv, .{ .register = tmp_reg });5304 try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
4959 },5305 },
4960 .Float => try self.genBinOpMir(switch (lhs_ty.tag()) {5306 .Float => try self.genBinOpMir(switch (lhs_ty.floatBits(self.target.*)) {
4961 .f32 => switch (tag) {5307 32 => switch (tag) {
4962 .min => .minss,5308 .min => .minss,
4963 .max => .maxss,5309 .max => .maxss,
4964 else => unreachable,5310 else => unreachable,
4965 },5311 },
4966 .f64 => switch (tag) {5312 64 => switch (tag) {
4967 .min => .minsd,5313 .min => .minsd,
4968 .max => .maxsd,5314 .max => .maxsd,
4969 else => unreachable,5315 else => unreachable,
...@@ -4981,25 +5327,38 @@ fn genBinOp(...@@ -4981,25 +5327,38 @@ fn genBinOp(
4981fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {5327fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
4982 const abi_size = @intCast(u32, ty.abiSize(self.target.*));5328 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4983 switch (dst_mcv) {5329 switch (dst_mcv) {
4984 .none => unreachable,5330 .none,
4985 .undef => unreachable,5331 .unreach,
4986 .dead, .unreach, .immediate => unreachable,5332 .dead,
4987 .eflags => unreachable,5333 .undef,
4988 .register_overflow => unreachable,5334 .immediate,
5335 .register_offset,
5336 .eflags,
5337 .register_overflow,
5338 .lea_direct,
5339 .lea_got,
5340 .lea_tlv,
5341 .lea_frame,
5342 .reserved_frame,
5343 => unreachable, // unmodifiable destination
4989 .register => |dst_reg| {5344 .register => |dst_reg| {
4990 const dst_alias = registerAlias(dst_reg, abi_size);5345 const dst_alias = registerAlias(dst_reg, abi_size);
4991 switch (src_mcv) {5346 switch (src_mcv) {
4992 .none => unreachable,5347 .none,
4993 .undef => unreachable,5348 .unreach,
4994 .dead, .unreach => unreachable,5349 .dead,
4995 .register_overflow => unreachable,5350 .undef,
5351 .register_overflow,
5352 .reserved_frame,
5353 => unreachable,
4996 .register => |src_reg| switch (ty.zigTypeTag()) {5354 .register => |src_reg| switch (ty.zigTypeTag()) {
4997 .Float => {5355 .Float => {
4998 if (intrinsicsAllowed(self.target.*, ty)) {5356 if (!Target.x86.featureSetHas(self.target.cpu.features, .sse))
4999 return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());5357 return self.fail("TODO genBinOpMir for {s} {} without sse", .{
5000 }5358 @tagName(mir_tag),
50015359 ty.fmt(self.bin_file.options.module.?),
5002 return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});5360 });
5361 return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
5003 },5362 },
5004 else => try self.asmRegisterRegister(5363 else => try self.asmRegisterRegister(
5005 mir_tag,5364 mir_tag,
...@@ -5041,14 +5400,16 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5041,14 +5400,16 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5041 )),5400 )),
5042 else => unreachable,5401 else => unreachable,
5043 },5402 },
5044 .ptr_stack_offset,5403 .register_offset,
5404 .eflags,
5045 .memory,5405 .memory,
5046 .load_got,
5047 .lea_direct,
5048 .load_direct,5406 .load_direct,
5049 .lea_tlv,5407 .lea_direct,
5408 .load_got,
5409 .lea_got,
5050 .load_tlv,5410 .load_tlv,
5051 .eflags,5411 .lea_tlv,
5412 .lea_frame,
5052 => {5413 => {
5053 assert(abi_size <= 8);5414 assert(abi_size <= 8);
5054 const dst_reg_lock = self.register_manager.lockReg(dst_reg);5415 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
...@@ -5057,75 +5418,84 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5057,75 +5418,84 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5057 const reg = try self.copyToTmpRegister(ty, src_mcv);5418 const reg = try self.copyToTmpRegister(ty, src_mcv);
5058 return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });5419 return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });
5059 },5420 },
5060 .stack_offset => |off| try self.asmRegisterMemory(5421 .indirect, .load_frame => try self.asmRegisterMemory(
5061 mir_tag,5422 mir_tag,
5062 registerAlias(dst_reg, abi_size),5423 registerAlias(dst_reg, abi_size),
5063 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),5424 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
5425 .indirect => |reg_off| .{
5426 .base = .{ .reg = reg_off.reg },
5427 .disp = reg_off.off,
5428 },
5429 .load_frame => |frame_addr| .{
5430 .base = .{ .frame = frame_addr.index },
5431 .disp = frame_addr.off,
5432 },
5433 else => unreachable,
5434 }),
5064 ),5435 ),
5065 }5436 }
5066 },5437 },
5067 .memory, .load_got, .load_direct, .load_tlv, .stack_offset => {5438 .memory, .indirect, .load_got, .load_direct, .load_tlv, .load_frame => {
5068 const dst: ?struct {5439 const OpInfo = ?struct { addr_reg: Register, addr_lock: RegisterLock };
5069 addr_reg: Register,5440 const limb_abi_size = @min(abi_size, 8);
5070 addr_lock: RegisterLock,5441
5071 } = switch (dst_mcv) {5442 const dst_info: OpInfo = switch (dst_mcv) {
5072 else => unreachable,5443 else => unreachable,
5073 .memory, .load_got, .load_direct, .load_tlv => dst: {5444 .memory, .load_got, .load_direct, .load_tlv => dst: {
5074 const dst_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();5445 const dst_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
5075 const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);5446 const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
5076 errdefer self.register_manager.unlockReg(dst_addr_lock);5447 errdefer self.register_manager.unlockReg(dst_addr_lock);
50775448
5078 try self.genSetReg(Type.usize, dst_addr_reg, switch (dst_mcv) {5449 try self.genSetReg(dst_addr_reg, Type.usize, dst_mcv.address());
5079 .memory => |addr| .{ .immediate = addr },
5080 .load_direct => |sym_index| .{ .lea_direct = sym_index },
5081 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
5082 else => dst_mcv,
5083 });
5084
5085 break :dst .{5450 break :dst .{
5086 .addr_reg = dst_addr_reg,5451 .addr_reg = dst_addr_reg,
5087 .addr_lock = dst_addr_lock,5452 .addr_lock = dst_addr_lock,
5088 };5453 };
5089 },5454 },
5090 .stack_offset => null,5455 .load_frame => null,
5091 };5456 };
5092 defer if (dst) |lock| self.register_manager.unlockReg(lock.addr_lock);5457 defer if (dst_info) |info| self.register_manager.unlockReg(info.addr_lock);
50935458
5094 const src: ?struct {5459 const src_info: OpInfo = switch (src_mcv) {
5095 limb_reg: Register,5460 .none,
5096 limb_lock: RegisterLock,5461 .unreach,
5097 addr_reg: Register,5462 .dead,
5098 addr_lock: RegisterLock,5463 .undef,
5099 } = switch (src_mcv) {5464 .register_overflow,
5100 else => null,5465 .reserved_frame,
5466 => unreachable,
5467 .immediate,
5468 .register,
5469 .register_offset,
5470 .eflags,
5471 .indirect,
5472 .lea_direct,
5473 .lea_got,
5474 .lea_tlv,
5475 .load_frame,
5476 .lea_frame,
5477 => null,
5101 .memory, .load_got, .load_direct, .load_tlv => src: {5478 .memory, .load_got, .load_direct, .load_tlv => src: {
5102 const src_limb_reg = try self.register_manager.allocReg(null, gp);5479 switch (src_mcv) {
5103 const src_limb_lock = self.register_manager.lockRegAssumeUnused(src_limb_reg);5480 .memory => |addr| if (math.cast(i32, @bitCast(i64, addr)) != null and
5104 errdefer self.register_manager.unlockReg(src_limb_lock);5481 math.cast(i32, @bitCast(i64, addr) + abi_size - limb_abi_size) != null)
5482 break :src null,
5483 .load_got, .load_direct, .load_tlv => {},
5484 else => unreachable,
5485 }
51055486
5106 const src_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();5487 const src_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
5107 const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);5488 const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
5108 errdefer self.register_manager.unlockReg(src_addr_lock);5489 errdefer self.register_manager.unlockReg(src_addr_lock);
51095490
5110 try self.genSetReg(Type.usize, src_addr_reg, switch (src_mcv) {5491 try self.genSetReg(src_addr_reg, Type.usize, src_mcv.address());
5111 .memory => |addr| .{ .immediate = addr },
5112 .load_direct => |sym_index| .{ .lea_direct = sym_index },
5113 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
5114 else => src_mcv,
5115 });
5116
5117 break :src .{5492 break :src .{
5118 .addr_reg = src_addr_reg,5493 .addr_reg = src_addr_reg,
5119 .addr_lock = src_addr_lock,5494 .addr_lock = src_addr_lock,
5120 .limb_reg = src_limb_reg,
5121 .limb_lock = src_limb_lock,
5122 };5495 };
5123 },5496 },
5124 };5497 };
5125 defer if (src) |locks| {5498 defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock);
5126 self.register_manager.unlockReg(locks.limb_lock);
5127 self.register_manager.unlockReg(locks.addr_lock);
5128 };
51295499
5130 const ty_signedness =5500 const ty_signedness =
5131 if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;5501 if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;
...@@ -5133,7 +5503,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5133,7 +5503,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5133 .signed => Type.usize,5503 .signed => Type.usize,
5134 .unsigned => Type.isize,5504 .unsigned => Type.isize,
5135 };5505 };
5136 const limb_abi_size = @min(abi_size, 8);
5137 var off: i32 = 0;5506 var off: i32 = 0;
5138 while (off < abi_size) : (off += 8) {5507 while (off < abi_size) : (off += 8) {
5139 const mir_limb_tag = switch (off) {5508 const mir_limb_tag = switch (off) {
...@@ -5150,35 +5519,45 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5150,35 +5519,45 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5150 const dst_limb_mem = Memory.sib(5519 const dst_limb_mem = Memory.sib(
5151 Memory.PtrSize.fromSize(limb_abi_size),5520 Memory.PtrSize.fromSize(limb_abi_size),
5152 switch (dst_mcv) {5521 switch (dst_mcv) {
5153 else => unreachable,
5154 .stack_offset => |dst_off| .{
5155 .base = .rbp,
5156 .disp = off - dst_off,
5157 },
5158 .memory,5522 .memory,
5159 .load_got,5523 .load_got,
5160 .load_direct,5524 .load_direct,
5161 .load_tlv,5525 .load_tlv,
5162 => .{ .base = dst.?.addr_reg, .disp = off },5526 => .{ .base = .{ .reg = dst_info.?.addr_reg }, .disp = off },
5527 .indirect => |reg_off| .{
5528 .base = .{ .reg = reg_off.reg },
5529 .disp = reg_off.off + off,
5530 },
5531 .load_frame => |frame_addr| .{
5532 .base = .{ .frame = frame_addr.index },
5533 .disp = frame_addr.off + off,
5534 },
5535 else => unreachable,
5163 },5536 },
5164 );5537 );
5165 switch (src_mcv) {5538 switch (src_mcv) {
5166 .none => unreachable,5539 .none,
5167 .undef => unreachable,5540 .unreach,
5168 .dead, .unreach => unreachable,5541 .dead,
5169 .register_overflow => unreachable,5542 .undef,
5170 .register => |src_reg| {5543 .register_overflow,
5171 assert(off == 0);5544 .reserved_frame,
5172 try self.asmMemoryRegister(5545 => unreachable,
5546 .register => |src_reg| switch (off) {
5547 0 => try self.asmMemoryRegister(
5173 mir_limb_tag,5548 mir_limb_tag,
5174 dst_limb_mem,5549 dst_limb_mem,
5175 registerAlias(src_reg, limb_abi_size),5550 registerAlias(src_reg, limb_abi_size),
5176 );5551 ),
5552 else => unreachable,
5177 },5553 },
5178 .immediate => |src_imm| {5554 .immediate => |src_imm| {
5179 const imm = if (off == 0) src_imm else switch (ty_signedness) {5555 const imm = switch (off) {
5180 .signed => @bitCast(u64, @bitCast(i64, src_imm) >> 63),5556 0 => src_imm,
5181 .unsigned => 0,5557 else => switch (ty_signedness) {
5558 .signed => @bitCast(u64, @bitCast(i64, src_imm) >> 63),
5559 .unsigned => 0,
5560 },
5182 };5561 };
5183 switch (self.regBitSize(limb_ty)) {5562 switch (self.regBitSize(limb_ty)) {
5184 8 => try self.asmMemoryImmediate(5563 8 => try self.asmMemoryImmediate(
...@@ -5223,47 +5602,43 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5223,47 +5602,43 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5223 else => unreachable,5602 else => unreachable,
5224 }5603 }
5225 },5604 },
5605 .register_offset,
5606 .eflags,
5226 .memory,5607 .memory,
5227 .load_got,5608 .indirect,
5228 .load_direct,5609 .load_direct,
5229 .load_tlv,
5230 => {
5231 try self.asmRegisterMemory(
5232 .mov,
5233 registerAlias(src.?.limb_reg, limb_abi_size),
5234 Memory.sib(
5235 Memory.PtrSize.fromSize(limb_abi_size),
5236 .{ .base = src.?.addr_reg, .disp = off },
5237 ),
5238 );
5239 try self.asmMemoryRegister(
5240 mir_limb_tag,
5241 dst_limb_mem,
5242 registerAlias(src.?.limb_reg, limb_abi_size),
5243 );
5244 },
5245 .stack_offset,
5246 .ptr_stack_offset,
5247 .eflags,
5248 .lea_direct,5610 .lea_direct,
5611 .load_got,
5612 .lea_got,
5613 .load_tlv,
5249 .lea_tlv,5614 .lea_tlv,
5615 .load_frame,
5616 .lea_frame,
5250 => {5617 => {
5251 const src_limb_reg = try self.copyToTmpRegister(limb_ty, switch (src_mcv) {5618 const src_limb_reg = try self.copyToTmpRegister(limb_ty, if (src_info) |info| .{
5252 .stack_offset => |src_off| .{ .stack_offset = src_off - off },5619 .indirect = .{ .reg = info.addr_reg, .off = off },
5253 .ptr_stack_offset,5620 } else switch (src_mcv) {
5254 .eflags,5621 .eflags,
5255 .load_got,5622 .register_offset,
5256 .lea_direct,5623 .lea_direct,
5624 .lea_got,
5257 .lea_tlv,5625 .lea_tlv,
5258 => off: {5626 .lea_frame,
5259 assert(off == 0);5627 => switch (off) {
5260 break :off src_mcv;5628 0 => src_mcv,
5629 else => .{ .immediate = 0 },
5261 },5630 },
5631 .memory => |addr| .{ .memory = @bitCast(u64, @bitCast(i64, addr) + off) },
5632 .indirect => |reg_off| .{ .indirect = .{
5633 .reg = reg_off.reg,
5634 .off = reg_off.off + off,
5635 } },
5636 .load_frame => |frame_addr| .{ .load_frame = .{
5637 .index = frame_addr.index,
5638 .off = frame_addr.off + off,
5639 } },
5262 else => unreachable,5640 else => unreachable,
5263 });5641 });
5264 const src_limb_lock = self.register_manager.lockReg(src_limb_reg);
5265 defer if (src_limb_lock) |lock| self.register_manager.unlockReg(lock);
5266
5267 try self.asmMemoryRegister(5642 try self.asmMemoryRegister(
5268 mir_limb_tag,5643 mir_limb_tag,
5269 dst_limb_mem,5644 dst_limb_mem,
...@@ -5273,9 +5648,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5273,9 +5648,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5273 }5648 }
5274 }5649 }
5275 },5650 },
5276 .ptr_stack_offset => unreachable,
5277 .lea_tlv => unreachable,
5278 .lea_direct => unreachable,
5279 }5651 }
5280}5652}
52815653
...@@ -5284,27 +5656,33 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5284,27 +5656,33 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5284fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {5656fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
5285 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));5657 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
5286 switch (dst_mcv) {5658 switch (dst_mcv) {
5287 .none => unreachable,5659 .none,
5288 .undef => unreachable,5660 .unreach,
5289 .dead, .unreach, .immediate => unreachable,5661 .dead,
5290 .eflags => unreachable,5662 .undef,
5291 .ptr_stack_offset => unreachable,5663 .immediate,
5292 .lea_direct => unreachable,5664 .register_offset,
5293 .lea_tlv => unreachable,5665 .eflags,
5294 .register_overflow => unreachable,5666 .register_overflow,
5667 .lea_direct,
5668 .lea_got,
5669 .lea_tlv,
5670 .lea_frame,
5671 .reserved_frame,
5672 => unreachable, // unmodifiable destination
5295 .register => |dst_reg| {5673 .register => |dst_reg| {
5296 const dst_alias = registerAlias(dst_reg, abi_size);5674 const dst_alias = registerAlias(dst_reg, abi_size);
5297 const dst_lock = self.register_manager.lockReg(dst_reg);5675 const dst_lock = self.register_manager.lockReg(dst_reg);
5298 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);5676 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
52995677
5300 switch (src_mcv) {5678 switch (src_mcv) {
5301 .none => unreachable,5679 .none,
5302 .undef => try self.genSetReg(dst_ty, dst_reg, .undef),5680 .unreach,
5303 .dead, .unreach => unreachable,5681 .dead,
5304 .ptr_stack_offset => unreachable,5682 .undef,
5305 .lea_direct => unreachable,5683 .register_overflow,
5306 .lea_tlv => unreachable,5684 .reserved_frame,
5307 .register_overflow => unreachable,5685 => unreachable,
5308 .register => |src_reg| try self.asmRegisterRegister(5686 .register => |src_reg| try self.asmRegisterRegister(
5309 .imul,5687 .imul,
5310 dst_alias,5688 dst_alias,
...@@ -5323,74 +5701,54 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M...@@ -5323,74 +5701,54 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
5323 return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });5701 return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });
5324 }5702 }
5325 },5703 },
5326 .stack_offset => |off| {5704 .register_offset,
5327 try self.asmRegisterMemory(5705 .eflags,
5328 .imul,
5329 dst_alias,
5330 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
5331 );
5332 },
5333 .memory,
5334 .load_got,
5335 .load_direct,5706 .load_direct,
5336 .load_tlv,5707 .lea_direct,
5337 => {
5338 return self.fail("TODO implement x86 multiply source memory", .{});
5339 },
5340 .eflags => {
5341 return self.fail("TODO implement x86 multiply source eflags", .{});
5342 },
5343 }
5344 },
5345 .stack_offset => |off| {
5346 switch (src_mcv) {
5347 .none => unreachable,
5348 .undef => return self.genSetStack(dst_ty, off, .undef, .{}),
5349 .dead, .unreach => unreachable,
5350 .ptr_stack_offset => unreachable,
5351 .lea_direct => unreachable,
5352 .lea_tlv => unreachable,
5353 .register_overflow => unreachable,
5354 .register => |src_reg| {
5355 // copy dst to a register
5356 const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
5357 try self.asmRegisterRegister(
5358 .imul,
5359 registerAlias(dst_reg, abi_size),
5360 registerAlias(src_reg, abi_size),
5361 );
5362 // copy dst_reg back out
5363 return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});
5364 },
5365 .immediate => {
5366 // copy dst to a register
5367 const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
5368 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
5369 defer self.register_manager.unlockReg(dst_reg_lock);
5370
5371 try self.genIntMulComplexOpMir(dst_ty, .{ .register = dst_reg }, src_mcv);
5372
5373 return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});
5374 },
5375 .memory,
5376 .load_got,5708 .load_got,
5377 .load_direct,5709 .lea_got,
5378 .load_tlv,5710 .load_tlv,
5379 .stack_offset,5711 .lea_tlv,
5380 => {5712 .lea_frame,
5381 return self.fail("TODO implement x86 multiply source memory", .{});5713 => try self.asmRegisterRegister(
5382 },5714 .imul,
5383 .eflags => {5715 dst_alias,
5384 return self.fail("TODO implement x86 multiply source eflags", .{});5716 registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
5385 },5717 ),
5718 .memory, .indirect, .load_frame => try self.asmRegisterMemory(
5719 .imul,
5720 dst_alias,
5721 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
5722 .memory => |addr| .{
5723 .base = .{ .reg = .ds },
5724 .disp = math.cast(i32, @bitCast(i64, addr)) orelse
5725 return self.asmRegisterRegister(
5726 .imul,
5727 dst_alias,
5728 registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
5729 ),
5730 },
5731 .indirect => |reg_off| .{
5732 .base = .{ .reg = reg_off.reg },
5733 .disp = reg_off.off,
5734 },
5735 .load_frame => |frame_addr| .{
5736 .base = .{ .frame = frame_addr.index },
5737 .disp = frame_addr.off,
5738 },
5739 else => unreachable,
5740 }),
5741 ),
5386 }5742 }
5387 },5743 },
5388 .memory,5744 .memory, .indirect, .load_direct, .load_got, .load_tlv, .load_frame => {
5389 .load_got,5745 const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
5390 .load_direct,5746 const tmp_mcv = MCValue{ .register = tmp_reg };
5391 .load_tlv,5747 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
5392 => {5748 defer self.register_manager.unlockReg(tmp_lock);
5393 return self.fail("TODO implement x86 multiply destination memory", .{});5749
5750 try self.genIntMulComplexOpMir(dst_ty, tmp_mcv, src_mcv);
5751 try self.genCopy(dst_ty, dst_mcv, tmp_mcv);
5394 },5752 },
5395 }5753 }
5396}5754}
...@@ -5401,24 +5759,19 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -5401,24 +5759,19 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
5401 while (self.args[arg_index] == .none) arg_index += 1;5759 while (self.args[arg_index] == .none) arg_index += 1;
5402 self.arg_index = arg_index + 1;5760 self.arg_index = arg_index + 1;
54035761
5404 const ty = self.air.typeOfIndex(inst);
5405 const mcv = self.args[arg_index];
5406 const src_index = self.air.instructions.items(.data)[inst].arg.src_index;
5407 const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index);
5408
5409 const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {5762 const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
5410 const dst_mcv: MCValue = switch (mcv) {5763 const dst_mcv = self.args[arg_index];
5411 .register => |reg| blk: {5764 switch (dst_mcv) {
5412 self.register_manager.getRegAssumeFree(reg.to64(), inst);5765 .register => |reg| self.register_manager.getRegAssumeFree(reg, inst),
5413 break :blk MCValue{ .register = reg };5766 .load_frame => {},
5414 },5767 else => return self.fail("TODO implement arg for {}", .{dst_mcv}),
5415 .stack_offset => |off| blk: {5768 }
5416 const offset = @intCast(i32, self.max_end_stack) - off + 16;5769
5417 break :blk MCValue{ .stack_offset = -offset };5770 const ty = self.air.typeOfIndex(inst);
5418 },5771 const src_index = self.air.instructions.items(.data)[inst].arg.src_index;
5419 else => return self.fail("TODO implement arg for {}", .{mcv}),5772 const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index);
5420 };
5421 try self.genArgDbgInfo(ty, name, dst_mcv);5773 try self.genArgDbgInfo(ty, name, dst_mcv);
5774
5422 break :result dst_mcv;5775 break :result dst_mcv;
5423 };5776 };
5424 return self.finishAir(inst, result, .{ .none, .none, .none });5777 return self.finishAir(inst, result, .{ .none, .none, .none });
...@@ -5429,13 +5782,15 @@ fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void {...@@ -5429,13 +5782,15 @@ fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void {
5429 .dwarf => |dw| {5782 .dwarf => |dw| {
5430 const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {5783 const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {
5431 .register => |reg| .{ .register = reg.dwarfLocOp() },5784 .register => |reg| .{ .register = reg.dwarfLocOp() },
5432 .stack_offset => |off| .{5785 // TODO use a frame index
5433 .stack = .{5786 .load_frame => return,
5434 // TODO handle -fomit-frame-pointer5787 //.stack_offset => |off| .{
5435 .fp_register = Register.rbp.dwarfLocOpDeref(),5788 // .stack = .{
5436 .offset = -off,5789 // // TODO handle -fomit-frame-pointer
5437 },5790 // .fp_register = Register.rbp.dwarfLocOpDeref(),
5438 },5791 // .offset = -off,
5792 // },
5793 //},
5439 else => unreachable, // not a valid function parameter5794 else => unreachable, // not a valid function parameter
5440 };5795 };
5441 try dw.genArgDbgInfo(name, ty, self.mod_fn.owner_decl, loc);5796 try dw.genArgDbgInfo(name, ty, self.mod_fn.owner_decl, loc);
...@@ -5462,12 +5817,12 @@ fn genVarDbgInfo(...@@ -5462,12 +5817,12 @@ fn genVarDbgInfo(
5462 .dwarf => |dw| {5817 .dwarf => |dw| {
5463 const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {5818 const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {
5464 .register => |reg| .{ .register = reg.dwarfLocOp() },5819 .register => |reg| .{ .register = reg.dwarfLocOp() },
5465 .ptr_stack_offset,5820 // TODO use a frame index
5466 .stack_offset,5821 .load_frame, .lea_frame => return,
5467 => |off| .{ .stack = .{5822 //=> |off| .{ .stack = .{
5468 .fp_register = Register.rbp.dwarfLocOpDeref(),5823 // .fp_register = Register.rbp.dwarfLocOpDeref(),
5469 .offset = -off,5824 // .offset = -off,
5470 } },5825 //} },
5471 .memory => |address| .{ .memory = address },5826 .memory => |address| .{ .memory = address },
5472 .load_got => |sym_index| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } },5827 .load_got => |sym_index| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } },
5473 .load_direct => |sym_index| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } },5828 .load_direct => |sym_index| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } },
...@@ -5498,15 +5853,13 @@ fn airBreakpoint(self: *Self) !void {...@@ -5498,15 +5853,13 @@ fn airBreakpoint(self: *Self) !void {
54985853
5499fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {5854fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
5500 const dst_mcv = try self.allocRegOrMem(inst, true);5855 const dst_mcv = try self.allocRegOrMem(inst, true);
5501 try self.setRegOrMem(Type.usize, dst_mcv, .{5856 try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } });
5502 .stack_offset = -@as(i32, @divExact(self.target.cpu.arch.ptrBitWidth(), 8)),
5503 });
5504 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });5857 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
5505}5858}
55065859
5507fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {5860fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
5508 const dst_mcv = try self.allocRegOrMem(inst, true);5861 const dst_mcv = try self.allocRegOrMem(inst, true);
5509 try self.setRegOrMem(Type.usize, dst_mcv, .{ .register = .rbp });5862 try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } });
5510 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });5863 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
5511}5864}
55125865
...@@ -5534,94 +5887,69 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -5534,94 +5887,69 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5534 else => unreachable,5887 else => unreachable,
5535 };5888 };
55365889
5537 var info = try self.resolveCallingConventionValues(fn_ty, args[fn_ty.fnParamLen()..]);5890 var info = try self.resolveCallingConventionValues(fn_ty, args[fn_ty.fnParamLen()..], .call_frame);
5538 defer info.deinit(self);5891 defer info.deinit(self);
55395892
5893 // We need a properly aligned and sized call frame to be able to call this function.
5894 {
5895 const needed_call_frame =
5896 FrameAlloc.init(.{ .size = info.stack_byte_count, .alignment = info.stack_align });
5897 const frame_allocs_slice = self.frame_allocs.slice();
5898 const stack_frame_size =
5899 &frame_allocs_slice.items(.abi_size)[@enumToInt(FrameIndex.call_frame)];
5900 stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size);
5901 const stack_frame_align =
5902 &frame_allocs_slice.items(.abi_align)[@enumToInt(FrameIndex.call_frame)];
5903 stack_frame_align.* = @max(stack_frame_align.*, needed_call_frame.abi_align);
5904 }
5905
5540 try self.spillEflagsIfOccupied();5906 try self.spillEflagsIfOccupied();
5541 try self.spillRegisters(abi.getCallerPreservedRegs(self.target.*));5907 try self.spillRegisters(abi.getCallerPreservedRegs(self.target.*));
55425908
5543 // set stack arguments first because this can clobber registers5909 // set stack arguments first because this can clobber registers
5544 // also clobber spill arguments as we go5910 // also clobber spill arguments as we go
5545 if (info.return_value == .stack_offset) {5911 switch (info.return_value.long) {
5546 try self.spillRegisters(&.{abi.getCAbiIntParamRegs(self.target.*)[0]});5912 .none, .unreach => {},
5913 .indirect => |reg_off| try self.spillRegisters(&.{reg_off.reg}),
5914 else => unreachable,
5547 }5915 }
5548 for (args, info.args) |arg, mc_arg| {5916 for (args, info.args) |arg, mc_arg| {
5549 const arg_ty = self.air.typeOf(arg);5917 const arg_ty = self.air.typeOf(arg);
5550 const arg_mcv = try self.resolveInst(arg);5918 const arg_mcv = try self.resolveInst(arg);
5551 // Here we do not use setRegOrMem even though the logic is similar, because
5552 // the function call will move the stack pointer, so the offsets are different.
5553 switch (mc_arg) {5919 switch (mc_arg) {
5554 .none => {},5920 .none => {},
5555 .register => |reg| try self.spillRegisters(&.{reg}),5921 .register => |reg| try self.spillRegisters(&.{reg}),
5556 .stack_offset => |off| {5922 .load_frame => try self.genCopy(arg_ty, mc_arg, arg_mcv),
5557 // TODO rewrite using `genSetStack`5923 else => unreachable,
5558 try self.genSetStackArg(arg_ty, off, arg_mcv);
5559 },
5560 .ptr_stack_offset => {
5561 return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
5562 },
5563 .undef => unreachable,
5564 .immediate => unreachable,
5565 .unreach => unreachable,
5566 .dead => unreachable,
5567 .memory => unreachable,
5568 .eflags => unreachable,
5569 .register_overflow => unreachable,
5570 .load_got => unreachable,
5571 .lea_direct => unreachable,
5572 .load_direct => unreachable,
5573 .lea_tlv => unreachable,
5574 .load_tlv => unreachable,
5575 }5924 }
5576 }5925 }
55775926
5578 // now we are free to set register arguments5927 // now we are free to set register arguments
5579 const ret_reg_lock: ?RegisterLock = blk: {5928 const ret_lock = switch (info.return_value.long) {
5580 if (info.return_value == .stack_offset) {5929 .none, .unreach => null,
5930 .indirect => |reg_off| lock: {
5581 const ret_ty = fn_ty.fnReturnType();5931 const ret_ty = fn_ty.fnReturnType();
5582 const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));5932 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ret_ty, self.target.*));
5583 const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));5933 try self.genSetReg(reg_off.reg, Type.usize, .{
5584 const stack_offset = @intCast(i32, try self.allocMem(inst, ret_abi_size, ret_abi_align));5934 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
5585 log.debug("airCall: return value on stack at offset {}", .{stack_offset});5935 });
55865936 info.return_value.short = .{ .load_frame = .{ .index = frame_index } };
5587 const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0];5937 break :lock self.register_manager.lockRegAssumeUnused(reg_off.reg);
5588 try self.genSetReg(Type.usize, ret_reg, .{ .ptr_stack_offset = stack_offset });5938 },
5589 const ret_reg_lock = self.register_manager.lockRegAssumeUnused(ret_reg);5939 else => unreachable,
5590
5591 info.return_value.stack_offset = stack_offset;
5592
5593 break :blk ret_reg_lock;
5594 }
5595 break :blk null;
5596 };5940 };
5597 defer if (ret_reg_lock) |lock| self.register_manager.unlockReg(lock);5941 defer if (ret_lock) |lock| self.register_manager.unlockReg(lock);
55985942
5599 for (args, info.args) |arg, mc_arg| {5943 for (args, info.args) |arg, mc_arg| {
5600 const arg_ty = self.air.typeOf(arg);5944 const arg_ty = self.air.typeOf(arg);
5601 const arg_mcv = try self.resolveInst(arg);5945 const arg_mcv = try self.resolveInst(arg);
5602 switch (mc_arg) {5946 switch (mc_arg) {
5603 .none, .stack_offset, .ptr_stack_offset => {},5947 .none, .load_frame => {},
5604 .register => |reg| try self.genSetReg(arg_ty, reg, arg_mcv),5948 .register => try self.genCopy(arg_ty, mc_arg, arg_mcv),
5605 .undef => unreachable,5949 else => unreachable,
5606 .immediate => unreachable,
5607 .unreach => unreachable,
5608 .dead => unreachable,
5609 .memory => unreachable,
5610 .eflags => unreachable,
5611 .register_overflow => unreachable,
5612 .load_got => unreachable,
5613 .lea_direct => unreachable,
5614 .load_direct => unreachable,
5615 .lea_tlv => unreachable,
5616 .load_tlv => unreachable,
5617 }5950 }
5618 }5951 }
56195952
5620 if (info.stack_byte_count > 0) {
5621 // Adjust the stack
5622 try self.asmRegisterImmediate(.sub, .rsp, Immediate.u(info.stack_byte_count));
5623 }
5624
5625 // Due to incremental compilation, how function calls are generated depends5953 // Due to incremental compilation, how function calls are generated depends
5626 // on linking.5954 // on linking.
5627 const mod = self.bin_file.options.module.?;5955 const mod = self.bin_file.options.module.?;
...@@ -5635,16 +5963,16 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -5635,16 +5963,16 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5635 _ = try atom.getOrCreateOffsetTableEntry(elf_file);5963 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
5636 const got_addr = atom.getOffsetTableAddress(elf_file);5964 const got_addr = atom.getOffsetTableAddress(elf_file);
5637 try self.asmMemory(.call, Memory.sib(.qword, .{5965 try self.asmMemory(.call, Memory.sib(.qword, .{
5638 .base = .ds,5966 .base = .{ .reg = .ds },
5639 .disp = @intCast(i32, got_addr),5967 .disp = @intCast(i32, got_addr),
5640 }));5968 }));
5641 } else if (self.bin_file.cast(link.File.Coff)) |_| {5969 } else if (self.bin_file.cast(link.File.Coff)) |_| {
5642 const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);5970 const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);
5643 try self.genSetReg(Type.usize, .rax, .{ .load_got = sym_index });5971 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
5644 try self.asmRegister(.call, .rax);5972 try self.asmRegister(.call, .rax);
5645 } else if (self.bin_file.cast(link.File.MachO)) |_| {5973 } else if (self.bin_file.cast(link.File.MachO)) |_| {
5646 const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);5974 const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);
5647 try self.genSetReg(Type.usize, .rax, .{ .load_got = sym_index });5975 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
5648 try self.asmRegister(.call, .rax);5976 try self.asmRegister(.call, .rax);
5649 } else if (self.bin_file.cast(link.File.Plan9)) |p9| {5977 } else if (self.bin_file.cast(link.File.Plan9)) |p9| {
5650 const decl_block_index = try p9.seeDecl(func.owner_decl);5978 const decl_block_index = try p9.seeDecl(func.owner_decl);
...@@ -5655,7 +5983,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -5655,7 +5983,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5655 const got_index = decl_block.got_index.?;5983 const got_index = decl_block.got_index.?;
5656 const fn_got_addr = got_addr + got_index * ptr_bytes;5984 const fn_got_addr = got_addr + got_index * ptr_bytes;
5657 try self.asmMemory(.call, Memory.sib(.qword, .{5985 try self.asmMemory(.call, Memory.sib(.qword, .{
5658 .base = .ds,5986 .base = .{ .reg = .ds },
5659 .disp = @intCast(i32, fn_got_addr),5987 .disp = @intCast(i32, fn_got_addr),
5660 }));5988 }));
5661 } else unreachable;5989 } else unreachable;
...@@ -5696,41 +6024,15 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -5696,41 +6024,15 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5696 } else {6024 } else {
5697 assert(ty.zigTypeTag() == .Pointer);6025 assert(ty.zigTypeTag() == .Pointer);
5698 const mcv = try self.resolveInst(callee);6026 const mcv = try self.resolveInst(callee);
5699 try self.genSetReg(Type.usize, .rax, mcv);6027 try self.genSetReg(.rax, Type.usize, mcv);
5700 try self.asmRegister(.call, .rax);6028 try self.asmRegister(.call, .rax);
5701 }6029 }
57026030
5703 if (info.stack_byte_count > 0) {
5704 // Readjust the stack
5705 try self.asmRegisterImmediate(.add, .rsp, Immediate.u(info.stack_byte_count));
5706 }
5707
5708 const result: MCValue = result: {
5709 if (self.liveness.isUnused(inst)) break :result .unreach;
5710
5711 switch (info.return_value) {
5712 .register => {
5713 // Save function return value in a new register
5714 break :result try self.copyToRegisterWithInstTracking(
5715 inst,
5716 self.air.typeOfIndex(inst),
5717 info.return_value,
5718 );
5719 },
5720 else => {},
5721 }
5722 break :result info.return_value;
5723 };
5724
5725 if (args.len <= Liveness.bpi - 2) {
5726 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
5727 buf[0] = callee;
5728 std.mem.copy(Air.Inst.Ref, buf[1..], args);
5729 return self.finishAir(inst, result, buf);
5730 }
5731 var bt = self.liveness.iterateBigTomb(inst);6031 var bt = self.liveness.iterateBigTomb(inst);
5732 self.feed(&bt, callee);6032 self.feed(&bt, callee);
5733 for (args) |arg| self.feed(&bt, arg);6033 for (args) |arg| self.feed(&bt, arg);
6034
6035 const result = if (self.liveness.isUnused(inst)) .unreach else info.return_value.short;
5734 return self.finishAirResult(inst, result);6036 return self.finishAirResult(inst, result);
5735}6037}
57366038
...@@ -5738,18 +6040,19 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {...@@ -5738,18 +6040,19 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
5738 const un_op = self.air.instructions.items(.data)[inst].un_op;6040 const un_op = self.air.instructions.items(.data)[inst].un_op;
5739 const operand = try self.resolveInst(un_op);6041 const operand = try self.resolveInst(un_op);
5740 const ret_ty = self.fn_type.fnReturnType();6042 const ret_ty = self.fn_type.fnReturnType();
5741 switch (self.ret_mcv) {6043 switch (self.ret_mcv.short) {
5742 .immediate => {6044 .none => {},
5743 assert(ret_ty.isError());6045 .register => try self.genCopy(ret_ty, self.ret_mcv.short, operand),
5744 },6046 .indirect => |reg_off| {
5745 .stack_offset => {6047 try self.register_manager.getReg(reg_off.reg, null);
5746 try self.store(self.ret_mcv, operand, Type.usize, ret_ty);6048 const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
5747 },6049 defer self.register_manager.unlockReg(lock);
5748 else => {6050
5749 try self.setRegOrMem(ret_ty, self.ret_mcv, operand);6051 try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long);
6052 try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, operand);
5750 },6053 },
6054 else => unreachable,
5751 }6055 }
5752 // TODO when implementing defer, this will need to jump to the appropriate defer expression.
5753 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction6056 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
5754 // which is available if the jump is 127 bytes or less forward.6057 // which is available if the jump is 127 bytes or less forward.
5755 const jmp_reloc = try self.asmJmpReloc(undefined);6058 const jmp_reloc = try self.asmJmpReloc(undefined);
...@@ -5761,25 +6064,12 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -5761,25 +6064,12 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
5761 const un_op = self.air.instructions.items(.data)[inst].un_op;6064 const un_op = self.air.instructions.items(.data)[inst].un_op;
5762 const ptr = try self.resolveInst(un_op);6065 const ptr = try self.resolveInst(un_op);
5763 const ptr_ty = self.air.typeOf(un_op);6066 const ptr_ty = self.air.typeOf(un_op);
5764 const elem_ty = ptr_ty.elemType();6067 switch (self.ret_mcv.short) {
5765 const abi_size = elem_ty.abiSize(self.target.*);6068 .none => {},
5766 switch (self.ret_mcv) {6069 .register => try self.load(self.ret_mcv.short, ptr_ty, ptr),
5767 .immediate => {6070 .indirect => |reg_off| try self.genSetReg(reg_off.reg, ptr_ty, ptr),
5768 assert(elem_ty.isError());6071 else => unreachable,
5769 },
5770 .stack_offset => {
5771 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
5772 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
5773 defer self.register_manager.unlockReg(reg_lock);
5774
5775 try self.genInlineMemcpy(.{ .register = reg }, ptr, .{ .immediate = abi_size }, .{});
5776 },
5777 else => {
5778 try self.load(self.ret_mcv, ptr, ptr_ty);
5779 try self.setRegOrMem(elem_ty, self.ret_mcv, self.ret_mcv);
5780 },
5781 }6072 }
5782 // TODO when implementing defer, this will need to jump to the appropriate defer expression.
5783 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction6073 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
5784 // which is available if the jump is 127 bytes or less forward.6074 // which is available if the jump is 127 bytes or less forward.
5785 const jmp_reloc = try self.asmJmpReloc(undefined);6075 const jmp_reloc = try self.asmJmpReloc(undefined);
...@@ -5822,7 +6112,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -5822,7 +6112,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
5822 break :dst rhs_mcv;6112 break :dst rhs_mcv;
5823 } else if (dst_mem_ok) dst: {6113 } else if (dst_mem_ok) dst: {
5824 const dst_mcv = try self.allocTempRegOrMem(ty, true);6114 const dst_mcv = try self.allocTempRegOrMem(ty, true);
5825 try self.setRegOrMem(ty, dst_mcv, lhs_mcv);6115 try self.genCopy(ty, dst_mcv, lhs_mcv);
5826 break :dst dst_mcv;6116 break :dst dst_mcv;
5827 } else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };6117 } else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };
5828 const dst_lock = switch (dst_mcv) {6118 const dst_lock = switch (dst_mcv) {
...@@ -5865,26 +6155,27 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {...@@ -5865,26 +6155,27 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
5865 const atom = elf_file.getAtom(atom_index);6155 const atom = elf_file.getAtom(atom_index);
5866 _ = try atom.getOrCreateOffsetTableEntry(elf_file);6156 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
5867 const got_addr = atom.getOffsetTableAddress(elf_file);6157 const got_addr = atom.getOffsetTableAddress(elf_file);
5868 try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{6158 try self.asmRegisterMemory(
5869 .base = .ds,6159 .mov,
5870 .disp = @intCast(i32, got_addr),6160 addr_reg.to64(),
5871 }));6161 Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
6162 );
5872 } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {6163 } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
5873 const atom_index = try coff_file.getOrCreateAtomForLazySymbol(6164 const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
5874 .{ .kind = .const_data, .ty = Type.anyerror },6165 .{ .kind = .const_data, .ty = Type.anyerror },
5875 4, // dword alignment6166 4, // dword alignment
5876 );6167 );
5877 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;6168 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
5878 try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });6169 try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
5879 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {6170 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
5880 const atom_index = try macho_file.getOrCreateAtomForLazySymbol(6171 const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
5881 .{ .kind = .const_data, .ty = Type.anyerror },6172 .{ .kind = .const_data, .ty = Type.anyerror },
5882 4, // dword alignment6173 4, // dword alignment
5883 );6174 );
5884 const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;6175 const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
5885 try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });6176 try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
5886 } else {6177 } else {
5887 return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});6178 return self.fail("TODO implement airCmpLtErrorsLen for x86_64 {s}", .{@tagName(self.bin_file.tag)});
5888 }6179 }
58896180
5890 try self.spillEflagsIfOccupied();6181 try self.spillEflagsIfOccupied();
...@@ -5900,7 +6191,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {...@@ -5900,7 +6191,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
5900 try self.asmRegisterMemory(6191 try self.asmRegisterMemory(
5901 .cmp,6192 .cmp,
5902 registerAlias(dst_reg, op_abi_size),6193 registerAlias(dst_reg, op_abi_size),
5903 Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = addr_reg }),6194 Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),
5904 );6195 );
5905 const result = MCValue{ .eflags = .b };6196 const result = MCValue{ .eflags = .b };
5906 return self.finishAir(inst, result, .{ un_op, .none, .none });6197 return self.finishAir(inst, result, .{ un_op, .none, .none });
...@@ -5971,14 +6262,13 @@ fn genTry(...@@ -5971,14 +6262,13 @@ fn genTry(
59716262
5972fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {6263fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
5973 const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;6264 const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
5974 const payload = try self.addExtra(Mir.DbgLineColumn{
5975 .line = dbg_stmt.line,
5976 .column = dbg_stmt.column,
5977 });
5978 _ = try self.addInst(.{6265 _ = try self.addInst(.{
5979 .tag = .dbg_line,6266 .tag = .dbg_line,
5980 .ops = undefined,6267 .ops = undefined,
5981 .data = .{ .payload = payload },6268 .data = .{ .line_column = .{
6269 .line = dbg_stmt.line,
6270 .column = dbg_stmt.column,
6271 } },
5982 });6272 });
5983 return self.finishAirBookkeeping();6273 return self.finishAirBookkeeping();
5984}6274}
...@@ -6002,8 +6292,6 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {...@@ -6002,8 +6292,6 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
6002 const ty = self.air.typeOf(operand);6292 const ty = self.air.typeOf(operand);
6003 const mcv = try self.resolveInst(operand);6293 const mcv = try self.resolveInst(operand);
60046294
6005 log.debug("airDbgVar: %{d}: {}, {}", .{ inst, ty.fmtDebug(), mcv });
6006
6007 const name = self.air.nullTerminatedString(pl_op.payload);6295 const name = self.air.nullTerminatedString(pl_op.payload);
60086296
6009 const tag = self.air.instructions.items(.tag)[inst];6297 const tag = self.air.instructions.items(.tag)[inst];
...@@ -6025,7 +6313,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {...@@ -6025,7 +6313,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
6025 return self.asmJccReloc(undefined, .e);6313 return self.asmJccReloc(undefined, .e);
6026 },6314 },
6027 .immediate,6315 .immediate,
6028 .stack_offset,6316 .load_frame,
6029 => {6317 => {
6030 try self.spillEflagsIfOccupied();6318 try self.spillEflagsIfOccupied();
6031 if (abi_size <= 8) {6319 if (abi_size <= 8) {
...@@ -6118,11 +6406,14 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC...@@ -6118,11 +6406,14 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6118 .dead,6406 .dead,
6119 .undef,6407 .undef,
6120 .immediate,6408 .immediate,
6121 .register_overflow,
6122 .ptr_stack_offset,
6123 .eflags,6409 .eflags,
6410 .register_offset,
6411 .register_overflow,
6124 .lea_direct,6412 .lea_direct,
6413 .lea_got,
6125 .lea_tlv,6414 .lea_tlv,
6415 .lea_frame,
6416 .reserved_frame,
6126 => unreachable,6417 => unreachable,
61276418
6128 .register => |opt_reg| {6419 .register => |opt_reg| {
...@@ -6152,27 +6443,36 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC...@@ -6152,27 +6443,36 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6152 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);6443 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6153 defer self.register_manager.unlockReg(addr_reg_lock);6444 defer self.register_manager.unlockReg(addr_reg_lock);
61546445
6155 try self.genSetReg(Type.usize, addr_reg, switch (opt_mcv) {6446 try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());
6156 .memory => |addr| .{ .immediate = addr },
6157 .load_direct => |sym_index| .{ .lea_direct = sym_index },
6158 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
6159 else => opt_mcv,
6160 });
6161
6162 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));6447 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6163 try self.asmMemoryImmediate(.cmp, Memory.sib(6448 try self.asmMemoryImmediate(
6164 Memory.PtrSize.fromSize(some_abi_size),6449 .cmp,
6165 .{ .base = addr_reg, .disp = some_info.off },6450 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
6166 ), Immediate.u(0));6451 .base = .{ .reg = addr_reg },
6452 .disp = some_info.off,
6453 }),
6454 Immediate.u(0),
6455 );
6167 return .{ .eflags = .e };6456 return .{ .eflags = .e };
6168 },6457 },
61696458
6170 .stack_offset => |off| {6459 .indirect, .load_frame => {
6171 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));6460 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6172 try self.asmMemoryImmediate(.cmp, Memory.sib(6461 try self.asmMemoryImmediate(
6173 Memory.PtrSize.fromSize(some_abi_size),6462 .cmp,
6174 .{ .base = .rbp, .disp = some_info.off - off },6463 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
6175 ), Immediate.u(0));6464 .indirect => |reg_off| .{
6465 .base = .{ .reg = reg_off.reg },
6466 .disp = reg_off.off + some_info.off,
6467 },
6468 .load_frame => |frame_addr| .{
6469 .base = .{ .frame = frame_addr.index },
6470 .disp = frame_addr.off + some_info.off,
6471 },
6472 else => unreachable,
6473 }),
6474 Immediate.u(0),
6475 );
6176 return .{ .eflags = .e };6476 return .{ .eflags = .e };
6177 },6477 },
6178 }6478 }
...@@ -6200,10 +6500,14 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)...@@ -6200,10 +6500,14 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
6200 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);6500 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
62016501
6202 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));6502 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6203 try self.asmMemoryImmediate(.cmp, Memory.sib(6503 try self.asmMemoryImmediate(
6204 Memory.PtrSize.fromSize(some_abi_size),6504 .cmp,
6205 .{ .base = ptr_reg, .disp = some_info.off },6505 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
6206 ), Immediate.u(0));6506 .base = .{ .reg = ptr_reg },
6507 .disp = some_info.off,
6508 }),
6509 Immediate.u(0),
6510 );
6207 return .{ .eflags = .e };6511 return .{ .eflags = .e };
6208}6512}
62096513
...@@ -6221,10 +6525,6 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !...@@ -6221,10 +6525,6 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
62216525
6222 const err_off = errUnionErrorOffset(ty.errorUnionPayload(), self.target.*);6526 const err_off = errUnionErrorOffset(ty.errorUnionPayload(), self.target.*);
6223 switch (operand) {6527 switch (operand) {
6224 .stack_offset => |off| {
6225 const offset = off - @intCast(i32, err_off);
6226 try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 });
6227 },
6228 .register => |reg| {6528 .register => |reg| {
6229 const eu_lock = self.register_manager.lockReg(reg);6529 const eu_lock = self.register_manager.lockReg(reg);
6230 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);6530 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
...@@ -6238,6 +6538,15 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !...@@ -6238,6 +6538,15 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
6238 }6538 }
6239 try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });6539 try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });
6240 },6540 },
6541 .load_frame => |frame_addr| try self.genBinOpMir(
6542 .cmp,
6543 Type.anyerror,
6544 .{ .load_frame = .{
6545 .index = frame_addr.index,
6546 .off = frame_addr.off + @intCast(i32, err_off),
6547 } },
6548 .{ .immediate = 0 },
6549 ),
6241 else => return self.fail("TODO implement isErr for {}", .{operand}),6550 else => return self.fail("TODO implement isErr for {}", .{operand}),
6242 }6551 }
62436552
...@@ -6324,9 +6633,9 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6324,9 +6633,9 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6324 }6633 }
6325 };6634 };
6326 const ptr_ty = self.air.typeOf(un_op);6635 const ptr_ty = self.air.typeOf(un_op);
6327 try self.load(operand, operand_ptr, ptr_ty);6636 try self.load(operand, ptr_ty, operand_ptr);
63286637
6329 const result = try self.isErr(inst, ptr_ty.elemType(), operand);6638 const result = try self.isErr(inst, ptr_ty.childType(), operand);
63306639
6331 return self.finishAir(inst, result, .{ un_op, .none, .none });6640 return self.finishAir(inst, result, .{ un_op, .none, .none });
6332}6641}
...@@ -6358,9 +6667,9 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6358,9 +6667,9 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6358 }6667 }
6359 };6668 };
6360 const ptr_ty = self.air.typeOf(un_op);6669 const ptr_ty = self.air.typeOf(un_op);
6361 try self.load(operand, operand_ptr, ptr_ty);6670 try self.load(operand, ptr_ty, operand_ptr);
63626671
6363 const result = try self.isNonErr(inst, ptr_ty.elemType(), operand);6672 const result = try self.isNonErr(inst, ptr_ty.childType(), operand);
63646673
6365 return self.finishAir(inst, result, .{ un_op, .none, .none });6674 return self.finishAir(inst, result, .{ un_op, .none, .none });
6366}6675}
...@@ -6389,6 +6698,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {...@@ -6389,6 +6698,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
63896698
6390fn airBlock(self: *Self, inst: Air.Inst.Index) !void {6699fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
6391 // A block is a setup to be able to jump to the end.6700 // A block is a setup to be able to jump to the end.
6701 const inst_tracking_i = self.inst_tracking.count();
6392 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));6702 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));
63936703
6394 self.scope_generation += 1;6704 self.scope_generation += 1;
...@@ -6412,8 +6722,9 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {...@@ -6412,8 +6722,9 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
6412 for (block_data.value.relocs.items) |reloc| try self.performReloc(reloc);6722 for (block_data.value.relocs.items) |reloc| try self.performReloc(reloc);
6413 }6723 }
64146724
6415 const tracking = self.inst_tracking.getPtr(inst).?;6725 if (std.debug.runtime_safety) assert(self.inst_tracking.getIndex(inst).? == inst_tracking_i);
6416 if (self.liveness.isUnused(inst)) tracking.die(self);6726 const tracking = &self.inst_tracking.values()[inst_tracking_i];
6727 if (self.liveness.isUnused(inst)) tracking.die(self, inst);
6417 self.getValue(tracking.short, inst);6728 self.getValue(tracking.short, inst);
6418 self.finishAirBookkeeping();6729 self.finishAirBookkeeping();
6419}6730}
...@@ -6520,35 +6831,38 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6520,35 +6831,38 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
6520 !block_ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(br.block_inst);6831 !block_ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(br.block_inst);
6521 const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;6832 const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
6522 const block_data = self.blocks.getPtr(br.block_inst).?;6833 const block_data = self.blocks.getPtr(br.block_inst).?;
6834 const first_br = block_data.relocs.items.len == 0;
6835 const block_result = result: {
6836 if (block_unused) break :result .none;
6837
6838 if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
6839 if (first_br) break :result src_mcv;
6840
6841 if (block_tracking.getReg()) |block_reg|
6842 try self.register_manager.getReg(block_reg, br.block_inst);
6843 // .long = .none to avoid merging operand and block result stack frames.
6844 var current_tracking = InstTracking{ .long = .none, .short = src_mcv };
6845 try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*);
6846 if (src_mcv.getReg()) |src_reg| self.register_manager.freeReg(src_reg);
6847 break :result block_tracking.short;
6848 }
65236849
6524 if (block_data.relocs.items.len == 0) {6850 const dst_mcv = if (first_br) try self.allocRegOrMem(br.block_inst, true) else dst: {
6525 block_tracking.* = InstTracking.init(result: {6851 self.getValue(block_tracking.short, br.block_inst);
6526 if (block_unused) break :result .none;6852 break :dst block_tracking.short;
6527 if (self.reuseOperand(inst, br.operand, 0, src_mcv)) {6853 };
6528 // Fix instruction tracking6854 try self.genCopy(block_ty, dst_mcv, src_mcv);
6529 switch (src_mcv) {6855 break :result dst_mcv;
6530 .register => |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {6856 };
6531 self.register_manager.registers[index] = br.block_inst;
6532 },
6533 else => {},
6534 }
6535 break :result src_mcv;
6536 }
6537
6538 const new_mcv = try self.allocRegOrMem(br.block_inst, true);
6539 try self.setRegOrMem(block_ty, new_mcv, src_mcv);
6540 break :result new_mcv;
6541 });
6542 } else if (!block_unused) try self.setRegOrMem(block_ty, block_tracking.short, src_mcv);
65436857
6544 // Process operand death so that it is properly accounted for in the State below.6858 // Process operand death so that it is properly accounted for in the State below.
6545 if (self.liveness.operandDies(inst, 0)) {6859 if (self.liveness.operandDies(inst, 0)) {
6546 if (Air.refToIndex(br.operand)) |op_inst| self.processDeath(op_inst);6860 if (Air.refToIndex(br.operand)) |op_inst| self.processDeath(op_inst);
6547 }6861 }
65486862
6549 if (block_data.relocs.items.len == 0) {6863 if (first_br) {
6864 block_tracking.* = InstTracking.init(block_result);
6550 try self.saveRetroactiveState(&block_data.state);6865 try self.saveRetroactiveState(&block_data.state);
6551 block_tracking.die(self);
6552 } else try self.restoreState(block_data.state, &.{}, .{6866 } else try self.restoreState(block_data.state, &.{}, .{
6553 .emit_instructions = true,6867 .emit_instructions = true,
6554 .update_tracking = false,6868 .update_tracking = false,
...@@ -6556,6 +6870,9 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6556,6 +6870,9 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
6556 .close_scope = false,6870 .close_scope = false,
6557 });6871 });
65586872
6873 // Stop tracking block result without forgetting tracking info
6874 self.freeValue(block_tracking.short);
6875
6559 // Emit a jump with a relocation. It will be patched up after the block ends.6876 // Emit a jump with a relocation. It will be patched up after the block ends.
6560 // Leave the jump offset undefined6877 // Leave the jump offset undefined
6561 const jmp_reloc = try self.asmJmpReloc(undefined);6878 const jmp_reloc = try self.asmJmpReloc(undefined);
...@@ -6634,7 +6951,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -6634,7 +6951,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
66346951
6635 const arg_mcv = try self.resolveInst(input);6952 const arg_mcv = try self.resolveInst(input);
6636 try self.register_manager.getReg(reg, null);6953 try self.register_manager.getReg(reg, null);
6637 try self.genSetReg(self.air.typeOf(input), reg, arg_mcv);6954 try self.genSetReg(reg, self.air.typeOf(input), arg_mcv);
6638 }6955 }
66396956
6640 {6957 {
...@@ -6666,11 +6983,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -6666,11 +6983,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6666 .qword6983 .qword
6667 else6984 else
6668 null;6985 null;
6669 const mnem = std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse6986 const mnem = mnem: {
6670 (if (mnem_size) |_|6987 if (mnem_size) |_| {
6671 std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])6988 if (std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])) |mnem| {
6672 else6989 break :mnem mnem;
6673 null) orelse return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});6990 }
6991 }
6992 break :mnem std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse
6993 return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});
6994 };
66746995
6675 var op_it = mem.tokenize(u8, mnem_it.rest(), ",");6996 var op_it = mem.tokenize(u8, mnem_it.rest(), ",");
6676 var ops = [1]encoder.Instruction.Operand{.none} ** 4;6997 var ops = [1]encoder.Instruction.Operand{.none} ** 4;
...@@ -6686,7 +7007,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -6686,7 +7007,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6686 return self.fail("Invalid displacement: '{s}'", .{op_str});7007 return self.fail("Invalid displacement: '{s}'", .{op_str});
6687 op.* = .{ .mem = Memory.sib(7008 op.* = .{ .mem = Memory.sib(
6688 mnem_size orelse return self.fail("Unknown size: '{s}'", .{op_str}),7009 mnem_size orelse return self.fail("Unknown size: '{s}'", .{op_str}),
6689 .{ .base = reg, .disp = disp },7010 .{ .base = .{ .reg = reg }, .disp = disp },
6690 ) };7011 ) };
6691 } else {7012 } else {
6692 if (mnem_size) |size| if (reg.bitSize() != size.bitSize())7013 if (mnem_size) |size| if (reg.bitSize() != size.bitSize())
...@@ -6805,335 +7126,326 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -6805,335 +7126,326 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6805 return self.finishAirResult(inst, result);7126 return self.finishAirResult(inst, result);
6806}7127}
68077128
6808/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.7129fn movMirTag(self: *Self, ty: Type) !Mir.Inst.Tag {
6809fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {7130 return switch (ty.zigTypeTag()) {
6810 switch (loc) {7131 else => .mov,
6811 .none => return,7132 .Float => switch (ty.floatBits(self.target.*)) {
6812 .immediate => unreachable,7133 16 => .mov,
6813 .register => |reg| return self.genSetReg(ty, reg, val),7134 32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse)) .movss else .mov,
6814 .stack_offset => |off| return self.genSetStack(ty, off, val, .{}),7135 64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2)) .movsd else .mov,
6815 .memory => {7136 else => return self.fail("TODO movMirTag for {}", .{
6816 return self.fail("TODO implement setRegOrMem for memory", .{});7137 ty.fmt(self.bin_file.options.module.?),
7138 }),
6817 },7139 },
6818 else => {7140 };
6819 return self.fail("TODO implement setRegOrMem for {}", .{loc});7141}
7142
7143fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
7144 const src_lock = switch (src_mcv) {
7145 .register => |reg| self.register_manager.lockReg(reg),
7146 .register_overflow => |ro| self.register_manager.lockReg(ro.reg),
7147 else => null,
7148 };
7149 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
7150
7151 switch (dst_mcv) {
7152 .none,
7153 .unreach,
7154 .dead,
7155 .undef,
7156 .immediate,
7157 .eflags,
7158 .register_overflow,
7159 .lea_direct,
7160 .lea_got,
7161 .lea_tlv,
7162 .lea_frame,
7163 .reserved_frame,
7164 => unreachable, // unmodifiable destination
7165 .register => |reg| try self.genSetReg(reg, ty, src_mcv),
7166 .register_offset => |dst_reg_off| try self.genSetReg(dst_reg_off.reg, ty, switch (src_mcv) {
7167 .none,
7168 .unreach,
7169 .dead,
7170 .undef,
7171 .register_overflow,
7172 .reserved_frame,
7173 => unreachable,
7174 .immediate,
7175 .register,
7176 .register_offset,
7177 .lea_frame,
7178 => src_mcv.offset(-dst_reg_off.off),
7179 else => .{ .register_offset = .{
7180 .reg = try self.copyToTmpRegister(ty, src_mcv),
7181 .off = -dst_reg_off.off,
7182 } },
7183 }),
7184 .indirect => |reg_off| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),
7185 .memory, .load_direct, .load_got, .load_tlv => {
7186 switch (dst_mcv) {
7187 .memory => |addr| if (math.cast(i32, @bitCast(i64, addr))) |small_addr|
7188 return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv),
7189 .load_direct, .load_got, .load_tlv => {},
7190 else => unreachable,
7191 }
7192
7193 const addr_reg = try self.copyToTmpRegister(Type.usize, dst_mcv.address());
7194 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
7195 defer self.register_manager.unlockReg(addr_lock);
7196
7197 try self.genSetMem(.{ .reg = addr_reg }, 0, ty, src_mcv);
6820 },7198 },
7199 .load_frame => |frame_addr| try self.genSetMem(
7200 .{ .frame = frame_addr.index },
7201 frame_addr.off,
7202 ty,
7203 src_mcv,
7204 ),
6821 }7205 }
6822}7206}
68237207
6824fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerError!void {7208fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {
6825 const abi_size = @intCast(u32, ty.abiSize(self.target.*));7209 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
6826 switch (mcv) {7210 if (abi_size > 8) return self.fail("genSetReg called with a value larger than one register", .{});
6827 .dead => unreachable,7211 switch (src_mcv) {
6828 .unreach, .none => return,7212 .none,
6829 .undef => {7213 .unreach,
6830 if (!self.wantSafety()) return; // The already existing value will do just fine.7214 .dead,
6831 if (abi_size <= 8) {7215 .register_overflow,
6832 const reg = try self.copyToTmpRegister(ty, mcv);7216 .reserved_frame,
6833 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });7217 => unreachable,
7218 .undef => if (self.wantSafety())
7219 try self.genSetReg(dst_reg.to64(), Type.usize, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
7220 .eflags => |cc| try self.asmSetccRegister(dst_reg.to8(), cc),
7221 .immediate => |imm| {
7222 if (imm == 0) {
7223 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
7224 // register is the fastest way to zero a register.
7225 try self.asmRegisterRegister(.xor, dst_reg.to32(), dst_reg.to32());
7226 } else if (abi_size > 4 and math.cast(u32, imm) != null) {
7227 // 32-bit moves zero-extend to 64-bit.
7228 try self.asmRegisterImmediate(.mov, dst_reg.to32(), Immediate.u(imm));
7229 } else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
7230 try self.asmRegisterImmediate(
7231 .mov,
7232 registerAlias(dst_reg, abi_size),
7233 Immediate.s(@intCast(i32, @bitCast(i64, imm))),
7234 );
7235 } else {
7236 try self.asmRegisterImmediate(
7237 .mov,
7238 registerAlias(dst_reg, abi_size),
7239 Immediate.u(imm),
7240 );
6834 }7241 }
6835 try self.genInlineMemset(
6836 .{ .ptr_stack_offset = stack_offset },
6837 .{ .immediate = 0xaa },
6838 .{ .immediate = abi_size },
6839 .{ .dest_stack_base = .rsp },
6840 );
6841 },
6842 .register_overflow => return self.fail("TODO genSetStackArg for register with overflow bit", .{}),
6843 .eflags => {
6844 const reg = try self.copyToTmpRegister(ty, mcv);
6845 return self.genSetStackArg(ty, stack_offset, .{ .register = reg });
6846 },7242 },
6847 .immediate => |imm| {7243 .register => |src_reg| if (dst_reg.id() != src_reg.id()) try self.asmRegisterRegister(
6848 switch (abi_size) {7244 if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))
6849 1, 2, 4 => {7245 try self.movMirTag(ty)
6850 // TODO7246 else switch (abi_size) {
6851 // We have a positive stack offset value but we want a twos complement negative7247 4 => .movd,
6852 // offset from rbp, which is at the top of the stack frame.7248 8 => .movq,
6853 const immediate = if (ty.isSignedInt())7249 else => return self.fail(
6854 Immediate.s(@intCast(i32, @bitCast(i64, imm)))7250 "unsupported register copy from {s} to {s}",
6855 else7251 .{ @tagName(src_reg), @tagName(dst_reg) },
6856 Immediate.u(@intCast(u32, imm));7252 ),
6857 try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{7253 },
6858 .base = .rsp,7254 registerAlias(dst_reg, abi_size),
6859 .disp = -stack_offset,7255 registerAlias(src_reg, abi_size),
6860 }), immediate);7256 ),
7257 .register_offset, .indirect, .load_frame, .lea_frame => try self.asmRegisterMemory(
7258 switch (src_mcv) {
7259 .register_offset => |reg_off| switch (reg_off.off) {
7260 0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
7261 else => .lea,
6861 },7262 },
6862 8 => {7263 .indirect, .load_frame => try self.movMirTag(ty),
6863 const reg = try self.copyToTmpRegister(ty, mcv);7264 .lea_frame => .lea,
6864 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });7265 else => unreachable,
7266 },
7267 registerAlias(dst_reg, abi_size),
7268 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
7269 .register_offset, .indirect => |reg_off| .{
7270 .base = .{ .reg = reg_off.reg },
7271 .disp = reg_off.off,
6865 },7272 },
6866 else => return self.fail("TODO implement inputs on stack for {} with abi size > 8", .{mcv}),7273 .load_frame, .lea_frame => |frame_addr| .{
6867 }7274 .base = .{ .frame = frame_addr.index },
6868 },7275 .disp = frame_addr.off,
6869 .memory,7276 },
6870 .load_direct,7277 else => unreachable,
6871 .load_tlv,7278 }),
6872 => {7279 ),
6873 if (abi_size <= 8) {7280 .memory, .load_direct, .load_got, .load_tlv => {
6874 const reg = try self.copyToTmpRegister(ty, mcv);7281 switch (src_mcv) {
6875 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });7282 .memory => |addr| if (math.cast(i32, @bitCast(i64, addr))) |small_addr|
7283 return self.asmRegisterMemory(
7284 try self.movMirTag(ty),
7285 registerAlias(dst_reg, abi_size),
7286 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7287 .base = .{ .reg = .ds },
7288 .disp = small_addr,
7289 }),
7290 ),
7291 .load_direct => |sym_index| if (try self.movMirTag(ty) == .mov) {
7292 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7293 _ = try self.addInst(.{
7294 .tag = .mov_linker,
7295 .ops = .direct_reloc,
7296 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7297 .reg = @enumToInt(dst_reg.to64()),
7298 .atom_index = atom_index,
7299 .sym_index = sym_index,
7300 }) },
7301 });
7302 return;
7303 },
7304 .load_got, .load_tlv => {},
7305 else => unreachable,
6876 }7306 }
68777307
6878 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();7308 const addr_reg = try self.copyToTmpRegister(Type.usize, src_mcv.address());
6879 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);7309 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6880 defer self.register_manager.unlockReg(addr_lock);7310 defer self.register_manager.unlockReg(addr_lock);
68817311
6882 try self.genSetReg(Type.usize, addr_reg, switch (mcv) {7312 try self.asmRegisterMemory(
6883 .memory => |addr| .{ .immediate = addr },7313 try self.movMirTag(ty),
6884 .load_direct => |sym_index| .{ .lea_direct = sym_index },7314 registerAlias(dst_reg, abi_size),
6885 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },7315 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
6886 else => unreachable,
6887 });
6888 try self.genInlineMemcpy(
6889 .{ .ptr_stack_offset = stack_offset },
6890 .{ .register = addr_reg },
6891 .{ .immediate = abi_size },
6892 .{ .dest_stack_base = .rsp },
6893 );7316 );
6894 },7317 },
6895 .register => |reg| {7318 .lea_direct, .lea_got => |sym_index| {
6896 switch (ty.zigTypeTag()) {7319 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
6897 .Float => {7320 _ = try self.addInst(.{
6898 if (intrinsicsAllowed(self.target.*, ty)) {7321 .tag = switch (src_mcv) {
6899 const tag: Mir.Inst.Tag = switch (ty.tag()) {7322 .lea_direct => .lea_linker,
6900 .f32 => .movss,7323 .lea_got => .mov_linker,
6901 .f64 => .movsd,7324 else => unreachable,
6902 else => return self.fail(
6903 "TODO genSetStackArg for register for type {}",
6904 .{ty.fmtDebug()},
6905 ),
6906 };
6907 // TODO verify this
6908 const ptr_size: Memory.PtrSize = switch (ty.tag()) {
6909 .f32 => .dword,
6910 .f64 => .qword,
6911 else => unreachable,
6912 };
6913 return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
6914 .base = .rsp,
6915 .disp = -stack_offset,
6916 }), reg.to128());
6917 }
6918
6919 return self.fail("TODO genSetStackArg for register with no intrinsics", .{});
6920 },7325 },
6921 else => {7326 .ops = switch (src_mcv) {
6922 try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{7327 .lea_direct => .direct_reloc,
6923 .base = .rsp,7328 .lea_got => .got_reloc,
6924 .disp = -stack_offset,7329 else => unreachable,
6925 }), registerAlias(reg, abi_size));
6926 },7330 },
6927 }7331 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
6928 },7332 .reg = @enumToInt(dst_reg.to64()),
6929 .ptr_stack_offset,7333 .atom_index = atom_index,
6930 .load_got,7334 .sym_index = sym_index,
6931 .lea_direct,7335 }) },
6932 .lea_tlv,7336 });
6933 => {
6934 const reg = try self.copyToTmpRegister(ty, mcv);
6935 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
6936 },7337 },
6937 .stack_offset => |mcv_off| {7338 .lea_tlv => |sym_index| {
6938 if (abi_size <= 8) {7339 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
6939 const reg = try self.copyToTmpRegister(ty, mcv);7340 if (self.bin_file.cast(link.File.MachO)) |_| {
6940 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });7341 _ = try self.addInst(.{
6941 }7342 .tag = .lea_linker,
69427343 .ops = .tlv_reloc,
6943 try self.genInlineMemcpy(7344 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
6944 .{ .ptr_stack_offset = stack_offset },7345 .reg = @enumToInt(Register.rdi),
6945 .{ .ptr_stack_offset = mcv_off },7346 .atom_index = atom_index,
6946 .{ .immediate = abi_size },7347 .sym_index = sym_index,
6947 .{ .dest_stack_base = .rsp },7348 }) },
6948 );7349 });
7350 // TODO: spill registers before calling
7351 try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
7352 try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });
7353 } else return self.fail("TODO emit ptr to TLV sequence on {s}", .{
7354 @tagName(self.bin_file.tag),
7355 });
6949 },7356 },
6950 }7357 }
6951}7358}
69527359
6953fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: InlineMemcpyOpts) InnerError!void {7360fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCValue) InnerError!void {
6954 const base_reg = opts.dest_stack_base orelse .rbp;
6955 const abi_size = @intCast(u32, ty.abiSize(self.target.*));7361 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
6956 switch (mcv) {7362 const dst_ptr_mcv: MCValue = switch (base) {
6957 .dead => unreachable,7363 .none => .{ .immediate = @bitCast(u64, @as(i64, disp)) },
6958 .unreach, .none => return, // Nothing to do.7364 .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
6959 .undef => {7365 .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
6960 if (!self.wantSafety()) return; // The already existing value will do just fine.7366 };
6961 // TODO Upgrade this to a memset call when we have that available.7367 switch (src_mcv) {
6962 switch (abi_size) {7368 .none, .unreach, .dead, .reserved_frame => unreachable,
6963 1, 2, 4 => {7369 .undef => if (self.wantSafety())
6964 const value: u64 = switch (abi_size) {7370 try self.genInlineMemset(dst_ptr_mcv, .{ .immediate = 0xaa }, .{ .immediate = abi_size }),
6965 1 => 0xaa,7371 .immediate => |imm| switch (abi_size) {
6966 2 => 0xaaaa,7372 1, 2, 4 => {
6967 4 => 0xaaaaaaaa,7373 const immediate = if (ty.isSignedInt())
6968 else => unreachable,7374 Immediate.s(@truncate(i32, @bitCast(i64, imm)))
6969 };7375 else
6970 return self.asmMemoryImmediate(.mov, Memory.sib(7376 Immediate.u(@intCast(u32, imm));
6971 Memory.PtrSize.fromSize(abi_size),7377 try self.asmMemoryImmediate(
6972 .{ .base = base_reg, .disp = -stack_offset },7378 .mov,
6973 ), Immediate.u(value));7379 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
6974 },7380 immediate,
6975 8 => return self.genSetStack(7381 );
6976 ty,7382 },
6977 stack_offset,7383 3, 5...7 => unreachable,
6978 .{ .immediate = 0xaaaaaaaaaaaaaaaa },7384 else => if (math.cast(i32, @bitCast(i64, imm))) |small| {
6979 opts,7385 try self.asmMemoryImmediate(
6980 ),7386 .mov,
6981 else => |x| return self.genInlineMemset(7387 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
6982 .{ .ptr_stack_offset = stack_offset },7388 Immediate.s(small),
6983 .{ .immediate = 0xaa },7389 );
6984 .{ .immediate = x },7390 } else {
6985 opts,7391 var offset: i32 = 0;
6986 ),7392 while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
6987 }7393 .mov,
7394 Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
7395 if (ty.isSignedInt())
7396 Immediate.s(@truncate(
7397 i32,
7398 @bitCast(i64, imm) >> (math.cast(u6, offset * 8) orelse 63),
7399 ))
7400 else
7401 Immediate.u(@truncate(
7402 u32,
7403 if (math.cast(u6, offset * 8)) |shift| imm >> shift else 0,
7404 )),
7405 );
7406 },
6988 },7407 },
7408 .eflags => |cc| try self.asmSetccMemory(Memory.sib(.byte, .{ .base = base, .disp = disp }), cc),
7409 .register => |reg| try self.asmMemoryRegister(
7410 try self.movMirTag(ty),
7411 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
7412 registerAlias(reg, abi_size),
7413 ),
6989 .register_overflow => |ro| {7414 .register_overflow => |ro| {
6990 const reg_lock = self.register_manager.lockReg(ro.reg);7415 try self.genSetMem(
6991 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);7416 base,
69927417 disp + @intCast(i32, ty.structFieldOffset(0, self.target.*)),
6993 const wrapped_ty = ty.structFieldType(0);7418 ty.structFieldType(0),
6994 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = ro.reg }, .{});7419 .{ .register = ro.reg },
69957420 );
6996 const overflow_bit_ty = ty.structFieldType(1);7421 try self.genSetMem(
6997 const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);7422 base,
6998 const tmp_reg = try self.register_manager.allocReg(null, gp);7423 disp + @intCast(i32, ty.structFieldOffset(1, self.target.*)),
6999 try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);7424 ty.structFieldType(1),
70007425 .{ .eflags = ro.eflags },
7001 return self.genSetStack(
7002 overflow_bit_ty,
7003 stack_offset - @intCast(i32, overflow_bit_offset),
7004 .{ .register = tmp_reg.to8() },
7005 .{},
7006 );7426 );
7007 },7427 },
7008 .eflags => |cc| try self.asmSetccMemory(7428 .register_offset,
7009 Memory.sib(.byte, .{ .base = base_reg, .disp = -stack_offset }),
7010 cc,
7011 ),
7012 .immediate => |imm| {
7013 // TODO
7014 switch (abi_size) {
7015 0 => {
7016 assert(ty.isError());
7017 try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{
7018 .base = base_reg,
7019 .disp = -stack_offset,
7020 }), Immediate.u(@truncate(u8, imm)));
7021 },
7022 1, 2, 4 => {
7023 const immediate = if (ty.isSignedInt())
7024 Immediate.s(@truncate(i32, @bitCast(i64, imm)))
7025 else
7026 Immediate.u(@intCast(u32, imm));
7027 try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7028 .base = base_reg,
7029 .disp = -stack_offset,
7030 }), immediate);
7031 },
7032 3, 5...7 => unreachable,
7033 else => {
7034 // 64 bit write to memory would take two mov's anyways so we
7035 // insted just use two 32 bit writes to avoid register allocation
7036 if (math.cast(i32, @bitCast(i64, imm))) |small| {
7037 try self.asmMemoryImmediate(.mov, Memory.sib(
7038 Memory.PtrSize.fromSize(abi_size),
7039 .{ .base = base_reg, .disp = -stack_offset },
7040 ), Immediate.s(small));
7041 } else {
7042 var offset: i32 = 0;
7043 while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
7044 .mov,
7045 Memory.sib(.dword, .{ .base = base_reg, .disp = offset - stack_offset }),
7046 if (ty.isSignedInt())
7047 Immediate.s(@truncate(
7048 i32,
7049 @bitCast(i64, imm) >> (math.cast(u6, offset * 8) orelse 63),
7050 ))
7051 else
7052 Immediate.u(@truncate(
7053 u32,
7054 if (math.cast(u6, offset * 8)) |shift| imm >> shift else 0,
7055 )),
7056 );
7057 }
7058 },
7059 }
7060 },
7061 .register => |reg| {
7062 switch (ty.zigTypeTag()) {
7063 .Float => {
7064 if (intrinsicsAllowed(self.target.*, ty)) {
7065 const tag: Mir.Inst.Tag = switch (ty.tag()) {
7066 .f32 => .movss,
7067 .f64 => .movsd,
7068 else => return self.fail(
7069 "TODO genSetStack for register for type {}",
7070 .{ty.fmtDebug()},
7071 ),
7072 };
7073 const ptr_size: Memory.PtrSize = switch (ty.tag()) {
7074 .f32 => .dword,
7075 .f64 => .qword,
7076 else => unreachable,
7077 };
7078 return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
7079 .base = base_reg.to64(),
7080 .disp = -stack_offset,
7081 }), reg.to128());
7082 }
7083
7084 return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});
7085 },
7086 else => {
7087 try self.genInlineMemcpyRegisterRegister(ty, base_reg, reg, stack_offset);
7088 },
7089 }
7090 },
7091 .memory,7429 .memory,
7430 .indirect,
7092 .load_direct,7431 .load_direct,
7093 .load_tlv,
7094 .lea_direct,7432 .lea_direct,
7095 .lea_tlv,
7096 => if (abi_size <= 8) {
7097 const reg = try self.copyToTmpRegister(ty, mcv);
7098 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
7099 } else {
7100 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
7101 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
7102 defer self.register_manager.unlockReg(addr_lock);
7103
7104 try self.genSetReg(Type.usize, addr_reg, switch (mcv) {
7105 .memory => |addr| .{ .immediate = addr },
7106 .load_direct => |sym_index| .{ .lea_direct = sym_index },
7107 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
7108 else => mcv,
7109 });
7110 try self.genInlineMemcpy(
7111 .{ .ptr_stack_offset = stack_offset },
7112 .{ .register = addr_reg },
7113 .{ .immediate = abi_size },
7114 .{},
7115 );
7116 },
7117 .stack_offset => |off| if (abi_size <= 8) {
7118 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7119 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7120 defer self.register_manager.unlockReg(tmp_lock);
7121
7122 try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
7123 } else try self.genInlineMemcpy(
7124 .{ .ptr_stack_offset = stack_offset },
7125 .{ .ptr_stack_offset = off },
7126 .{ .immediate = abi_size },
7127 .{},
7128 ),
7129 .ptr_stack_offset,
7130 .load_got,7433 .load_got,
7131 => {7434 .lea_got,
7132 const tmp_reg = try self.copyToTmpRegister(ty, mcv);7435 .load_tlv,
7133 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);7436 .lea_tlv,
7134 defer self.register_manager.unlockReg(tmp_lock);7437 .load_frame,
71357438 .lea_frame,
7136 try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);7439 => switch (abi_size) {
7440 0 => {},
7441 1, 2, 4, 8 => {
7442 const src_reg = try self.copyToTmpRegister(ty, src_mcv);
7443 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
7444 defer self.register_manager.unlockReg(src_lock);
7445
7446 try self.genSetMem(base, disp, ty, .{ .register = src_reg });
7447 },
7448 else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }),
7137 },7449 },
7138 }7450 }
7139}7451}
...@@ -7165,10 +7477,14 @@ fn genInlineMemcpyRegisterRegister(...@@ -7165,10 +7477,14 @@ fn genInlineMemcpyRegisterRegister(
7165 var remainder = abi_size;7477 var remainder = abi_size;
7166 while (remainder > 0) {7478 while (remainder > 0) {
7167 const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));7479 const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
7168 try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{7480 try self.asmMemoryRegister(
7169 .base = dst_reg,7481 .mov,
7170 .disp = -next_offset,7482 Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
7171 }), registerAlias(tmp_reg, nearest_power_of_two));7483 .base = dst_reg,
7484 .disp = -next_offset,
7485 }),
7486 registerAlias(tmp_reg, nearest_power_of_two),
7487 );
71727488
7173 if (nearest_power_of_two > 1) {7489 if (nearest_power_of_two > 1) {
7174 try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{7490 try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{
...@@ -7191,121 +7507,11 @@ fn genInlineMemcpyRegisterRegister(...@@ -7191,121 +7507,11 @@ fn genInlineMemcpyRegisterRegister(
7191 }7507 }
7192}7508}
71937509
7194const InlineMemcpyOpts = struct {7510fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {
7195 source_stack_base: ?Register = null,
7196 dest_stack_base: ?Register = null,
7197};
7198
7199fn genInlineMemcpy(
7200 self: *Self,
7201 dst_ptr: MCValue,
7202 src_ptr: MCValue,
7203 len: MCValue,
7204 opts: InlineMemcpyOpts,
7205) InnerError!void {
7206 const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) |reg|
7207 self.register_manager.lockReg(reg)
7208 else
7209 null;
7210 defer if (ssbase_lock) |reg| self.register_manager.unlockReg(reg);
7211
7212 const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) |reg|
7213 self.register_manager.lockReg(reg)
7214 else
7215 null;
7216 defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
7217
7218 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });7511 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
72197512 try self.genSetReg(.rdi, Type.usize, dst_ptr);
7220 switch (dst_ptr) {7513 try self.genSetReg(.rsi, Type.usize, src_ptr);
7221 .lea_tlv,7514 try self.genSetReg(.rcx, Type.usize, len);
7222 .load_tlv,
7223 => {
7224 try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7225 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
7226 else => dst_ptr,
7227 });
7228 },
7229 .memory,
7230 .load_got,
7231 .lea_direct,
7232 .load_direct,
7233 => {
7234 try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7235 .memory => |addr| .{ .immediate = addr },
7236 .load_direct => |sym_index| .{ .lea_direct = sym_index },
7237 else => dst_ptr,
7238 });
7239 // Load the pointer, which is stored in memory
7240 try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
7241 },
7242 .stack_offset, .ptr_stack_offset => |off| {
7243 try self.asmRegisterMemory(switch (dst_ptr) {
7244 .stack_offset => .mov,
7245 .ptr_stack_offset => .lea,
7246 else => unreachable,
7247 }, .rdi, Memory.sib(.qword, .{
7248 .base = opts.dest_stack_base orelse .rbp,
7249 .disp = -off,
7250 }));
7251 },
7252 .register => |reg| {
7253 try self.asmRegisterRegister(
7254 .mov,
7255 registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7256 reg,
7257 );
7258 },
7259 else => {
7260 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
7261 },
7262 }
7263
7264 switch (src_ptr) {
7265 .lea_tlv,
7266 .load_tlv,
7267 => {
7268 try self.genSetReg(Type.usize, .rsi, switch (src_ptr) {
7269 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
7270 else => dst_ptr,
7271 });
7272 },
7273 .memory,
7274 .load_got,
7275 .lea_direct,
7276 .load_direct,
7277 => {
7278 try self.genSetReg(Type.usize, .rsi, switch (src_ptr) {
7279 .memory => |addr| .{ .immediate = addr },
7280 .load_direct => |sym_index| .{ .lea_direct = sym_index },
7281 else => src_ptr,
7282 });
7283 // Load the pointer, which is stored in memory
7284 try self.asmRegisterMemory(.mov, .rsi, Memory.sib(.qword, .{ .base = .rsi }));
7285 },
7286 .stack_offset, .ptr_stack_offset => |off| {
7287 try self.asmRegisterMemory(switch (src_ptr) {
7288 .stack_offset => .mov,
7289 .ptr_stack_offset => .lea,
7290 else => unreachable,
7291 }, .rsi, Memory.sib(.qword, .{
7292 .base = opts.source_stack_base orelse .rbp,
7293 .disp = -off,
7294 }));
7295 },
7296 .register => |reg| {
7297 try self.asmRegisterRegister(
7298 .mov,
7299 registerAlias(.rsi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7300 reg,
7301 );
7302 },
7303 else => {
7304 return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
7305 },
7306 }
7307
7308 try self.genSetReg(Type.usize, .rcx, len);
7309 _ = try self.addInst(.{7515 _ = try self.addInst(.{
7310 .tag = .movs,7516 .tag = .movs,
7311 .ops = .string,7517 .ops = .string,
...@@ -7313,67 +7519,11 @@ fn genInlineMemcpy(...@@ -7313,67 +7519,11 @@ fn genInlineMemcpy(
7313 });7519 });
7314}7520}
73157521
7316fn genInlineMemset(7522fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {
7317 self: *Self,
7318 dst_ptr: MCValue,
7319 value: MCValue,
7320 len: MCValue,
7321 opts: InlineMemcpyOpts,
7322) InnerError!void {
7323 const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) |reg|
7324 self.register_manager.lockReg(reg)
7325 else
7326 null;
7327 defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
7328
7329 try self.spillRegisters(&.{ .rdi, .al, .rcx });7523 try self.spillRegisters(&.{ .rdi, .al, .rcx });
73307524 try self.genSetReg(.rdi, Type.usize, dst_ptr);
7331 switch (dst_ptr) {7525 try self.genSetReg(.al, Type.u8, value);
7332 .lea_tlv,7526 try self.genSetReg(.rcx, Type.usize, len);
7333 .load_tlv,
7334 => {
7335 try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7336 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
7337 else => dst_ptr,
7338 });
7339 },
7340 .load_got => try self.genSetReg(Type.usize, .rdi, dst_ptr),
7341 .memory,
7342 .lea_direct,
7343 .load_direct,
7344 => {
7345 try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7346 .memory => |addr| .{ .immediate = addr },
7347 .load_direct => |sym_index| .{ .lea_direct = sym_index },
7348 else => dst_ptr,
7349 });
7350 // Load the pointer, which is stored in memory
7351 try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
7352 },
7353 .stack_offset, .ptr_stack_offset => |off| {
7354 try self.asmRegisterMemory(switch (dst_ptr) {
7355 .stack_offset => .mov,
7356 .ptr_stack_offset => .lea,
7357 else => unreachable,
7358 }, .rdi, Memory.sib(.qword, .{
7359 .base = opts.dest_stack_base orelse .rbp,
7360 .disp = -off,
7361 }));
7362 },
7363 .register => |reg| {
7364 try self.asmRegisterRegister(
7365 .mov,
7366 registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7367 reg,
7368 );
7369 },
7370 else => {
7371 return self.fail("TODO implement memset for setting stack when dest is {}", .{dst_ptr});
7372 },
7373 }
7374
7375 try self.genSetReg(Type.u8, .al, value);
7376 try self.genSetReg(Type.usize, .rcx, len);
7377 _ = try self.addInst(.{7527 _ = try self.addInst(.{
7378 .tag = .stos,7528 .tag = .stos,
7379 .ops = .string,7529 .ops = .string,
...@@ -7381,307 +7531,6 @@ fn genInlineMemset(...@@ -7381,307 +7531,6 @@ fn genInlineMemset(
7381 });7531 });
7382}7532}
73837533
7384fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
7385 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
7386 if (abi_size > 8) return self.fail("genSetReg called with a value larger than one register", .{});
7387 switch (mcv) {
7388 .dead => unreachable,
7389 .register_overflow => unreachable,
7390 .ptr_stack_offset => |off| {
7391 try self.asmRegisterMemory(
7392 .lea,
7393 registerAlias(reg, abi_size),
7394 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
7395 );
7396 },
7397 .unreach, .none => return, // Nothing to do.
7398 .undef => {
7399 if (!self.wantSafety()) return; // The already existing value will do just fine.
7400 // Write the debug undefined value.
7401 switch (self.regBitSize(ty)) {
7402 8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),
7403 16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }),
7404 32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),
7405 64 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
7406 else => unreachable,
7407 }
7408 },
7409 .eflags => |cc| {
7410 return self.asmSetccRegister(reg.to8(), cc);
7411 },
7412 .immediate => |imm| {
7413 if (imm == 0) {
7414 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
7415 // register is the fastest way to zero a register.
7416 try self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
7417 } else if (abi_size > 4 and math.cast(u32, imm) != null) {
7418 // 32-bit moves zero-extend to 64-bit.
7419 try self.asmRegisterImmediate(.mov, reg.to32(), Immediate.u(imm));
7420 } else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
7421 try self.asmRegisterImmediate(
7422 .mov,
7423 registerAlias(reg, abi_size),
7424 Immediate.s(@intCast(i32, @bitCast(i64, imm))),
7425 );
7426 } else {
7427 try self.asmRegisterImmediate(
7428 .mov,
7429 registerAlias(reg, abi_size),
7430 Immediate.u(imm),
7431 );
7432 }
7433 },
7434 .register => |src_reg| {
7435 // If the registers are the same, nothing to do.
7436 if (src_reg.id() == reg.id())
7437 return;
7438
7439 switch (ty.zigTypeTag()) {
7440 .Int => switch (ty.intInfo(self.target.*).signedness) {
7441 .signed => {
7442 if (abi_size <= 4) {
7443 return self.asmRegisterRegister(
7444 .movsx,
7445 reg.to64(),
7446 registerAlias(src_reg, abi_size),
7447 );
7448 }
7449 },
7450 .unsigned => {
7451 if (abi_size <= 2) {
7452 return self.asmRegisterRegister(
7453 .movzx,
7454 reg.to64(),
7455 registerAlias(src_reg, abi_size),
7456 );
7457 }
7458 },
7459 },
7460 .Float => {
7461 if (intrinsicsAllowed(self.target.*, ty)) {
7462 const tag: Mir.Inst.Tag = switch (ty.tag()) {
7463 .f32 => .movss,
7464 .f64 => .movsd,
7465 else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
7466 };
7467 return self.asmRegisterRegister(tag, reg.to128(), src_reg.to128());
7468 }
7469 return self.fail("TODO genSetReg from register for float with no intrinsics", .{});
7470 },
7471 else => {},
7472 }
7473
7474 try self.asmRegisterRegister(.mov, registerAlias(reg, abi_size), registerAlias(src_reg, abi_size));
7475 },
7476 .memory => |addr| switch (ty.zigTypeTag()) {
7477 .Float => {
7478 const base_reg = (try self.register_manager.allocReg(null, gp)).to64();
7479 try self.genSetReg(Type.usize, base_reg, .{ .immediate = addr });
7480
7481 if (intrinsicsAllowed(self.target.*, ty)) {
7482 return self.asmRegisterMemory(
7483 switch (ty.tag()) {
7484 .f32 => .movss,
7485 .f64 => .movsd,
7486 else => return self.fail("TODO genSetReg from memory for {}", .{
7487 ty.fmt(self.bin_file.options.module.?),
7488 }),
7489 },
7490 reg.to128(),
7491 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7492 );
7493 }
7494
7495 return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
7496 },
7497 else => {
7498 if (addr <= math.maxInt(i32)) {
7499 try self.asmRegisterMemory(
7500 .mov,
7501 registerAlias(reg, abi_size),
7502 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7503 .base = .ds,
7504 .disp = @intCast(i32, addr),
7505 }),
7506 );
7507 } else {
7508 if (reg.to64() == .rax) {
7509 // If this is RAX, we can use a direct load.
7510 // Otherwise, we need to load the address, then indirectly load the value.
7511 _ = try self.addInst(.{
7512 .tag = .mov_moffs,
7513 .ops = .rax_moffs,
7514 .data = .{ .payload = try self.addExtra(Mir.MemoryMoffs.encode(.ds, addr)) },
7515 });
7516 } else {
7517 // Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
7518 try self.genSetReg(Type.usize, reg, MCValue{ .immediate = addr });
7519 try self.asmRegisterMemory(
7520 .mov,
7521 registerAlias(reg, abi_size),
7522 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
7523 );
7524 }
7525 }
7526 },
7527 },
7528 .load_got, .lea_direct => |sym_index| {
7529 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7530 _ = try self.addInst(.{
7531 .tag = switch (mcv) {
7532 .load_got => .mov_linker,
7533 .lea_direct => .lea_linker,
7534 else => unreachable,
7535 },
7536 .ops = switch (mcv) {
7537 .load_got => .got_reloc,
7538 .lea_direct => .direct_reloc,
7539 else => unreachable,
7540 },
7541 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7542 .reg = @enumToInt(reg),
7543 .atom_index = atom_index,
7544 .sym_index = sym_index,
7545 }) },
7546 });
7547 },
7548 .load_direct => |sym_index| {
7549 switch (ty.zigTypeTag()) {
7550 .Float => {
7551 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
7552 try self.genSetReg(Type.usize, addr_reg, .{ .lea_direct = sym_index });
7553
7554 if (intrinsicsAllowed(self.target.*, ty)) {
7555 return self.asmRegisterMemory(
7556 switch (ty.tag()) {
7557 .f32 => .movss,
7558 .f64 => .movsd,
7559 else => return self.fail("TODO genSetReg from memory for {}", .{
7560 ty.fmt(self.bin_file.options.module.?),
7561 }),
7562 },
7563 reg.to128(),
7564 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
7565 );
7566 }
7567
7568 return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
7569 },
7570 else => {
7571 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7572 _ = try self.addInst(.{
7573 .tag = .mov_linker,
7574 .ops = .direct_reloc,
7575 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7576 .reg = @enumToInt(registerAlias(reg, abi_size)),
7577 .atom_index = atom_index,
7578 .sym_index = sym_index,
7579 }) },
7580 });
7581 },
7582 }
7583 },
7584 .lea_tlv => |sym_index| {
7585 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7586 if (self.bin_file.cast(link.File.MachO)) |_| {
7587 _ = try self.addInst(.{
7588 .tag = .lea_linker,
7589 .ops = .tlv_reloc,
7590 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7591 .reg = @enumToInt(Register.rdi),
7592 .atom_index = atom_index,
7593 .sym_index = sym_index,
7594 }) },
7595 });
7596 // TODO: spill registers before calling
7597 try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .rdi }));
7598 try self.genSetReg(Type.usize, reg, .{ .register = .rax });
7599 } else return self.fail("TODO emit ptr to TLV sequence on {s}", .{@tagName(self.bin_file.tag)});
7600 },
7601 .load_tlv => |sym_index| {
7602 const base_reg = switch (ty.zigTypeTag()) {
7603 .Float => (try self.register_manager.allocReg(null, gp)).to64(),
7604 else => reg.to64(),
7605 };
7606 try self.genSetReg(Type.usize, base_reg, .{ .lea_tlv = sym_index });
7607 switch (ty.zigTypeTag()) {
7608 .Float => if (intrinsicsAllowed(self.target.*, ty)) {
7609 return self.asmRegisterMemory(
7610 switch (ty.tag()) {
7611 .f32 => .movss,
7612 .f64 => .movsd,
7613 else => return self.fail("TODO genSetReg from memory for {}", .{
7614 ty.fmt(self.bin_file.options.module.?),
7615 }),
7616 },
7617 reg.to128(),
7618 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7619 );
7620 } else return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}),
7621 else => try self.asmRegisterMemory(
7622 .mov,
7623 registerAlias(reg, abi_size),
7624 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7625 ),
7626 }
7627 },
7628 .stack_offset => |off| {
7629 switch (ty.zigTypeTag()) {
7630 .Int => switch (ty.intInfo(self.target.*).signedness) {
7631 .signed => {
7632 if (abi_size <= 4) {
7633 return self.asmRegisterMemory(
7634 .movsx,
7635 reg.to64(),
7636 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7637 .base = .rbp,
7638 .disp = -off,
7639 }),
7640 );
7641 }
7642 },
7643 .unsigned => {
7644 if (abi_size <= 2) {
7645 return self.asmRegisterMemory(
7646 .movzx,
7647 reg.to64(),
7648 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7649 .base = .rbp,
7650 .disp = -off,
7651 }),
7652 );
7653 }
7654 },
7655 },
7656 .Float => {
7657 if (intrinsicsAllowed(self.target.*, ty)) {
7658 const tag: Mir.Inst.Tag = switch (ty.tag()) {
7659 .f32 => .movss,
7660 .f64 => .movsd,
7661 else => return self.fail(
7662 "TODO genSetReg from stack offset for {}",
7663 .{ty.fmtDebug()},
7664 ),
7665 };
7666 return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(
7667 Memory.PtrSize.fromSize(abi_size),
7668 .{ .base = .rbp, .disp = -off },
7669 ));
7670 }
7671 return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{});
7672 },
7673 else => {},
7674 }
7675
7676 try self.asmRegisterMemory(
7677 .mov,
7678 registerAlias(reg, abi_size),
7679 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
7680 );
7681 },
7682 }
7683}
7684
7685fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {7534fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
7686 const un_op = self.air.instructions.items(.data)[inst].un_op;7535 const un_op = self.air.instructions.items(.data)[inst].un_op;
7687 const result = result: {7536 const result = result: {
...@@ -7691,7 +7540,7 @@ fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {...@@ -7691,7 +7540,7 @@ fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
76917540
7692 const dst_mcv = try self.allocRegOrMem(inst, true);7541 const dst_mcv = try self.allocRegOrMem(inst, true);
7693 const dst_ty = self.air.typeOfIndex(inst);7542 const dst_ty = self.air.typeOfIndex(inst);
7694 try self.setRegOrMem(dst_ty, dst_mcv, src_mcv);7543 try self.genCopy(dst_ty, dst_mcv, src_mcv);
7695 break :result dst_mcv;7544 break :result dst_mcv;
7696 };7545 };
7697 return self.finishAir(inst, result, .{ un_op, .none, .none });7546 return self.finishAir(inst, result, .{ un_op, .none, .none });
...@@ -7699,9 +7548,14 @@ fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {...@@ -7699,9 +7548,14 @@ fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
76997548
7700fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {7549fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
7701 const ty_op = self.air.instructions.items(.data)[inst].ty_op;7550 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
7551 const dst_ty = self.air.typeOfIndex(inst);
7552 const src_ty = self.air.typeOf(ty_op.operand);
7553
7702 const result = result: {7554 const result = result: {
7555 const dst_rc = try self.regClassForType(dst_ty);
7556 const src_rc = try self.regClassForType(src_ty);
7703 const operand = try self.resolveInst(ty_op.operand);7557 const operand = try self.resolveInst(ty_op.operand);
7704 if (self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand;7558 if (dst_rc.eql(src_rc) and self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand;
77057559
7706 const operand_lock = switch (operand) {7560 const operand_lock = switch (operand) {
7707 .register => |reg| self.register_manager.lockReg(reg),7561 .register => |reg| self.register_manager.lockReg(reg),
...@@ -7711,25 +7565,31 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -7711,25 +7565,31 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
7711 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);7565 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
77127566
7713 const dest = try self.allocRegOrMem(inst, true);7567 const dest = try self.allocRegOrMem(inst, true);
7714 try self.setRegOrMem(self.air.typeOfIndex(inst), dest, operand);7568 try self.genCopy(self.air.typeOfIndex(inst), dest, operand);
7715 break :result dest;7569 break :result dest;
7716 };7570 };
7717 log.debug("airBitCast(%{d}): {}", .{ inst, result });
7718 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });7571 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7719}7572}
77207573
7721fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {7574fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
7722 const ty_op = self.air.instructions.items(.data)[inst].ty_op;7575 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
7576
7577 const slice_ty = self.air.typeOfIndex(inst);
7723 const ptr_ty = self.air.typeOf(ty_op.operand);7578 const ptr_ty = self.air.typeOf(ty_op.operand);
7724 const ptr = try self.resolveInst(ty_op.operand);7579 const ptr = try self.resolveInst(ty_op.operand);
7725 const array_ty = ptr_ty.childType();7580 const array_ty = ptr_ty.childType();
7726 const array_len = array_ty.arrayLen();7581 const array_len = array_ty.arrayLen();
77277582
7728 const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));7583 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, self.target.*));
7729 try self.genSetStack(ptr_ty, stack_offset, ptr, .{});7584 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
7730 try self.genSetStack(Type.u64, stack_offset - 8, .{ .immediate = array_len }, .{});7585 try self.genSetMem(
7586 .{ .frame = frame_index },
7587 @intCast(i32, ptr_ty.abiSize(self.target.*)),
7588 Type.usize,
7589 .{ .immediate = array_len },
7590 );
77317591
7732 const result = MCValue{ .stack_offset = stack_offset };7592 const result = MCValue{ .load_frame = .{ .index = frame_index } };
7733 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });7593 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7734}7594}
77357595
...@@ -7758,29 +7618,31 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {...@@ -7758,29 +7618,31 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
7758 }7618 }
77597619
7760 // move float src to ST(0)7620 // move float src to ST(0)
7761 const stack_offset = switch (operand) {7621 const frame_addr: FrameAddr = switch (operand) {
7762 .stack_offset, .ptr_stack_offset => |offset| offset,7622 .load_frame => |frame_addr| frame_addr,
7763 else => blk: {7623 else => frame_addr: {
7764 const offset = @intCast(i32, try self.allocMem(7624 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(src_ty, self.target.*));
7765 inst,7625 try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, operand);
7766 src_abi_size,7626 break :frame_addr .{ .index = frame_index };
7767 src_ty.abiAlignment(self.target.*),
7768 ));
7769 try self.genSetStack(src_ty, offset, operand, .{});
7770 break :blk offset;
7771 },7627 },
7772 };7628 };
7773 try self.asmMemory(.fld, Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{7629 try self.asmMemory(
7774 .base = .rbp,7630 .fld,
7775 .disp = -stack_offset,7631 Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
7776 }));7632 .base = .{ .frame = frame_addr.index },
7633 .disp = frame_addr.off,
7634 }),
7635 );
77777636
7778 // convert7637 // convert
7779 const stack_dst = try self.allocRegOrMem(inst, false);7638 const stack_dst = try self.allocRegOrMem(inst, false);
7780 try self.asmMemory(.fisttp, Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{7639 try self.asmMemory(
7781 .base = .rbp,7640 .fisttp,
7782 .disp = -stack_dst.stack_offset,7641 Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
7783 }));7642 .base = .{ .frame = stack_dst.load_frame.index },
7643 .disp = stack_dst.load_frame.off,
7644 }),
7645 );
77847646
7785 return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });7647 return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });
7786}7648}
...@@ -7800,20 +7662,32 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -7800,20 +7662,32 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
78007662
7801 const exp_mcv = try self.resolveInst(extra.expected_value);7663 const exp_mcv = try self.resolveInst(extra.expected_value);
7802 if (val_abi_size > 8) switch (exp_mcv) {7664 if (val_abi_size > 8) switch (exp_mcv) {
7803 .stack_offset => |exp_off| {7665 .load_frame => |frame_addr| {
7804 try self.genSetReg(Type.usize, .rax, .{ .stack_offset = exp_off - 0 });7666 try self.genSetReg(.rax, Type.usize, .{ .load_frame = .{
7805 try self.genSetReg(Type.usize, .rdx, .{ .stack_offset = exp_off - 8 });7667 .index = frame_addr.index,
7668 .off = frame_addr.off + 0,
7669 } });
7670 try self.genSetReg(.rdx, Type.usize, .{ .load_frame = .{
7671 .index = frame_addr.index,
7672 .off = frame_addr.off + 8,
7673 } });
7806 },7674 },
7807 else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),7675 else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
7808 } else try self.genSetReg(val_ty, .rax, exp_mcv);7676 } else try self.genSetReg(.rax, val_ty, exp_mcv);
7809 const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);7677 const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
7810 defer self.register_manager.unlockReg(rax_lock);7678 defer self.register_manager.unlockReg(rax_lock);
78117679
7812 const new_mcv = try self.resolveInst(extra.new_value);7680 const new_mcv = try self.resolveInst(extra.new_value);
7813 const new_reg: Register = if (val_abi_size > 8) switch (new_mcv) {7681 const new_reg: Register = if (val_abi_size > 8) switch (new_mcv) {
7814 .stack_offset => |new_off| new: {7682 .load_frame => |frame_addr| new: {
7815 try self.genSetReg(Type.usize, .rbx, .{ .stack_offset = new_off - 0 });7683 try self.genSetReg(.rbx, Type.usize, .{ .load_frame = .{
7816 try self.genSetReg(Type.usize, .rcx, .{ .stack_offset = new_off - 8 });7684 .index = frame_addr.index,
7685 .off = frame_addr.off + 0,
7686 } });
7687 try self.genSetReg(.rcx, Type.usize, .{ .load_frame = .{
7688 .index = frame_addr.index,
7689 .off = frame_addr.off + 8,
7690 } });
7817 break :new undefined;7691 break :new undefined;
7818 },7692 },
7819 else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),7693 else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
...@@ -7823,11 +7697,19 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -7823,11 +7697,19 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
78237697
7824 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);7698 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
7825 const ptr_mem = switch (ptr_mcv) {7699 const ptr_mem = switch (ptr_mcv) {
7826 .register => |reg| Memory.sib(ptr_size, .{ .base = reg }),7700 .register => |reg| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),
7827 .ptr_stack_offset => |off| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),7701 .lea_frame => |frame_addr| Memory.sib(ptr_size, .{
7828 else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),7702 .base = .{ .frame = frame_addr.index },
7703 .disp = frame_addr.off,
7704 }),
7705 else => Memory.sib(ptr_size, .{ .base = .{
7706 .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
7707 } }),
7708 };
7709 const mem_lock = switch (ptr_mem.base()) {
7710 .none, .frame => null,
7711 .reg => |reg| self.register_manager.lockReg(reg),
7829 };7712 };
7830 const mem_lock = if (ptr_mem.base()) |reg| self.register_manager.lockReg(reg) else null;
7831 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);7713 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
78327714
7833 try self.spillEflagsIfOccupied();7715 try self.spillEflagsIfOccupied();
...@@ -7851,9 +7733,24 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -7851,9 +7733,24 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
7851 }7733 }
78527734
7853 const dst_mcv = try self.allocRegOrMem(inst, false);7735 const dst_mcv = try self.allocRegOrMem(inst, false);
7854 try self.genSetStack(Type.bool, dst_mcv.stack_offset - 16, .{ .eflags = .ne }, .{});7736 try self.genSetMem(
7855 try self.genSetStack(Type.usize, dst_mcv.stack_offset - 8, .{ .register = .rdx }, .{});7737 .{ .frame = dst_mcv.load_frame.index },
7856 try self.genSetStack(Type.usize, dst_mcv.stack_offset - 0, .{ .register = .rax }, .{});7738 dst_mcv.load_frame.off + 16,
7739 Type.bool,
7740 .{ .eflags = .ne },
7741 );
7742 try self.genSetMem(
7743 .{ .frame = dst_mcv.load_frame.index },
7744 dst_mcv.load_frame.off + 8,
7745 Type.usize,
7746 .{ .register = .rdx },
7747 );
7748 try self.genSetMem(
7749 .{ .frame = dst_mcv.load_frame.index },
7750 dst_mcv.load_frame.off + 0,
7751 Type.usize,
7752 .{ .register = .rax },
7753 );
7857 break :result dst_mcv;7754 break :result dst_mcv;
7858 };7755 };
7859 return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });7756 return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
...@@ -7884,11 +7781,19 @@ fn atomicOp(...@@ -7884,11 +7781,19 @@ fn atomicOp(
7884 const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));7781 const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
7885 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);7782 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
7886 const ptr_mem = switch (ptr_mcv) {7783 const ptr_mem = switch (ptr_mcv) {
7887 .register => |reg| Memory.sib(ptr_size, .{ .base = reg }),7784 .register => |reg| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),
7888 .ptr_stack_offset => |off| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),7785 .lea_frame => |frame_addr| Memory.sib(ptr_size, .{
7889 else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),7786 .base = .{ .frame = frame_addr.index },
7787 .disp = frame_addr.off,
7788 }),
7789 else => Memory.sib(ptr_size, .{ .base = .{
7790 .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
7791 } }),
7792 };
7793 const mem_lock = switch (ptr_mem.base()) {
7794 .none, .frame => null,
7795 .reg => |reg| self.register_manager.lockReg(reg),
7890 };7796 };
7891 const mem_lock = if (ptr_mem.base()) |reg| self.register_manager.lockReg(reg) else null;
7892 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);7797 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
78937798
7894 const method: enum { lock, loop, libcall } = if (val_ty.isRuntimeFloat())7799 const method: enum { lock, loop, libcall } = if (val_ty.isRuntimeFloat())
...@@ -7928,7 +7833,7 @@ fn atomicOp(...@@ -7928,7 +7833,7 @@ fn atomicOp(
7928 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);7833 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
7929 defer self.register_manager.unlockReg(dst_lock);7834 defer self.register_manager.unlockReg(dst_lock);
79307835
7931 try self.genSetReg(val_ty, dst_reg, val_mcv);7836 try self.genSetReg(dst_reg, val_ty, val_mcv);
7932 if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {7837 if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
7933 try self.genUnOpMir(.neg, val_ty, dst_mcv);7838 try self.genUnOpMir(.neg, val_ty, dst_mcv);
7934 }7839 }
...@@ -7952,10 +7857,10 @@ fn atomicOp(...@@ -7952,10 +7857,10 @@ fn atomicOp(
7952 try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);7857 try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);
7953 const loop = @intCast(u32, self.mir_instructions.len);7858 const loop = @intCast(u32, self.mir_instructions.len);
7954 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {7859 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
7955 try self.genSetReg(val_ty, tmp_reg, .{ .register = .rax });7860 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
7956 }7861 }
7957 if (rmw_op) |op| switch (op) {7862 if (rmw_op) |op| switch (op) {
7958 .Xchg => try self.genSetReg(val_ty, tmp_reg, val_mcv),7863 .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
7959 .Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),7864 .Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),
7960 .Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),7865 .Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),
7961 .And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),7866 .And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),
...@@ -7990,12 +7895,12 @@ fn atomicOp(...@@ -7990,12 +7895,12 @@ fn atomicOp(
7990 registerAlias(val_reg, cmov_abi_size),7895 registerAlias(val_reg, cmov_abi_size),
7991 cc,7896 cc,
7992 ),7897 ),
7993 .stack_offset => |val_off| try self.asmCmovccRegisterMemory(7898 .load_frame => |frame_addr| try self.asmCmovccRegisterMemory(
7994 registerAlias(tmp_reg, cmov_abi_size),7899 registerAlias(tmp_reg, cmov_abi_size),
7995 Memory.sib(7900 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{
7996 Memory.PtrSize.fromSize(cmov_abi_size),7901 .base = .{ .frame = frame_addr.index },
7997 .{ .base = .rbp, .disp = -val_off },7902 .disp = frame_addr.off,
7998 ),7903 }),
7999 cc,7904 cc,
8000 ),7905 ),
8001 else => {7906 else => {
...@@ -8028,9 +7933,15 @@ fn atomicOp(...@@ -8028,9 +7933,15 @@ fn atomicOp(
8028 }));7933 }));
8029 const loop = @intCast(u32, self.mir_instructions.len);7934 const loop = @intCast(u32, self.mir_instructions.len);
8030 switch (val_mcv) {7935 switch (val_mcv) {
8031 .stack_offset => |val_off| {7936 .load_frame => |frame_addr| {
8032 const val_lo_mem = Memory.sib(.qword, .{ .base = .rbp, .disp = 0 - val_off });7937 const val_lo_mem = Memory.sib(.qword, .{
8033 const val_hi_mem = Memory.sib(.qword, .{ .base = .rbp, .disp = 8 - val_off });7938 .base = .{ .frame = frame_addr.index },
7939 .disp = frame_addr.off + 0,
7940 });
7941 const val_hi_mem = Memory.sib(.qword, .{
7942 .base = .{ .frame = frame_addr.index },
7943 .disp = frame_addr.off + 8,
7944 });
80347945
8035 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {7946 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
8036 try self.asmRegisterRegister(.mov, .rbx, .rax);7947 try self.asmRegisterRegister(.mov, .rbx, .rax);
...@@ -8087,12 +7998,18 @@ fn atomicOp(...@@ -8087,12 +7998,18 @@ fn atomicOp(
8087 const dst_mcv = try self.allocTempRegOrMem(val_ty, false);7998 const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
8088 try self.asmMemoryRegister(7999 try self.asmMemoryRegister(
8089 .mov,8000 .mov,
8090 Memory.sib(.qword, .{ .base = .rbp, .disp = 0 - dst_mcv.stack_offset }),8001 Memory.sib(.qword, .{
8002 .base = .{ .frame = dst_mcv.load_frame.index },
8003 .disp = dst_mcv.load_frame.off + 0,
8004 }),
8091 .rax,8005 .rax,
8092 );8006 );
8093 try self.asmMemoryRegister(8007 try self.asmMemoryRegister(
8094 .mov,8008 .mov,
8095 Memory.sib(.qword, .{ .base = .rbp, .disp = 8 - dst_mcv.stack_offset }),8009 Memory.sib(.qword, .{
8010 .base = .{ .frame = dst_mcv.load_frame.index },
8011 .disp = dst_mcv.load_frame.off + 8,
8012 }),
8096 .rdx,8013 .rdx,
8097 );8014 );
8098 return dst_mcv;8015 return dst_mcv;
...@@ -8139,7 +8056,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -8139,7 +8056,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
8139 else8056 else
8140 try self.allocRegOrMem(inst, true);8057 try self.allocRegOrMem(inst, true);
81418058
8142 try self.load(dst_mcv, ptr_mcv, ptr_ty);8059 try self.load(dst_mcv, ptr_ty, ptr_mcv);
8143 return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });8060 return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });
8144}8061}
81458062
...@@ -8184,16 +8101,16 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -8184,16 +8101,16 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8184 const elem_abi_size = @intCast(u31, elem_ty.abiSize(self.target.*));8101 const elem_abi_size = @intCast(u31, elem_ty.abiSize(self.target.*));
81858102
8186 if (elem_abi_size == 1) {8103 if (elem_abi_size == 1) {
8187 const ptr = switch (dst_ptr_ty.ptrSize()) {8104 const ptr: MCValue = switch (dst_ptr_ty.ptrSize()) {
8188 // TODO: this only handles slices stored in the stack8105 // TODO: this only handles slices stored in the stack
8189 .Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 0 }),8106 .Slice => dst_ptr,
8190 .One => dst_ptr,8107 .One => dst_ptr,
8191 .C, .Many => unreachable,8108 .C, .Many => unreachable,
8192 };8109 };
8193 const len = switch (dst_ptr_ty.ptrSize()) {8110 const len: MCValue = switch (dst_ptr_ty.ptrSize()) {
8194 // TODO: this only handles slices stored in the stack8111 // TODO: this only handles slices stored in the stack
8195 .Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),8112 .Slice => dst_ptr.address().offset(8).deref(),
8196 .One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),8113 .One => .{ .immediate = dst_ptr_ty.childType().arrayLen() },
8197 .C, .Many => unreachable,8114 .C, .Many => unreachable,
8198 };8115 };
8199 const len_lock: ?RegisterLock = switch (len) {8116 const len_lock: ?RegisterLock = switch (len) {
...@@ -8202,7 +8119,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -8202,7 +8119,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8202 };8119 };
8203 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);8120 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
82048121
8205 try self.genInlineMemset(ptr, src_val, len, .{});8122 try self.genInlineMemset(ptr, src_val, len);
8206 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });8123 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
8207 }8124 }
82088125
...@@ -8215,8 +8132,8 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -8215,8 +8132,8 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8215 const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(&buf);8132 const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(&buf);
82168133
8217 // TODO: this only handles slices stored in the stack8134 // TODO: this only handles slices stored in the stack
8218 const ptr = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 0 });8135 const ptr = dst_ptr;
8219 const len = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 });8136 const len = dst_ptr.address().offset(8).deref();
82208137
8221 // Used to store the number of elements for comparison.8138 // Used to store the number of elements for comparison.
8222 // After comparison, updated to store number of bytes needed to copy.8139 // After comparison, updated to store number of bytes needed to copy.
...@@ -8225,55 +8142,51 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -8225,55 +8142,51 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8225 const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);8142 const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
8226 defer self.register_manager.unlockReg(len_lock);8143 defer self.register_manager.unlockReg(len_lock);
82278144
8228 try self.asmRegisterMemory(.mov, len_reg, Memory.sib(.qword, .{8145 try self.genSetReg(len_reg, Type.usize, len);
8229 .base = .rbp,
8230 .disp = -len.stack_offset,
8231 }));
82328146
8233 const skip_reloc = try self.asmJccReloc(undefined, .z);8147 const skip_reloc = try self.asmJccReloc(undefined, .z);
8234 try self.store(ptr, src_val, slice_ptr_ty, elem_ty);8148 try self.store(slice_ptr_ty, ptr, src_val);
82358149
8236 const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);8150 const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
8237 const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };8151 const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
8238 const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);8152 const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
8239 defer self.register_manager.unlockReg(second_elem_ptr_lock);8153 defer self.register_manager.unlockReg(second_elem_ptr_lock);
82408154
8241 try self.asmRegisterMemory(8155 try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
8242 .lea,8156 .reg = try self.copyToTmpRegister(Type.usize, ptr),
8243 second_elem_ptr_reg,8157 .off = elem_abi_size,
8244 Memory.sib(.qword, .{8158 } });
8245 .base = try self.copyToTmpRegister(Type.usize, ptr),
8246 .disp = elem_abi_size,
8247 }),
8248 );
82498159
8250 try self.genBinOpMir(.sub, Type.usize, len_mcv, .{ .immediate = 1 });8160 try self.genBinOpMir(.sub, Type.usize, len_mcv, .{ .immediate = 1 });
8251 try self.asmRegisterRegisterImmediate(.imul, len_reg, len_reg, Immediate.u(elem_abi_size));8161 try self.asmRegisterRegisterImmediate(.imul, len_reg, len_reg, Immediate.u(elem_abi_size));
8252 try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv, .{});8162 try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);
82538163
8254 try self.performReloc(skip_reloc);8164 try self.performReloc(skip_reloc);
8255 },8165 },
8256 .One => {8166 .One => {
8167 var elem_ptr_pl = Type.Payload.ElemType{
8168 .base = .{ .tag = .single_mut_pointer },
8169 .data = elem_ty,
8170 };
8171 const elem_ptr_ty = Type.initPayload(&elem_ptr_pl.base);
8172
8257 const len = dst_ptr_ty.childType().arrayLen();8173 const len = dst_ptr_ty.childType().arrayLen();
8174
8258 assert(len != 0); // prevented by Sema8175 assert(len != 0); // prevented by Sema
8259 try self.store(dst_ptr, src_val, dst_ptr_ty, elem_ty);8176 try self.store(elem_ptr_ty, dst_ptr, src_val);
82608177
8261 const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);8178 const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
8262 const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };8179 const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
8263 const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);8180 const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
8264 defer self.register_manager.unlockReg(second_elem_ptr_lock);8181 defer self.register_manager.unlockReg(second_elem_ptr_lock);
82658182
8266 try self.asmRegisterMemory(8183 try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
8267 .lea,8184 .reg = try self.copyToTmpRegister(Type.usize, dst_ptr),
8268 second_elem_ptr_reg,8185 .off = elem_abi_size,
8269 Memory.sib(.qword, .{8186 } });
8270 .base = try self.copyToTmpRegister(Type.usize, dst_ptr),
8271 .disp = elem_abi_size,
8272 }),
8273 );
82748187
8275 const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };8188 const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
8276 try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy, .{});8189 try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy);
8277 },8190 },
8278 .C, .Many => unreachable,8191 .C, .Many => unreachable,
8279 }8192 }
...@@ -8299,9 +8212,9 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -8299,9 +8212,9 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
8299 };8212 };
8300 defer if (src_ptr_lock) |lock| self.register_manager.unlockReg(lock);8213 defer if (src_ptr_lock) |lock| self.register_manager.unlockReg(lock);
83018214
8302 const len = switch (dst_ptr_ty.ptrSize()) {8215 const len: MCValue = switch (dst_ptr_ty.ptrSize()) {
8303 .Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),8216 .Slice => dst_ptr.address().offset(8).deref(),
8304 .One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),8217 .One => .{ .immediate = dst_ptr_ty.childType().arrayLen() },
8305 .C, .Many => unreachable,8218 .C, .Many => unreachable,
8306 };8219 };
8307 const len_lock: ?RegisterLock = switch (len) {8220 const len_lock: ?RegisterLock = switch (len) {
...@@ -8311,7 +8224,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -8311,7 +8224,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
8311 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);8224 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
83128225
8313 // TODO: dst_ptr and src_ptr could be slices rather than raw pointers8226 // TODO: dst_ptr and src_ptr could be slices rather than raw pointers
8314 try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{});8227 try self.genInlineMemcpy(dst_ptr, src_ptr, len);
83158228
8316 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });8229 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
8317}8230}
...@@ -8345,24 +8258,25 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {...@@ -8345,24 +8258,25 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8345 const atom = elf_file.getAtom(atom_index);8258 const atom = elf_file.getAtom(atom_index);
8346 _ = try atom.getOrCreateOffsetTableEntry(elf_file);8259 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
8347 const got_addr = atom.getOffsetTableAddress(elf_file);8260 const got_addr = atom.getOffsetTableAddress(elf_file);
8348 try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{8261 try self.asmRegisterMemory(
8349 .base = .ds,8262 .mov,
8350 .disp = @intCast(i32, got_addr),8263 addr_reg.to64(),
8351 }));8264 Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
8265 );
8352 } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {8266 } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
8353 const atom_index = try coff_file.getOrCreateAtomForLazySymbol(8267 const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
8354 .{ .kind = .const_data, .ty = Type.anyerror },8268 .{ .kind = .const_data, .ty = Type.anyerror },
8355 4, // dword alignment8269 4, // dword alignment
8356 );8270 );
8357 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;8271 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
8358 try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });8272 try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
8359 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {8273 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
8360 const atom_index = try macho_file.getOrCreateAtomForLazySymbol(8274 const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
8361 .{ .kind = .const_data, .ty = Type.anyerror },8275 .{ .kind = .const_data, .ty = Type.anyerror },
8362 4, // dword alignment8276 4, // dword alignment
8363 );8277 );
8364 const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;8278 const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
8365 try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });8279 try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
8366 } else {8280 } else {
8367 return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});8281 return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
8368 }8282 }
...@@ -8377,36 +8291,60 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {...@@ -8377,36 +8291,60 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
83778291
8378 try self.truncateRegister(err_ty, err_reg.to32());8292 try self.truncateRegister(err_ty, err_reg.to32());
83798293
8380 try self.asmRegisterMemory(.mov, start_reg.to32(), Memory.sib(.dword, .{8294 try self.asmRegisterMemory(
8381 .base = addr_reg.to64(),8295 .mov,
8382 .scale_index = .{ .scale = 4, .index = err_reg.to64() },8296 start_reg.to32(),
8383 .disp = 4,8297 Memory.sib(.dword, .{
8384 }));8298 .base = .{ .reg = addr_reg.to64() },
8385 try self.asmRegisterMemory(.mov, end_reg.to32(), Memory.sib(.dword, .{8299 .scale_index = .{ .scale = 4, .index = err_reg.to64() },
8386 .base = addr_reg.to64(),8300 .disp = 4,
8387 .scale_index = .{ .scale = 4, .index = err_reg.to64() },8301 }),
8388 .disp = 8,8302 );
8389 }));8303 try self.asmRegisterMemory(
8304 .mov,
8305 end_reg.to32(),
8306 Memory.sib(.dword, .{
8307 .base = .{ .reg = addr_reg.to64() },
8308 .scale_index = .{ .scale = 4, .index = err_reg.to64() },
8309 .disp = 8,
8310 }),
8311 );
8390 try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());8312 try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());
8391 try self.asmRegisterMemory(.lea, start_reg.to64(), Memory.sib(.byte, .{8313 try self.asmRegisterMemory(
8392 .base = addr_reg.to64(),8314 .lea,
8393 .scale_index = .{ .scale = 1, .index = start_reg.to64() },8315 start_reg.to64(),
8394 .disp = 0,8316 Memory.sib(.byte, .{
8395 }));8317 .base = .{ .reg = addr_reg.to64() },
8396 try self.asmRegisterMemory(.lea, end_reg.to32(), Memory.sib(.byte, .{8318 .scale_index = .{ .scale = 1, .index = start_reg.to64() },
8397 .base = end_reg.to64(),8319 .disp = 0,
8398 .disp = -1,8320 }),
8399 }));8321 );
8322 try self.asmRegisterMemory(
8323 .lea,
8324 end_reg.to32(),
8325 Memory.sib(.byte, .{
8326 .base = .{ .reg = end_reg.to64() },
8327 .disp = -1,
8328 }),
8329 );
84008330
8401 const dst_mcv = try self.allocRegOrMem(inst, false);8331 const dst_mcv = try self.allocRegOrMem(inst, false);
8402 try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{8332 try self.asmMemoryRegister(
8403 .base = .rbp,8333 .mov,
8404 .disp = 0 - dst_mcv.stack_offset,8334 Memory.sib(.qword, .{
8405 }), start_reg.to64());8335 .base = .{ .frame = dst_mcv.load_frame.index },
8406 try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{8336 .disp = dst_mcv.load_frame.off,
8407 .base = .rbp,8337 }),
8408 .disp = 8 - dst_mcv.stack_offset,8338 start_reg.to64(),
8409 }), end_reg.to64());8339 );
8340 try self.asmMemoryRegister(
8341 .mov,
8342 Memory.sib(.qword, .{
8343 .base = .{ .frame = dst_mcv.load_frame.index },
8344 .disp = dst_mcv.load_frame.off + 8,
8345 }),
8346 end_reg.to64(),
8347 );
84108348
8411 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });8349 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
8412}8350}
...@@ -8445,19 +8383,17 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8445,19 +8383,17 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8445 const len = @intCast(usize, result_ty.arrayLen());8383 const len = @intCast(usize, result_ty.arrayLen());
8446 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;8384 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
8447 const elements = @ptrCast([]const Air.Inst.Ref, self.air.extra[ty_pl.payload..][0..len]);8385 const elements = @ptrCast([]const Air.Inst.Ref, self.air.extra[ty_pl.payload..][0..len]);
8448 const abi_size = @intCast(u32, result_ty.abiSize(self.target.*));
8449 const abi_align = result_ty.abiAlignment(self.target.*);
8450 const result: MCValue = result: {8386 const result: MCValue = result: {
8451 switch (result_ty.zigTypeTag()) {8387 switch (result_ty.zigTypeTag()) {
8452 .Struct => {8388 .Struct => {
8453 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));8389 const frame_index =
8390 try self.allocFrameIndex(FrameAlloc.initType(result_ty, self.target.*));
8454 if (result_ty.containerLayout() == .Packed) {8391 if (result_ty.containerLayout() == .Packed) {
8455 const struct_obj = result_ty.castTag(.@"struct").?.data;8392 const struct_obj = result_ty.castTag(.@"struct").?.data;
8456 try self.genInlineMemset(8393 try self.genInlineMemset(
8457 .{ .ptr_stack_offset = stack_offset },8394 .{ .lea_frame = .{ .index = frame_index } },
8458 .{ .immediate = 0 },8395 .{ .immediate = 0 },
8459 .{ .immediate = abi_size },8396 .{ .immediate = result_ty.abiSize(self.target.*) },
8460 .{},
8461 );8397 );
8462 for (elements, 0..) |elem, elem_i| {8398 for (elements, 0..) |elem, elem_i| {
8463 if (result_ty.structFieldValueComptime(elem_i) != null) continue;8399 if (result_ty.structFieldValueComptime(elem_i) != null) continue;
...@@ -8465,7 +8401,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8465,7 +8401,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8465 const elem_ty = result_ty.structFieldType(elem_i);8401 const elem_ty = result_ty.structFieldType(elem_i);
8466 const elem_bit_size = @intCast(u32, elem_ty.bitSize(self.target.*));8402 const elem_bit_size = @intCast(u32, elem_ty.bitSize(self.target.*));
8467 if (elem_bit_size > 64) {8403 if (elem_bit_size > 64) {
8468 return self.fail("TODO airAggregateInit implement packed structs with large fields", .{});8404 return self.fail(
8405 "TODO airAggregateInit implement packed structs with large fields",
8406 .{},
8407 );
8469 }8408 }
8470 const elem_abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));8409 const elem_abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
8471 const elem_abi_bits = elem_abi_size * 8;8410 const elem_abi_bits = elem_abi_size * 8;
...@@ -8503,7 +8442,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8503,7 +8442,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8503 try self.genBinOpMir(8442 try self.genBinOpMir(
8504 .@"or",8443 .@"or",
8505 elem_ty,8444 elem_ty,
8506 .{ .stack_offset = stack_offset - elem_byte_off },8445 .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
8507 .{ .register = elem_reg },8446 .{ .register = elem_reg },
8508 );8447 );
8509 if (elem_bit_off > elem_extra_bits) {8448 if (elem_bit_off > elem_extra_bits) {
...@@ -8520,8 +8459,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8520,8 +8459,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8520 try self.genBinOpMir(8459 try self.genBinOpMir(
8521 .@"or",8460 .@"or",
8522 elem_ty,8461 elem_ty,
8523 .{ .stack_offset = stack_offset - elem_byte_off -8462 .{ .load_frame = .{
8524 @intCast(i32, elem_abi_size) },8463 .index = frame_index,
8464 .off = elem_byte_off + @intCast(i32, elem_abi_size),
8465 } },
8525 .{ .register = reg },8466 .{ .register = reg },
8526 );8467 );
8527 }8468 }
...@@ -8536,12 +8477,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8536,12 +8477,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8536 .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index },8477 .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index },
8537 else => elem_mcv,8478 else => elem_mcv,
8538 };8479 };
8539 try self.genSetStack(elem_ty, stack_offset - elem_off, mat_elem_mcv, .{});8480 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
8540 }8481 }
8541 break :result .{ .stack_offset = stack_offset };8482 break :result .{ .load_frame = .{ .index = frame_index } };
8542 },8483 },
8543 .Array => {8484 .Array => {
8544 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));8485 const frame_index =
8486 try self.allocFrameIndex(FrameAlloc.initType(result_ty, self.target.*));
8545 const elem_ty = result_ty.childType();8487 const elem_ty = result_ty.childType();
8546 const elem_size = @intCast(u32, elem_ty.abiSize(self.target.*));8488 const elem_size = @intCast(u32, elem_ty.abiSize(self.target.*));
85478489
...@@ -8552,9 +8494,9 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8552,9 +8494,9 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8552 else => elem_mcv,8494 else => elem_mcv,
8553 };8495 };
8554 const elem_off = @intCast(i32, elem_size * elem_i);8496 const elem_off = @intCast(i32, elem_size * elem_i);
8555 try self.genSetStack(elem_ty, stack_offset - elem_off, mat_elem_mcv, .{});8497 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
8556 }8498 }
8557 break :result MCValue{ .stack_offset = stack_offset };8499 break :result .{ .load_frame = .{ .index = frame_index } };
8558 },8500 },
8559 .Vector => return self.fail("TODO implement aggregate_init for vectors", .{}),8501 .Vector => return self.fail("TODO implement aggregate_init for vectors", .{}),
8560 else => unreachable,8502 else => unreachable,
...@@ -8596,7 +8538,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {...@@ -8596,7 +8538,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
8596 const ty = self.air.typeOf(ref);8538 const ty = self.air.typeOf(ref);
85978539
8598 // If the type has no codegen bits, no need to store it.8540 // If the type has no codegen bits, no need to store it.
8599 if (!ty.hasRuntimeBitsIgnoreComptime() and !ty.isError()) return .none;8541 if (!ty.hasRuntimeBitsIgnoreComptime()) return .none;
86008542
8601 if (Air.refToIndex(ref)) |inst| {8543 if (Air.refToIndex(ref)) |inst| {
8602 const mcv = switch (self.air.instructions.items(.tag)[inst]) {8544 const mcv = switch (self.air.instructions.items(.tag)[inst]) {
...@@ -8666,8 +8608,8 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {...@@ -8666,8 +8608,8 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
8666 .immediate => |imm| .{ .immediate = imm },8608 .immediate => |imm| .{ .immediate = imm },
8667 .memory => |addr| .{ .memory = addr },8609 .memory => |addr| .{ .memory = addr },
8668 .load_direct => |sym_index| .{ .load_direct = sym_index },8610 .load_direct => |sym_index| .{ .load_direct = sym_index },
8669 .load_got => |sym_index| .{ .load_got = sym_index },8611 .load_got => |sym_index| .{ .lea_got = sym_index },
8670 .load_tlv => |sym_index| .{ .load_tlv = sym_index },8612 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
8671 },8613 },
8672 .fail => |msg| {8614 .fail => |msg| {
8673 self.err_msg = msg;8615 self.err_msg = msg;
...@@ -8679,9 +8621,9 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {...@@ -8679,9 +8621,9 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
86798621
8680const CallMCValues = struct {8622const CallMCValues = struct {
8681 args: []MCValue,8623 args: []MCValue,
8682 return_value: MCValue,8624 return_value: InstTracking,
8683 stack_byte_count: u32,8625 stack_byte_count: u31,
8684 stack_align: u32,8626 stack_align: u31,
86858627
8686 fn deinit(self: *CallMCValues, func: *Self) void {8628 fn deinit(self: *CallMCValues, func: *Self) void {
8687 func.gpa.free(self.args);8629 func.gpa.free(self.args);
...@@ -8694,6 +8636,7 @@ fn resolveCallingConventionValues(...@@ -8694,6 +8636,7 @@ fn resolveCallingConventionValues(
8694 self: *Self,8636 self: *Self,
8695 fn_ty: Type,8637 fn_ty: Type,
8696 var_args: []const Air.Inst.Ref,8638 var_args: []const Air.Inst.Ref,
8639 stack_frame_base: FrameIndex,
8697) !CallMCValues {8640) !CallMCValues {
8698 const cc = fn_ty.fnCallingConvention();8641 const cc = fn_ty.fnCallingConvention();
8699 const param_len = fn_ty.fnParamLen();8642 const param_len = fn_ty.fnParamLen();
...@@ -8706,7 +8649,7 @@ fn resolveCallingConventionValues(...@@ -8706,7 +8649,7 @@ fn resolveCallingConventionValues(
8706 .args = try self.gpa.alloc(MCValue, param_types.len),8649 .args = try self.gpa.alloc(MCValue, param_types.len),
8707 // These undefined values must be populated before returning from this function.8650 // These undefined values must be populated before returning from this function.
8708 .return_value = undefined,8651 .return_value = undefined,
8709 .stack_byte_count = undefined,8652 .stack_byte_count = 0,
8710 .stack_align = undefined,8653 .stack_align = undefined,
8711 };8654 };
8712 errdefer self.gpa.free(result.args);8655 errdefer self.gpa.free(result.args);
...@@ -8716,40 +8659,46 @@ fn resolveCallingConventionValues(...@@ -8716,40 +8659,46 @@ fn resolveCallingConventionValues(
8716 switch (cc) {8659 switch (cc) {
8717 .Naked => {8660 .Naked => {
8718 assert(result.args.len == 0);8661 assert(result.args.len == 0);
8719 result.return_value = .unreach;8662 result.return_value = InstTracking.init(.unreach);
8720 result.stack_byte_count = 0;8663 result.stack_align = 8;
8721 result.stack_align = 1;
8722 return result;
8723 },8664 },
8724 .C => {8665 .C => {
8666 var param_reg_i: usize = 0;
8667 result.stack_align = 16;
8668
8669 switch (self.target.os.tag) {
8670 .windows => {
8671 // Align the stack to 16bytes before allocating shadow stack space (if any).
8672 result.stack_byte_count += @intCast(u31, 4 * Type.usize.abiSize(self.target.*));
8673 },
8674 else => {},
8675 }
8676
8725 // Return values8677 // Return values
8726 if (ret_ty.zigTypeTag() == .NoReturn) {8678 if (ret_ty.zigTypeTag() == .NoReturn) {
8727 result.return_value = .unreach;8679 result.return_value = InstTracking.init(.unreach);
8728 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {8680 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime()) {
8729 // TODO: is this even possible for C calling convention?8681 // TODO: is this even possible for C calling convention?
8730 result.return_value = .none;8682 result.return_value = InstTracking.init(.none);
8731 } else {8683 } else {
8732 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));8684 const ret_reg = abi.getCAbiIntReturnRegs(self.target.*)[0];
8733 if (ret_ty_size == 0) {8685 const ret_ty_size = @intCast(u31, ret_ty.abiSize(self.target.*));
8734 assert(ret_ty.isError());8686 if (ret_ty_size <= 8) {
8735 result.return_value = .{ .immediate = 0 };8687 const aliased_reg = registerAlias(ret_reg, ret_ty_size);
8736 } else if (ret_ty_size <= 8) {8688 result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
8737 const aliased_reg = registerAlias(abi.getCAbiIntReturnRegs(self.target.*)[0], ret_ty_size);
8738 result.return_value = .{ .register = aliased_reg };
8739 } else {8689 } else {
8740 // TODO: return argument cell should go first8690 const ret_indirect_reg = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i];
8741 result.return_value = .{ .stack_offset = 0 };8691 param_reg_i += 1;
8692 result.return_value = .{
8693 .short = .{ .indirect = .{ .reg = ret_reg } },
8694 .long = .{ .indirect = .{ .reg = ret_indirect_reg } },
8695 };
8742 }8696 }
8743 }8697 }
87448698
8745 // Input params8699 // Input params
8746 var next_stack_offset: u32 = switch (result.return_value) {8700 for (param_types, result.args) |ty, *arg| {
8747 .stack_offset => |off| @intCast(u32, off),8701 assert(ty.hasRuntimeBitsIgnoreComptime());
8748 else => 0,
8749 };
8750
8751 for (param_types, result.args, 0..) |ty, *arg, i| {
8752 assert(ty.hasRuntimeBits());
87538702
8754 const classes: []const abi.Class = switch (self.target.os.tag) {8703 const classes: []const abi.Class = switch (self.target.os.tag) {
8755 .windows => &[1]abi.Class{abi.classifyWindows(ty, self.target.*)},8704 .windows => &[1]abi.Class{abi.classifyWindows(ty, self.target.*)},
...@@ -8760,8 +8709,10 @@ fn resolveCallingConventionValues(...@@ -8760,8 +8709,10 @@ fn resolveCallingConventionValues(
8760 }8709 }
8761 switch (classes[0]) {8710 switch (classes[0]) {
8762 .integer => blk: {8711 .integer => blk: {
8763 if (i >= abi.getCAbiIntParamRegs(self.target.*).len) break :blk; // fallthrough8712 if (param_reg_i >= abi.getCAbiIntParamRegs(self.target.*).len) break :blk;
8764 arg.* = .{ .register = abi.getCAbiIntParamRegs(self.target.*)[i] };8713 const param_reg = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i];
8714 param_reg_i += 1;
8715 arg.* = .{ .register = param_reg };
8765 continue;8716 continue;
8766 },8717 },
8767 .memory => {}, // fallthrough8718 .memory => {}, // fallthrough
...@@ -8770,82 +8721,61 @@ fn resolveCallingConventionValues(...@@ -8770,82 +8721,61 @@ fn resolveCallingConventionValues(
8770 }),8721 }),
8771 }8722 }
87728723
8773 const param_size = @intCast(u32, ty.abiSize(self.target.*));8724 const param_size = @intCast(u31, ty.abiSize(self.target.*));
8774 const param_align = @intCast(u32, ty.abiAlignment(self.target.*));8725 const param_align = @intCast(u31, ty.abiAlignment(self.target.*));
8775 const offset = mem.alignForwardGeneric(u32, next_stack_offset + param_size, param_align);8726 result.stack_byte_count =
8776 arg.* = .{ .stack_offset = @intCast(i32, offset) };8727 mem.alignForwardGeneric(u31, result.stack_byte_count, param_align);
8777 next_stack_offset = offset;8728 arg.* = .{ .load_frame = .{
8778 }8729 .index = stack_frame_base,
87798730 .off = result.stack_byte_count,
8780 // Align the stack to 16bytes before allocating shadow stack space (if any).8731 } };
8781 const aligned_next_stack_offset = mem.alignForwardGeneric(u32, next_stack_offset, 16);8732 result.stack_byte_count += param_size;
8782 const padding = aligned_next_stack_offset - next_stack_offset;
8783 if (padding > 0) {
8784 for (result.args) |*arg| {
8785 if (arg.isRegister()) continue;
8786 arg.stack_offset += @intCast(i32, padding);
8787 }
8788 }8733 }
8789
8790 const shadow_stack_space: u32 = switch (self.target.os.tag) {
8791 .windows => @intCast(u32, 4 * @sizeOf(u64)),
8792 else => 0,
8793 };
8794
8795 // alignment padding | args ... | shadow stack space (if any) | ret addr | $rbp |
8796 result.stack_byte_count = aligned_next_stack_offset + shadow_stack_space;
8797 result.stack_align = 16;
8798 },8734 },
8799 .Unspecified => {8735 .Unspecified => {
8736 result.stack_align = 16;
8737
8800 // Return values8738 // Return values
8801 if (ret_ty.zigTypeTag() == .NoReturn) {8739 if (ret_ty.zigTypeTag() == .NoReturn) {
8802 result.return_value = .unreach;8740 result.return_value = InstTracking.init(.unreach);
8803 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {8741 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime()) {
8804 result.return_value = .none;8742 result.return_value = InstTracking.init(.none);
8805 } else {8743 } else {
8806 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));8744 const ret_reg = abi.getCAbiIntReturnRegs(self.target.*)[0];
8807 if (ret_ty_size == 0) {8745 const ret_ty_size = @intCast(u31, ret_ty.abiSize(self.target.*));
8808 assert(ret_ty.isError());8746 if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {
8809 result.return_value = .{ .immediate = 0 };8747 const aliased_reg = registerAlias(ret_reg, ret_ty_size);
8810 } else if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {8748 result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
8811 const aliased_reg = registerAlias(abi.getCAbiIntReturnRegs(self.target.*)[0], ret_ty_size);
8812 result.return_value = .{ .register = aliased_reg };
8813 } else {8749 } else {
8814 // We simply make the return MCValue a stack offset. However, the actual value8750 const ret_indirect_reg = abi.getCAbiIntParamRegs(self.target.*)[0];
8815 // for the offset will be populated later. We will also push the stack offset8751 result.return_value = .{
8816 // value into an appropriate register when we resolve the offset.8752 .short = .{ .indirect = .{ .reg = ret_reg } },
8817 result.return_value = .{ .stack_offset = 0 };8753 .long = .{ .indirect = .{ .reg = ret_indirect_reg } },
8754 };
8818 }8755 }
8819 }8756 }
88208757
8821 // Input params8758 // Input params
8822 var next_stack_offset: u32 = switch (result.return_value) {
8823 .stack_offset => |off| @intCast(u32, off),
8824 else => 0,
8825 };
8826
8827 for (param_types, result.args) |ty, *arg| {8759 for (param_types, result.args) |ty, *arg| {
8828 if (!ty.hasRuntimeBits()) {8760 if (!ty.hasRuntimeBitsIgnoreComptime()) {
8829 arg.* = .none;8761 arg.* = .none;
8830 continue;8762 continue;
8831 }8763 }
8832 const param_size = @intCast(u32, ty.abiSize(self.target.*));8764 const param_size = @intCast(u31, ty.abiSize(self.target.*));
8833 const param_align = @intCast(u32, ty.abiAlignment(self.target.*));8765 const param_align = @intCast(u31, ty.abiAlignment(self.target.*));
8834 const offset = mem.alignForwardGeneric(u32, next_stack_offset + param_size, param_align);8766 result.stack_byte_count =
8835 arg.* = .{ .stack_offset = @intCast(i32, offset) };8767 mem.alignForwardGeneric(u31, result.stack_byte_count, param_align);
8836 next_stack_offset = offset;8768 arg.* = .{ .load_frame = .{
8769 .index = stack_frame_base,
8770 .off = result.stack_byte_count,
8771 } };
8772 result.stack_byte_count += param_size;
8837 }8773 }
8838
8839 result.stack_align = 16;
8840 // TODO fix this so that the 16byte alignment padding is at the current value of $rsp, and push
8841 // the args onto the stack so that there is no padding between the first argument and
8842 // the standard preamble.
8843 // alignment padding | args ... | ret addr | $rbp |
8844 result.stack_byte_count = mem.alignForwardGeneric(u32, next_stack_offset, result.stack_align);
8845 },8774 },
8846 else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),8775 else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),
8847 }8776 }
88488777
8778 result.stack_byte_count = mem.alignForwardGeneric(u31, result.stack_byte_count, result.stack_align);
8849 return result;8779 return result;
8850}8780}
88518781
...@@ -8953,15 +8883,6 @@ fn regExtraBits(self: *Self, ty: Type) u64 {...@@ -8953,15 +8883,6 @@ fn regExtraBits(self: *Self, ty: Type) u64 {
8953 return self.regBitSize(ty) - ty.bitSize(self.target.*);8883 return self.regBitSize(ty) - ty.bitSize(self.target.*);
8954}8884}
89558885
8956fn intrinsicsAllowed(target: Target, ty: Type) bool {
8957 return switch (ty.tag()) {
8958 .f32,
8959 .f64,
8960 => Target.x86.featureSetHasAny(target.cpu.features, .{ .sse2, .avx, .avx2 }),
8961 else => unreachable, // TODO finish this off
8962 };
8963}
8964
8965fn hasAvxSupport(target: Target) bool {8886fn hasAvxSupport(target: Target) bool {
8966 return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 });8887 return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 });
8967}8888}
src/arch/x86_64/Emit.zig+8-7
...@@ -13,7 +13,9 @@ prev_di_pc: usize,...@@ -13,7 +13,9 @@ prev_di_pc: usize,
13code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},13code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
14relocs: std.ArrayListUnmanaged(Reloc) = .{},14relocs: std.ArrayListUnmanaged(Reloc) = .{},
1515
16pub const Error = Lower.Error || error{EmitFail};16pub const Error = Lower.Error || error{
17 EmitFail,
18};
1719
18pub fn emitMir(emit: *Emit) Error!void {20pub fn emitMir(emit: *Emit) Error!void {
19 for (0..emit.lower.mir.instructions.len) |i| {21 for (0..emit.lower.mir.instructions.len) |i| {
...@@ -22,7 +24,7 @@ pub fn emitMir(emit: *Emit) Error!void {...@@ -22,7 +24,7 @@ pub fn emitMir(emit: *Emit) Error!void {
2224
23 const start_offset = @intCast(u32, emit.code.items.len);25 const start_offset = @intCast(u32, emit.code.items.len);
24 try emit.code_offset_mapping.putNoClobber(emit.lower.allocator, index, start_offset);26 try emit.code_offset_mapping.putNoClobber(emit.lower.allocator, index, start_offset);
25 for (try emit.lower.lowerMir(inst)) |lower_inst| try lower_inst.encode(emit.code.writer());27 for (try emit.lower.lowerMir(inst)) |lower_inst| try lower_inst.encode(emit.code.writer(), .{});
26 const end_offset = @intCast(u32, emit.code.items.len);28 const end_offset = @intCast(u32, emit.code.items.len);
2729
28 switch (inst.tag) {30 switch (inst.tag) {
...@@ -120,11 +122,10 @@ pub fn emitMir(emit: *Emit) Error!void {...@@ -120,11 +122,10 @@ pub fn emitMir(emit: *Emit) Error!void {
120 .length = 6,122 .length = 6,
121 }),123 }),
122124
123 .dbg_line => {125 .dbg_line => try emit.dbgAdvancePCAndLine(
124 const dbg_line_column =126 inst.data.line_column.line,
125 emit.lower.mir.extraData(Mir.DbgLineColumn, inst.data.payload).data;127 inst.data.line_column.column,
126 try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);128 ),
127 },
128129
129 .dbg_prologue_end => {130 .dbg_prologue_end => {
130 switch (emit.debug_output) {131 switch (emit.debug_output) {
src/arch/x86_64/Encoding.zig+30-25
...@@ -47,17 +47,8 @@ pub fn findByMnemonic(...@@ -47,17 +47,8 @@ pub fn findByMnemonic(
47 },47 },
48 else => {},48 else => {},
49 } else false;49 } else false;
50 const rex_extended = for (ops) |op| switch (op) {50 const rex_extended = for (ops) |op| {
51 .reg => |r| if (r.isExtended()) break true,51 if (op.isBaseExtended() or op.isIndexExtended()) break true;
52 .mem => |m| {
53 if (m.base()) |base| {
54 if (base.isExtended()) break true;
55 }
56 if (m.scaleIndex()) |si| {
57 if (si.index.isExtended()) break true;
58 }
59 },
60 else => {},
61 } else false;52 } else false;
6253
63 if ((rex_required or rex_extended) and rex_invalid) return error.CannotEncode;54 if ((rex_required or rex_extended) and rex_invalid) return error.CannotEncode;
...@@ -67,11 +58,11 @@ pub fn findByMnemonic(...@@ -67,11 +58,11 @@ pub fn findByMnemonic(
67 next: for (mnemonic_to_encodings_map[@enumToInt(mnemonic)]) |data| {58 next: for (mnemonic_to_encodings_map[@enumToInt(mnemonic)]) |data| {
68 switch (data.mode) {59 switch (data.mode) {
69 .rex => if (!rex_required) continue,60 .rex => if (!rex_required) continue,
70 .long => {},61 .long, .sse2_long => {},
71 else => if (rex_required) continue,62 else => if (rex_required) continue,
72 }63 }
73 for (input_ops, data.ops) |input_op, data_op|64 for (input_ops, data.ops) |input_op, data_op|
74 if (!input_op.isSubset(data_op, data.mode)) continue :next;65 if (!input_op.isSubset(data_op)) continue :next;
7566
76 const enc = Encoding{ .mnemonic = mnemonic, .data = data };67 const enc = Encoding{ .mnemonic = mnemonic, .data = data };
77 if (shortest_enc) |previous_shortest_enc| {68 if (shortest_enc) |previous_shortest_enc| {
...@@ -99,7 +90,7 @@ pub fn findByOpcode(opc: []const u8, prefixes: struct {...@@ -99,7 +90,7 @@ pub fn findByOpcode(opc: []const u8, prefixes: struct {
99 if (prefixes.rex.w) {90 if (prefixes.rex.w) {
100 switch (data.mode) {91 switch (data.mode) {
101 .short, .fpu, .sse, .sse2, .sse4_1, .none => continue,92 .short, .fpu, .sse, .sse2, .sse4_1, .none => continue,
102 .long, .rex => {},93 .long, .sse2_long, .rex => {},
103 }94 }
104 } else if (prefixes.rex.present and !prefixes.rex.isSet()) {95 } else if (prefixes.rex.present and !prefixes.rex.isSet()) {
105 switch (data.mode) {96 switch (data.mode) {
...@@ -147,7 +138,7 @@ pub fn modRmExt(encoding: Encoding) u3 {...@@ -147,7 +138,7 @@ pub fn modRmExt(encoding: Encoding) u3 {
147pub fn operandBitSize(encoding: Encoding) u64 {138pub fn operandBitSize(encoding: Encoding) u64 {
148 switch (encoding.data.mode) {139 switch (encoding.data.mode) {
149 .short => return 16,140 .short => return 16,
150 .long => return 64,141 .long, .sse2_long => return 64,
151 else => {},142 else => {},
152 }143 }
153 const bit_size: u64 = switch (encoding.data.op_en) {144 const bit_size: u64 = switch (encoding.data.op_en) {
...@@ -172,7 +163,7 @@ pub fn format(...@@ -172,7 +163,7 @@ pub fn format(
172 _ = options;163 _ = options;
173 _ = fmt;164 _ = fmt;
174 switch (encoding.data.mode) {165 switch (encoding.data.mode) {
175 .long => try writer.writeAll("REX.W + "),166 .long, .sse2_long => try writer.writeAll("REX.W + "),
176 else => {},167 else => {},
177 }168 }
178169
...@@ -273,6 +264,8 @@ pub const Mnemonic = enum {...@@ -273,6 +264,8 @@ pub const Mnemonic = enum {
273 @"test", tzcnt,264 @"test", tzcnt,
274 ud2,265 ud2,
275 xadd, xchg, xor,266 xadd, xchg, xor,
267 // MMX
268 movd,
276 // SSE269 // SSE
277 addss,270 addss,
278 cmpss,271 cmpss,
...@@ -287,7 +280,7 @@ pub const Mnemonic = enum {...@@ -287,7 +280,7 @@ pub const Mnemonic = enum {
287 //cmpsd,280 //cmpsd,
288 divsd,281 divsd,
289 maxsd, minsd,282 maxsd, minsd,
290 movq, //movsd,283 movq, //movd, movsd,
291 mulsd,284 mulsd,
292 subsd,285 subsd,
293 ucomisd,286 ucomisd,
...@@ -470,6 +463,17 @@ pub const Op = enum {...@@ -470,6 +463,17 @@ pub const Op = enum {
470 };463 };
471 }464 }
472465
466 pub fn class(op: Op) bits.Register.Class {
467 return switch (op) {
468 else => unreachable,
469 .al, .ax, .eax, .rax, .cl => .general_purpose,
470 .r8, .r16, .r32, .r64 => .general_purpose,
471 .rm8, .rm16, .rm32, .rm64 => .general_purpose,
472 .sreg => .segment,
473 .xmm, .xmm_m32, .xmm_m64 => .floating_point,
474 };
475 }
476
473 pub fn isFloatingPointRegister(op: Op) bool {477 pub fn isFloatingPointRegister(op: Op) bool {
474 return switch (op) {478 return switch (op) {
475 .xmm, .xmm_m32, .xmm_m64 => true,479 .xmm, .xmm_m32, .xmm_m64 => true,
...@@ -478,7 +482,7 @@ pub const Op = enum {...@@ -478,7 +482,7 @@ pub const Op = enum {
478 }482 }
479483
480 /// Given an operand `op` checks if `target` is a subset for the purposes of the encoding.484 /// Given an operand `op` checks if `target` is a subset for the purposes of the encoding.
481 pub fn isSubset(op: Op, target: Op, mode: Mode) bool {485 pub fn isSubset(op: Op, target: Op) bool {
482 switch (op) {486 switch (op) {
483 .m, .o16, .o32, .o64 => unreachable,487 .m, .o16, .o32, .o64 => unreachable,
484 .moffs, .sreg => return op == target,488 .moffs, .sreg => return op == target,
...@@ -488,13 +492,13 @@ pub const Op = enum {...@@ -488,13 +492,13 @@ pub const Op = enum {
488 },492 },
489 else => {493 else => {
490 if (op.isRegister() and target.isRegister()) {494 if (op.isRegister() and target.isRegister()) {
491 switch (mode) {495 return switch (target) {
492 .sse, .sse2, .sse4_1 => return op.isFloatingPointRegister() and target.isFloatingPointRegister(),496 .cl, .al, .ax, .eax, .rax => op == target,
493 else => switch (target) {497 else => op.class() == target.class() and switch (target.class()) {
494 .cl, .al, .ax, .eax, .rax => return op == target,498 .floating_point => true,
495 else => return op.bitSize() == target.bitSize(),499 else => op.bitSize() == target.bitSize(),
496 },500 },
497 }501 };
498 }502 }
499 if (op.isMemory() and target.isMemory()) {503 if (op.isMemory() and target.isMemory()) {
500 switch (target) {504 switch (target) {
...@@ -532,6 +536,7 @@ pub const Mode = enum {...@@ -532,6 +536,7 @@ pub const Mode = enum {
532 long,536 long,
533 sse,537 sse,
534 sse2,538 sse2,
539 sse2_long,
535 sse4_1,540 sse4_1,
536};541};
537542
...@@ -544,7 +549,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op...@@ -544,7 +549,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
544 std.mem.copy(Operand, &inst.ops, ops);549 std.mem.copy(Operand, &inst.ops, ops);
545550
546 var cwriter = std.io.countingWriter(std.io.null_writer);551 var cwriter = std.io.countingWriter(std.io.null_writer);
547 inst.encode(cwriter.writer()) catch unreachable; // Not allowed to fail here unless OOM.552 inst.encode(cwriter.writer(), .{ .allow_frame_loc = true }) catch unreachable; // Not allowed to fail here unless OOM.
548 return @intCast(usize, cwriter.bytes_written);553 return @intCast(usize, cwriter.bytes_written);
549}554}
550555
src/arch/x86_64/Lower.zig+14-19
...@@ -60,6 +60,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {...@@ -60,6 +60,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
60 .mfence,60 .mfence,
61 .mov,61 .mov,
62 .movbe,62 .movbe,
63 .movd,
64 .movq,
63 .movzx,65 .movzx,
64 .mul,66 .mul,
65 .neg,67 .neg,
...@@ -136,7 +138,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {...@@ -136,7 +138,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
136 .setcc => try lower.mirSetcc(inst),138 .setcc => try lower.mirSetcc(inst),
137 .jcc => try lower.emit(.none, mnem_cc(.j, inst.data.inst_cc.cc), &.{.{ .imm = Immediate.s(0) }}),139 .jcc => try lower.emit(.none, mnem_cc(.j, inst.data.inst_cc.cc), &.{.{ .imm = Immediate.s(0) }}),
138140
139 .push_regs, .pop_regs => try lower.mirPushPopRegisterList(inst),141 .push_regs => try lower.mirPushPopRegisterList(inst, .push),
142 .pop_regs => try lower.mirPushPopRegisterList(inst, .pop),
140143
141 .dbg_line,144 .dbg_line,
142 .dbg_prologue_end,145 .dbg_prologue_end,
...@@ -190,7 +193,7 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {...@@ -190,7 +193,7 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
190}193}
191194
192fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {195fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
193 return switch (ops) {196 return lower.mir.resolveFrameLoc(switch (ops) {
194 .rm_sib,197 .rm_sib,
195 .rm_sib_cc,198 .rm_sib_cc,
196 .m_sib,199 .m_sib,
...@@ -227,7 +230,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {...@@ -227,7 +230,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
227 => lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),230 => lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),
228231
229 else => unreachable,232 else => unreachable,
230 };233 });
231}234}
232235
233fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {236fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
...@@ -417,23 +420,15 @@ fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -417,23 +420,15 @@ fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {
417 }420 }
418}421}
419422
420fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst) Error!void {423fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst, comptime mnemonic: Mnemonic) Error!void {
421 const save_reg_list = lower.mir.extraData(Mir.SaveRegisterList, inst.data.payload).data;424 const reg_list = Mir.RegisterList.fromInt(inst.data.payload);
422 const base = @intToEnum(Register, save_reg_list.base_reg);
423 var disp: i32 = -@intCast(i32, save_reg_list.stack_end);
424 const reg_list = Mir.RegisterList.fromInt(save_reg_list.register_list);
425 const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);425 const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);
426 for (callee_preserved_regs) |callee_preserved_reg| {426 var it = reg_list.iterator(.{ .direction = switch (mnemonic) {
427 if (!reg_list.isSet(callee_preserved_regs, callee_preserved_reg)) continue;427 .push => .reverse,
428 const reg_op = Operand{ .reg = callee_preserved_reg };428 .pop => .forward,
429 const mem_op = Operand{ .mem = Memory.sib(.qword, .{ .base = base, .disp = disp }) };429 else => unreachable,
430 try lower.emit(.none, .mov, switch (inst.tag) {430 } });
431 .push_regs => &.{ mem_op, reg_op },431 while (it.next()) |i| try lower.emit(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});
432 .pop_regs => &.{ reg_op, mem_op },
433 else => unreachable,
434 });
435 disp += 8;
436 }
437}432}
438433
439fn mirLeaLinker(lower: *Lower, inst: Mir.Inst) Error!void {434fn mirLeaLinker(lower: *Lower, inst: Mir.Inst) Error!void {
src/arch/x86_64/Mir.zig+69-39
...@@ -23,6 +23,7 @@ const Register = bits.Register;...@@ -23,6 +23,7 @@ const Register = bits.Register;
23instructions: std.MultiArrayList(Inst).Slice,23instructions: std.MultiArrayList(Inst).Slice,
24/// The meaning of this data is determined by `Inst.Tag` value.24/// The meaning of this data is determined by `Inst.Tag` value.
25extra: []const u32,25extra: []const u32,
26frame_locs: std.MultiArrayList(FrameLoc).Slice,
2627
27pub const Inst = struct {28pub const Inst = struct {
28 tag: Tag,29 tag: Tag,
...@@ -98,6 +99,10 @@ pub const Inst = struct {...@@ -98,6 +99,10 @@ pub const Inst = struct {
98 mov,99 mov,
99 /// Move data after swapping bytes100 /// Move data after swapping bytes
100 movbe,101 movbe,
102 /// Move doubleword
103 movd,
104 /// Move quadword
105 movq,
101 /// Move with sign extension106 /// Move with sign extension
102 movsx,107 movsx,
103 /// Move with zero extension108 /// Move with zero extension
...@@ -241,13 +246,13 @@ pub const Inst = struct {...@@ -241,13 +246,13 @@ pub const Inst = struct {
241 /// Start of epilogue246 /// Start of epilogue
242 dbg_epilogue_begin,247 dbg_epilogue_begin,
243 /// Update debug line248 /// Update debug line
244 /// Uses `payload` payload with data of type `DbgLineColumn`.249 /// Uses `line_column` payload containing the line and column.
245 dbg_line,250 dbg_line,
246 /// Push registers251 /// Push registers
247 /// Uses `payload` payload with data of type `SaveRegisterList`.252 /// Uses `payload` payload containing `RegisterList.asInt` directly.
248 push_regs,253 push_regs,
249 /// Pop registers254 /// Pop registers
250 /// Uses `payload` payload with data of type `SaveRegisterList`.255 /// Uses `payload` payload containing `RegisterList.asInt` directly.
251 pop_regs,256 pop_regs,
252257
253 /// Tombstone258 /// Tombstone
...@@ -500,6 +505,11 @@ pub const Inst = struct {...@@ -500,6 +505,11 @@ pub const Inst = struct {
500 /// Index into the linker's symbol table.505 /// Index into the linker's symbol table.
501 sym_index: u32,506 sym_index: u32,
502 },507 },
508 /// Debug line and column position
509 line_column: struct {
510 line: u32,
511 column: u32,
512 },
503 /// Index into `extra`. Meaning of what can be found there is context-dependent.513 /// Index into `extra`. Meaning of what can be found there is context-dependent.
504 payload: u32,514 payload: u32,
505 };515 };
...@@ -522,12 +532,11 @@ pub const LeaRegisterReloc = struct {...@@ -522,12 +532,11 @@ pub const LeaRegisterReloc = struct {
522 sym_index: u32,532 sym_index: u32,
523};533};
524534
525/// Used in conjunction with `SaveRegisterList` payload to transfer a list of used registers535/// Used in conjunction with payload to transfer a list of used registers in a compact manner.
526/// in a compact manner.
527pub const RegisterList = struct {536pub const RegisterList = struct {
528 bitset: BitSet = BitSet.initEmpty(),537 bitset: BitSet = BitSet.initEmpty(),
529538
530 const BitSet = IntegerBitSet(@ctz(@as(u32, 0)));539 const BitSet = IntegerBitSet(32);
531 const Self = @This();540 const Self = @This();
532541
533 fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt {542 fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt {
...@@ -547,6 +556,10 @@ pub const RegisterList = struct {...@@ -547,6 +556,10 @@ pub const RegisterList = struct {
547 return self.bitset.isSet(index);556 return self.bitset.isSet(index);
548 }557 }
549558
559 pub fn iterator(self: Self, comptime options: std.bit_set.IteratorOptions) BitSet.Iterator(options) {
560 return self.bitset.iterator(options);
561 }
562
550 pub fn asInt(self: Self) u32 {563 pub fn asInt(self: Self) u32 {
551 return self.bitset.mask;564 return self.bitset.mask;
552 }565 }
...@@ -562,14 +575,6 @@ pub const RegisterList = struct {...@@ -562,14 +575,6 @@ pub const RegisterList = struct {
562 }575 }
563};576};
564577
565pub const SaveRegisterList = struct {
566 /// Base register
567 base_reg: u32,
568 /// Use `RegisterList` to populate.
569 register_list: u32,
570 stack_end: u32,
571};
572
573pub const Imm64 = struct {578pub const Imm64 = struct {
574 msb: u32,579 msb: u32,
575 lsb: u32,580 lsb: u32,
...@@ -593,41 +598,51 @@ pub const Imm64 = struct {...@@ -593,41 +598,51 @@ pub const Imm64 = struct {
593pub const MemorySib = struct {598pub const MemorySib = struct {
594 /// Size of the pointer.599 /// Size of the pointer.
595 ptr_size: u32,600 ptr_size: u32,
596 /// Base register. -1 means null, or no base register.601 /// Base register tag of type Memory.Base.Tag
597 base: i32,602 base_tag: u32,
598 /// Scale for index register. -1 means null, or no scale.603 /// Base register of type Register or FrameIndex
599 /// This has to be in sync with `index` field.604 base: u32,
600 scale: i32,605 /// Scale starting at bit 0 and index register starting at bit 4.
601 /// Index register. -1 means null, or no index register.606 scale_index: u32,
602 /// This has to be in sync with `scale` field.
603 index: i32,
604 /// Displacement value.607 /// Displacement value.
605 disp: i32,608 disp: i32,
606609
607 pub fn encode(mem: Memory) MemorySib {610 pub fn encode(mem: Memory) MemorySib {
608 const sib = mem.sib;611 const sib = mem.sib;
612 assert(sib.scale_index.scale == 0 or std.math.isPowerOfTwo(sib.scale_index.scale));
609 return .{613 return .{
610 .ptr_size = @enumToInt(sib.ptr_size),614 .ptr_size = @enumToInt(sib.ptr_size),
611 .base = if (sib.base) |r| @enumToInt(r) else -1,615 .base_tag = @enumToInt(@as(Memory.Base.Tag, sib.base)),
612 .scale = if (sib.scale_index) |si| si.scale else -1,616 .base = switch (sib.base) {
613 .index = if (sib.scale_index) |si| @enumToInt(si.index) else -1,617 .none => undefined,
618 .reg => |r| @enumToInt(r),
619 .frame => |fi| @enumToInt(fi),
620 },
621 .scale_index = @as(u32, sib.scale_index.scale) << 0 |
622 @as(u32, if (sib.scale_index.scale > 0)
623 @enumToInt(sib.scale_index.index)
624 else
625 undefined) << 4,
614 .disp = sib.disp,626 .disp = sib.disp,
615 };627 };
616 }628 }
617629
618 pub fn decode(msib: MemorySib) Memory {630 pub fn decode(msib: MemorySib) Memory {
619 const base: ?Register = if (msib.base == -1) null else @intToEnum(Register, msib.base);631 const scale = @truncate(u4, msib.scale_index);
620 const scale_index: ?Memory.ScaleIndex = if (msib.index == -1) null else .{632 assert(scale == 0 or std.math.isPowerOfTwo(scale));
621 .scale = @intCast(u4, msib.scale),633 return .{ .sib = .{
622 .index = @intToEnum(Register, msib.index),
623 };
624 const mem: Memory = .{ .sib = .{
625 .ptr_size = @intToEnum(Memory.PtrSize, msib.ptr_size),634 .ptr_size = @intToEnum(Memory.PtrSize, msib.ptr_size),
626 .base = base,635 .base = switch (@intToEnum(Memory.Base.Tag, msib.base_tag)) {
627 .scale_index = scale_index,636 .none => .none,
637 .reg => .{ .reg = @intToEnum(Register, msib.base) },
638 .frame => .{ .frame = @intToEnum(bits.FrameIndex, msib.base) },
639 },
640 .scale_index = .{
641 .scale = scale,
642 .index = if (scale > 0) @intToEnum(Register, msib.scale_index >> 4) else undefined,
643 },
628 .disp = msib.disp,644 .disp = msib.disp,
629 } };645 } };
630 return mem;
631 }646 }
632};647};
633648
...@@ -676,14 +691,10 @@ pub const MemoryMoffs = struct {...@@ -676,14 +691,10 @@ pub const MemoryMoffs = struct {
676 }691 }
677};692};
678693
679pub const DbgLineColumn = struct {
680 line: u32,
681 column: u32,
682};
683
684pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {694pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
685 mir.instructions.deinit(gpa);695 mir.instructions.deinit(gpa);
686 gpa.free(mir.extra);696 gpa.free(mir.extra);
697 mir.frame_locs.deinit(gpa);
687 mir.* = undefined;698 mir.* = undefined;
688}699}
689700
...@@ -704,3 +715,22 @@ pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end:...@@ -704,3 +715,22 @@ pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end:
704 .end = i,715 .end = i,
705 };716 };
706}717}
718
719pub const FrameLoc = struct {
720 base: Register,
721 disp: i32,
722};
723
724pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory {
725 return switch (mem) {
726 .sib => |sib| switch (sib.base) {
727 .none, .reg => mem,
728 .frame => |index| if (mir.frame_locs.len > 0) Memory.sib(sib.ptr_size, .{
729 .base = .{ .reg = mir.frame_locs.items(.base)[@enumToInt(index)] },
730 .disp = mir.frame_locs.items(.disp)[@enumToInt(index)] + sib.disp,
731 .scale_index = mem.scaleIndex(),
732 }) else mem,
733 },
734 .rip, .moffs => mem,
735 };
736}
src/arch/x86_64/bits.zig+68-10
...@@ -405,14 +405,69 @@ test "Register classes" {...@@ -405,14 +405,69 @@ test "Register classes" {
405 try expect(Register.fs.class() == .segment);405 try expect(Register.fs.class() == .segment);
406}406}
407407
408pub const FrameIndex = enum(u32) {
409 // This index refers to the start of the arguments passed to this function
410 args_frame,
411 // This index refers to the return address pushed by a `call` and popped by a `ret`.
412 ret_addr,
413 // This index refers to the base pointer pushed in the prologue and popped in the epilogue.
414 base_ptr,
415 // This index refers to the entire stack frame.
416 stack_frame,
417 // This index refers to the start of the call frame for arguments passed to called functions
418 call_frame,
419 // Other indices are used for local variable stack slots
420 _,
421
422 pub const named_count = @typeInfo(FrameIndex).Enum.fields.len;
423
424 pub fn isNamed(fi: FrameIndex) bool {
425 return @enumToInt(fi) < named_count;
426 }
427
428 pub fn format(
429 fi: FrameIndex,
430 comptime fmt: []const u8,
431 options: std.fmt.FormatOptions,
432 writer: anytype,
433 ) @TypeOf(writer).Error!void {
434 try writer.writeAll("FrameIndex");
435 if (fi.isNamed()) {
436 try writer.writeByte('.');
437 try writer.writeAll(@tagName(fi));
438 } else {
439 try writer.writeByte('(');
440 try std.fmt.formatType(@enumToInt(fi), fmt, options, writer, 0);
441 try writer.writeByte(')');
442 }
443 }
444};
445
408pub const Memory = union(enum) {446pub const Memory = union(enum) {
409 sib: Sib,447 sib: Sib,
410 rip: Rip,448 rip: Rip,
411 moffs: Moffs,449 moffs: Moffs,
412450
413 pub const ScaleIndex = packed struct {451 pub const Base = union(enum) {
452 none,
453 reg: Register,
454 frame: FrameIndex,
455
456 pub const Tag = @typeInfo(Base).Union.tag_type.?;
457
458 pub fn isExtended(self: Base) bool {
459 return switch (self) {
460 .none, .frame => false, // neither rsp nor rbp are extended
461 .reg => |reg| reg.isExtended(),
462 };
463 }
464 };
465
466 pub const ScaleIndex = struct {
414 scale: u4,467 scale: u4,
415 index: Register,468 index: Register,
469
470 const none = ScaleIndex{ .scale = 0, .index = undefined };
416 };471 };
417472
418 pub const PtrSize = enum {473 pub const PtrSize = enum {
...@@ -460,8 +515,8 @@ pub const Memory = union(enum) {...@@ -460,8 +515,8 @@ pub const Memory = union(enum) {
460515
461 pub const Sib = struct {516 pub const Sib = struct {
462 ptr_size: PtrSize,517 ptr_size: PtrSize,
463 base: ?Register,518 base: Base,
464 scale_index: ?ScaleIndex,519 scale_index: ScaleIndex,
465 disp: i32,520 disp: i32,
466 };521 };
467522
...@@ -482,7 +537,7 @@ pub const Memory = union(enum) {...@@ -482,7 +537,7 @@ pub const Memory = union(enum) {
482537
483 pub fn sib(ptr_size: PtrSize, args: struct {538 pub fn sib(ptr_size: PtrSize, args: struct {
484 disp: i32 = 0,539 disp: i32 = 0,
485 base: ?Register = null,540 base: Base = .none,
486 scale_index: ?ScaleIndex = null,541 scale_index: ?ScaleIndex = null,
487 }) Memory {542 }) Memory {
488 if (args.scale_index) |si| assert(std.math.isPowerOfTwo(si.scale));543 if (args.scale_index) |si| assert(std.math.isPowerOfTwo(si.scale));
...@@ -490,7 +545,7 @@ pub const Memory = union(enum) {...@@ -490,7 +545,7 @@ pub const Memory = union(enum) {
490 .base = args.base,545 .base = args.base,
491 .disp = args.disp,546 .disp = args.disp,
492 .ptr_size = ptr_size,547 .ptr_size = ptr_size,
493 .scale_index = args.scale_index,548 .scale_index = if (args.scale_index) |si| si else ScaleIndex.none,
494 } };549 } };
495 }550 }
496551
...@@ -502,22 +557,25 @@ pub const Memory = union(enum) {...@@ -502,22 +557,25 @@ pub const Memory = union(enum) {
502 return switch (mem) {557 return switch (mem) {
503 .moffs => true,558 .moffs => true,
504 .rip => false,559 .rip => false,
505 .sib => |s| if (s.base) |r| r.class() == .segment else false,560 .sib => |s| switch (s.base) {
561 .none, .frame => false,
562 .reg => |reg| reg.class() == .segment,
563 },
506 };564 };
507 }565 }
508566
509 pub fn base(mem: Memory) ?Register {567 pub fn base(mem: Memory) Base {
510 return switch (mem) {568 return switch (mem) {
511 .moffs => |m| m.seg,569 .moffs => |m| .{ .reg = m.seg },
512 .sib => |s| s.base,570 .sib => |s| s.base,
513 .rip => null,571 .rip => .none,
514 };572 };
515 }573 }
516574
517 pub fn scaleIndex(mem: Memory) ?ScaleIndex {575 pub fn scaleIndex(mem: Memory) ?ScaleIndex {
518 return switch (mem) {576 return switch (mem) {
519 .moffs, .rip => null,577 .moffs, .rip => null,
520 .sib => |s| s.scale_index,578 .sib => |s| if (s.scale_index.scale > 0) s.scale_index else null,
521 };579 };
522 }580 }
523581
src/arch/x86_64/encoder.zig+117-94
...@@ -54,6 +54,21 @@ pub const Instruction = struct {...@@ -54,6 +54,21 @@ pub const Instruction = struct {
54 };54 };
55 }55 }
5656
57 pub fn isBaseExtended(op: Operand) bool {
58 return switch (op) {
59 .none, .imm => false,
60 .reg => |reg| reg.isExtended(),
61 .mem => |mem| mem.base().isExtended(),
62 };
63 }
64
65 pub fn isIndexExtended(op: Operand) bool {
66 return switch (op) {
67 .none, .reg, .imm => false,
68 .mem => |mem| if (mem.scaleIndex()) |si| si.index.isExtended() else false,
69 };
70 }
71
57 fn format(72 fn format(
58 op: Operand,73 op: Operand,
59 comptime unused_format_string: []const u8,74 comptime unused_format_string: []const u8,
...@@ -98,17 +113,24 @@ pub const Instruction = struct {...@@ -98,17 +113,24 @@ pub const Instruction = struct {
98 try writer.print("{s} ptr ", .{@tagName(sib.ptr_size)});113 try writer.print("{s} ptr ", .{@tagName(sib.ptr_size)});
99114
100 if (mem.isSegmentRegister()) {115 if (mem.isSegmentRegister()) {
101 return writer.print("{s}:0x{x}", .{ @tagName(sib.base.?), sib.disp });116 return writer.print("{s}:0x{x}", .{ @tagName(sib.base.reg), sib.disp });
102 }117 }
103118
104 try writer.writeByte('[');119 try writer.writeByte('[');
105120
106 var any = false;121 var any = false;
107 if (sib.base) |base| {122 switch (sib.base) {
108 try writer.print("{s}", .{@tagName(base)});123 .none => {},
109 any = true;124 .reg => |reg| {
125 try writer.print("{s}", .{@tagName(reg)});
126 any = true;
127 },
128 .frame => |frame| {
129 try writer.print("{}", .{frame});
130 any = true;
131 },
110 }132 }
111 if (sib.scale_index) |si| {133 if (mem.scaleIndex()) |si| {
112 if (any) try writer.writeAll(" + ");134 if (any) try writer.writeAll(" + ");
113 try writer.print("{s} * {d}", .{ @tagName(si.index), si.scale });135 try writer.print("{s} * {d}", .{ @tagName(si.index), si.scale });
114 any = true;136 any = true;
...@@ -182,8 +204,8 @@ pub const Instruction = struct {...@@ -182,8 +204,8 @@ pub const Instruction = struct {
182 }204 }
183 }205 }
184206
185 pub fn encode(inst: Instruction, writer: anytype) !void {207 pub fn encode(inst: Instruction, writer: anytype, comptime opts: Options) !void {
186 const encoder = Encoder(@TypeOf(writer)){ .writer = writer };208 const encoder = Encoder(@TypeOf(writer), opts){ .writer = writer };
187 const enc = inst.encoding;209 const enc = inst.encoding;
188 const data = enc.data;210 const data = enc.data;
189211
...@@ -269,22 +291,24 @@ pub const Instruction = struct {...@@ -269,22 +291,24 @@ pub const Instruction = struct {
269291
270 const segment_override: ?Register = switch (op_en) {292 const segment_override: ?Register = switch (op_en) {
271 .i, .zi, .o, .oi, .d, .np => null,293 .i, .zi, .o, .oi, .d, .np => null,
272 .fd => inst.ops[1].mem.base().?,294 .fd => inst.ops[1].mem.base().reg,
273 .td => inst.ops[0].mem.base().?,295 .td => inst.ops[0].mem.base().reg,
274 .rm, .rmi => if (inst.ops[1].isSegmentRegister()) blk: {296 .rm, .rmi => if (inst.ops[1].isSegmentRegister())
275 break :blk switch (inst.ops[1]) {297 switch (inst.ops[1]) {
276 .reg => |r| r,298 .reg => |reg| reg,
277 .mem => |m| m.base().?,299 .mem => |mem| mem.base().reg,
278 else => unreachable,300 else => unreachable,
279 };301 }
280 } else null,302 else
281 .m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.ops[0].isSegmentRegister()) blk: {303 null,
282 break :blk switch (inst.ops[0]) {304 .m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.ops[0].isSegmentRegister())
283 .reg => |r| r,305 switch (inst.ops[0]) {
284 .mem => |m| m.base().?,306 .reg => |reg| reg,
307 .mem => |mem| mem.base().reg,
285 else => unreachable,308 else => unreachable,
286 };309 }
287 } else null,310 else
311 null,
288 };312 };
289 if (segment_override) |seg| {313 if (segment_override) |seg| {
290 legacy.setSegmentOverride(seg);314 legacy.setSegmentOverride(seg);
...@@ -298,7 +322,10 @@ pub const Instruction = struct {...@@ -298,7 +322,10 @@ pub const Instruction = struct {
298322
299 var rex = Rex{};323 var rex = Rex{};
300 rex.present = inst.encoding.data.mode == .rex;324 rex.present = inst.encoding.data.mode == .rex;
301 rex.w = inst.encoding.data.mode == .long;325 switch (inst.encoding.data.mode) {
326 .long, .sse2_long => rex.w = true,
327 else => {},
328 }
302329
303 switch (op_en) {330 switch (op_en) {
304 .np, .i, .zi, .fd, .td, .d => {},331 .np, .i, .zi, .fd, .td, .d => {},
...@@ -307,27 +334,17 @@ pub const Instruction = struct {...@@ -307,27 +334,17 @@ pub const Instruction = struct {
307 const r_op = switch (op_en) {334 const r_op = switch (op_en) {
308 .rm, .rmi => inst.ops[0],335 .rm, .rmi => inst.ops[0],
309 .mr, .mri, .mrc => inst.ops[1],336 .mr, .mri, .mrc => inst.ops[1],
310 else => null,337 else => .none,
311 };338 };
312 if (r_op) |op| {339 rex.r = r_op.isBaseExtended();
313 rex.r = op.reg.isExtended();
314 }
315340
316 const b_x_op = switch (op_en) {341 const b_x_op = switch (op_en) {
317 .rm, .rmi => inst.ops[1],342 .rm, .rmi => inst.ops[1],
318 .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],343 .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],
319 else => unreachable,344 else => unreachable,
320 };345 };
321 switch (b_x_op) {346 rex.b = b_x_op.isBaseExtended();
322 .reg => |r| {347 rex.x = b_x_op.isIndexExtended();
323 rex.b = r.isExtended();
324 },
325 .mem => |mem| {
326 rex.b = if (mem.base()) |base| base.isExtended() else false;
327 rex.x = if (mem.scaleIndex()) |si| si.index.isExtended() else false;
328 },
329 else => unreachable,
330 }
331 },348 },
332 }349 }
333350
...@@ -348,72 +365,75 @@ pub const Instruction = struct {...@@ -348,72 +365,75 @@ pub const Instruction = struct {
348365
349 switch (mem) {366 switch (mem) {
350 .moffs => unreachable,367 .moffs => unreachable,
351 .sib => |sib| {368 .sib => |sib| switch (sib.base) {
352 if (sib.base) |base| {369 .none => {
353 if (base.class() == .segment) {370 try encoder.modRm_SIBDisp0(operand_enc);
354 // TODO audit this wrt SIB371 if (mem.scaleIndex()) |si| {
355 try encoder.modRm_SIBDisp0(operand_enc);372 const scale = math.log2_int(u4, si.scale);
356 if (sib.scale_index) |si| {373 try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
357 const scale = math.log2_int(u4, si.scale);
358 try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
359 } else {
360 try encoder.sib_disp32();
361 }
362 try encoder.disp32(sib.disp);
363 } else {374 } else {
364 assert(base.class() == .general_purpose);375 try encoder.sib_disp32();
365 const dst = base.lowEnc();
366 const src = operand_enc;
367 if (dst == 4 or sib.scale_index != null) {
368 if (sib.disp == 0 and dst != 5) {
369 try encoder.modRm_SIBDisp0(src);
370 if (sib.scale_index) |si| {
371 const scale = math.log2_int(u4, si.scale);
372 try encoder.sib_scaleIndexBase(scale, si.index.lowEnc(), dst);
373 } else {
374 try encoder.sib_base(dst);
375 }
376 } else if (math.cast(i8, sib.disp)) |_| {
377 try encoder.modRm_SIBDisp8(src);
378 if (sib.scale_index) |si| {
379 const scale = math.log2_int(u4, si.scale);
380 try encoder.sib_scaleIndexBaseDisp8(scale, si.index.lowEnc(), dst);
381 } else {
382 try encoder.sib_baseDisp8(dst);
383 }
384 try encoder.disp8(@truncate(i8, sib.disp));
385 } else {
386 try encoder.modRm_SIBDisp32(src);
387 if (sib.scale_index) |si| {
388 const scale = math.log2_int(u4, si.scale);
389 try encoder.sib_scaleIndexBaseDisp32(scale, si.index.lowEnc(), dst);
390 } else {
391 try encoder.sib_baseDisp32(dst);
392 }
393 try encoder.disp32(sib.disp);
394 }
395 } else {
396 if (sib.disp == 0 and dst != 5) {
397 try encoder.modRm_indirectDisp0(src, dst);
398 } else if (math.cast(i8, sib.disp)) |_| {
399 try encoder.modRm_indirectDisp8(src, dst);
400 try encoder.disp8(@truncate(i8, sib.disp));
401 } else {
402 try encoder.modRm_indirectDisp32(src, dst);
403 try encoder.disp32(sib.disp);
404 }
405 }
406 }376 }
407 } else {377 try encoder.disp32(sib.disp);
378 },
379 .reg => |base| if (base.class() == .segment) {
380 // TODO audit this wrt SIB
408 try encoder.modRm_SIBDisp0(operand_enc);381 try encoder.modRm_SIBDisp0(operand_enc);
409 if (sib.scale_index) |si| {382 if (mem.scaleIndex()) |si| {
410 const scale = math.log2_int(u4, si.scale);383 const scale = math.log2_int(u4, si.scale);
411 try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());384 try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
412 } else {385 } else {
413 try encoder.sib_disp32();386 try encoder.sib_disp32();
414 }387 }
415 try encoder.disp32(sib.disp);388 try encoder.disp32(sib.disp);
416 }389 } else {
390 assert(base.class() == .general_purpose);
391 const dst = base.lowEnc();
392 const src = operand_enc;
393 if (dst == 4 or mem.scaleIndex() != null) {
394 if (sib.disp == 0 and dst != 5) {
395 try encoder.modRm_SIBDisp0(src);
396 if (mem.scaleIndex()) |si| {
397 const scale = math.log2_int(u4, si.scale);
398 try encoder.sib_scaleIndexBase(scale, si.index.lowEnc(), dst);
399 } else {
400 try encoder.sib_base(dst);
401 }
402 } else if (math.cast(i8, sib.disp)) |_| {
403 try encoder.modRm_SIBDisp8(src);
404 if (mem.scaleIndex()) |si| {
405 const scale = math.log2_int(u4, si.scale);
406 try encoder.sib_scaleIndexBaseDisp8(scale, si.index.lowEnc(), dst);
407 } else {
408 try encoder.sib_baseDisp8(dst);
409 }
410 try encoder.disp8(@truncate(i8, sib.disp));
411 } else {
412 try encoder.modRm_SIBDisp32(src);
413 if (mem.scaleIndex()) |si| {
414 const scale = math.log2_int(u4, si.scale);
415 try encoder.sib_scaleIndexBaseDisp32(scale, si.index.lowEnc(), dst);
416 } else {
417 try encoder.sib_baseDisp32(dst);
418 }
419 try encoder.disp32(sib.disp);
420 }
421 } else {
422 if (sib.disp == 0 and dst != 5) {
423 try encoder.modRm_indirectDisp0(src, dst);
424 } else if (math.cast(i8, sib.disp)) |_| {
425 try encoder.modRm_indirectDisp8(src, dst);
426 try encoder.disp8(@truncate(i8, sib.disp));
427 } else {
428 try encoder.modRm_indirectDisp32(src, dst);
429 try encoder.disp32(sib.disp);
430 }
431 }
432 },
433 .frame => if (@TypeOf(encoder).options.allow_frame_loc) {
434 try encoder.modRm_indirectDisp32(operand_enc, undefined);
435 try encoder.disp32(undefined);
436 } else return error.CannotEncode,
417 },437 },
418 .rip => |rip| {438 .rip => |rip| {
419 try encoder.modRm_RIPDisp32(operand_enc);439 try encoder.modRm_RIPDisp32(operand_enc);
...@@ -482,11 +502,14 @@ pub const LegacyPrefixes = packed struct {...@@ -482,11 +502,14 @@ pub const LegacyPrefixes = packed struct {
482 }502 }
483};503};
484504
485fn Encoder(comptime T: type) type {505pub const Options = struct { allow_frame_loc: bool = false };
506
507fn Encoder(comptime T: type, comptime opts: Options) type {
486 return struct {508 return struct {
487 writer: T,509 writer: T,
488510
489 const Self = @This();511 const Self = @This();
512 pub const options = opts;
490513
491 // --------514 // --------
492 // Prefixes515 // Prefixes
src/arch/x86_64/encodings.zig+6
...@@ -860,6 +860,12 @@ pub const table = [_]Entry{...@@ -860,6 +860,12 @@ pub const table = [_]Entry{
860860
861 .{ .minsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .sse2 },861 .{ .minsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .sse2 },
862862
863 .{ .movd, .rm, &.{ .xmm, .rm32 }, &.{ 0x66, 0x0f, 0x6e }, 0, .sse2 },
864 .{ .movd, .mr, &.{ .rm32, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .sse2 },
865
866 .{ .movq, .rm, &.{ .xmm, .rm64 }, &.{ 0x66, 0x0f, 0x6e }, 0, .sse2_long },
867 .{ .movq, .mr, &.{ .rm64, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .sse2_long },
868
863 .{ .movq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .sse2 },869 .{ .movq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .sse2 },
864 .{ .movq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .sse2 },870 .{ .movq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .sse2 },
865871
src/link/Coff.zig+19-7
...@@ -492,7 +492,7 @@ fn growSection(self: *Coff, sect_id: u32, needed_size: u32) !void {...@@ -492,7 +492,7 @@ fn growSection(self: *Coff, sect_id: u32, needed_size: u32) !void {
492492
493 const sect_vm_capacity = self.allocatedVirtualSize(header.virtual_address);493 const sect_vm_capacity = self.allocatedVirtualSize(header.virtual_address);
494 if (needed_size > sect_vm_capacity) {494 if (needed_size > sect_vm_capacity) {
495 self.markRelocsDirtyByAddress(header.virtual_address + needed_size);495 self.markRelocsDirtyByAddress(header.virtual_address + header.virtual_size);
496 try self.growSectionVirtualMemory(sect_id, needed_size);496 try self.growSectionVirtualMemory(sect_id, needed_size);
497 }497 }
498498
...@@ -759,7 +759,9 @@ fn writeAtom(self: *Coff, atom_index: Atom.Index, code: []u8) !void {...@@ -759,7 +759,9 @@ fn writeAtom(self: *Coff, atom_index: Atom.Index, code: []u8) !void {
759 if (self.relocs.getPtr(atom_index)) |rels| {759 if (self.relocs.getPtr(atom_index)) |rels| {
760 try relocs.ensureTotalCapacityPrecise(rels.items.len);760 try relocs.ensureTotalCapacityPrecise(rels.items.len);
761 for (rels.items) |*reloc| {761 for (rels.items) |*reloc| {
762 if (reloc.isResolvable(self)) relocs.appendAssumeCapacity(reloc);762 if (reloc.isResolvable(self) and reloc.dirty) {
763 relocs.appendAssumeCapacity(reloc);
764 }
763 }765 }
764 }766 }
765767
...@@ -904,18 +906,28 @@ fn markRelocsDirtyByTarget(self: *Coff, target: SymbolWithLoc) void {...@@ -904,18 +906,28 @@ fn markRelocsDirtyByTarget(self: *Coff, target: SymbolWithLoc) void {
904}906}
905907
906fn markRelocsDirtyByAddress(self: *Coff, addr: u32) void {908fn markRelocsDirtyByAddress(self: *Coff, addr: u32) void {
909 const got_moved = blk: {
910 const sect_id = self.got_section_index orelse break :blk false;
911 break :blk self.sections.items(.header)[sect_id].virtual_address > addr;
912 };
913
914 // TODO: dirty relocations targeting import table if that got moved in memory
915
907 for (self.relocs.values()) |*relocs| {916 for (self.relocs.values()) |*relocs| {
908 for (relocs.items) |*reloc| {917 for (relocs.items) |*reloc| {
909 const target_vaddr = reloc.getTargetAddress(self) orelse continue;918 if (reloc.isGotIndirection()) {
910 if (target_vaddr < addr) continue;919 reloc.dirty = reloc.dirty or got_moved;
911 reloc.dirty = true;920 } else {
921 const target_vaddr = reloc.getTargetAddress(self) orelse continue;
922 if (target_vaddr > addr) reloc.dirty = true;
923 }
912 }924 }
913 }925 }
914926
915 // TODO: dirty only really affected GOT cells927 // TODO: dirty only really affected GOT cells
916 for (self.got_table.entries.items) |entry| {928 for (self.got_table.entries.items) |entry| {
917 const target_addr = self.getSymbol(entry).value;929 const target_addr = self.getSymbol(entry).value;
918 if (target_addr >= addr) {930 if (target_addr > addr) {
919 self.got_table_contents_dirty = true;931 self.got_table_contents_dirty = true;
920 break;932 break;
921 }933 }
...@@ -1624,7 +1636,7 @@ pub fn flushModule(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod...@@ -1624,7 +1636,7 @@ pub fn flushModule(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod
16241636
1625 for (self.relocs.keys(), self.relocs.values()) |atom_index, relocs| {1637 for (self.relocs.keys(), self.relocs.values()) |atom_index, relocs| {
1626 const needs_update = for (relocs.items) |reloc| {1638 const needs_update = for (relocs.items) |reloc| {
1627 if (reloc.isResolvable(self)) break true;1639 if (reloc.dirty) break true;
1628 } else false;1640 } else false;
16291641
1630 if (!needs_update) continue;1642 if (!needs_update) continue;
src/link/Coff/Relocation.zig+18-12
...@@ -45,6 +45,19 @@ pcrel: bool,...@@ -45,6 +45,19 @@ pcrel: bool,
45length: u2,45length: u2,
46dirty: bool = true,46dirty: bool = true,
4747
48/// Returns true if and only if the reloc can be resolved.
49pub fn isResolvable(self: Relocation, coff_file: *Coff) bool {
50 _ = self.getTargetAddress(coff_file) orelse return false;
51 return true;
52}
53
54pub fn isGotIndirection(self: Relocation) bool {
55 return switch (self.type) {
56 .got, .got_page, .got_pageoff => true,
57 else => false,
58 };
59}
60
48/// Returns address of the target if any.61/// Returns address of the target if any.
49pub fn getTargetAddress(self: Relocation, coff_file: *const Coff) ?u32 {62pub fn getTargetAddress(self: Relocation, coff_file: *const Coff) ?u32 {
50 switch (self.type) {63 switch (self.type) {
...@@ -53,11 +66,6 @@ pub fn getTargetAddress(self: Relocation, coff_file: *const Coff) ?u32 {...@@ -53,11 +66,6 @@ pub fn getTargetAddress(self: Relocation, coff_file: *const Coff) ?u32 {
53 const header = coff_file.sections.items(.header)[coff_file.got_section_index.?];66 const header = coff_file.sections.items(.header)[coff_file.got_section_index.?];
54 return header.virtual_address + got_index * coff_file.ptr_width.size();67 return header.virtual_address + got_index * coff_file.ptr_width.size();
55 },68 },
56 .direct, .page, .pageoff => {
57 const target_atom_index = coff_file.getAtomIndexForSymbol(self.target) orelse return null;
58 const target_atom = coff_file.getAtom(target_atom_index);
59 return target_atom.getSymbol(coff_file).value;
60 },
61 .import, .import_page, .import_pageoff => {69 .import, .import_page, .import_pageoff => {
62 const sym = coff_file.getSymbol(self.target);70 const sym = coff_file.getSymbol(self.target);
63 const index = coff_file.import_tables.getIndex(sym.value) orelse return null;71 const index = coff_file.import_tables.getIndex(sym.value) orelse return null;
...@@ -68,16 +76,14 @@ pub fn getTargetAddress(self: Relocation, coff_file: *const Coff) ?u32 {...@@ -68,16 +76,14 @@ pub fn getTargetAddress(self: Relocation, coff_file: *const Coff) ?u32 {
68 .name_off = sym.value,76 .name_off = sym.value,
69 });77 });
70 },78 },
79 else => {
80 const target_atom_index = coff_file.getAtomIndexForSymbol(self.target) orelse return null;
81 const target_atom = coff_file.getAtom(target_atom_index);
82 return target_atom.getSymbol(coff_file).value;
83 },
71 }84 }
72}85}
7386
74/// Returns true if and only if the reloc is dirty AND the target address is available.
75pub fn isResolvable(self: Relocation, coff_file: *Coff) bool {
76 const addr = self.getTargetAddress(coff_file) orelse return false;
77 if (addr == 0) return false;
78 return self.dirty;
79}
80
81pub fn resolve(self: Relocation, atom_index: Atom.Index, code: []u8, image_base: u64, coff_file: *Coff) void {87pub fn resolve(self: Relocation, atom_index: Atom.Index, code: []u8, image_base: u64, coff_file: *Coff) void {
82 const atom = coff_file.getAtom(atom_index);88 const atom = coff_file.getAtom(atom_index);
83 const source_sym = atom.getSymbol(coff_file);89 const source_sym = atom.getSymbol(coff_file);
src/link/MachO.zig+26-8
...@@ -1146,7 +1146,9 @@ pub fn writeAtom(self: *MachO, atom_index: Atom.Index, code: []u8) !void {...@@ -1146,7 +1146,9 @@ pub fn writeAtom(self: *MachO, atom_index: Atom.Index, code: []u8) !void {
1146 if (self.relocs.getPtr(atom_index)) |rels| {1146 if (self.relocs.getPtr(atom_index)) |rels| {
1147 try relocs.ensureTotalCapacityPrecise(rels.items.len);1147 try relocs.ensureTotalCapacityPrecise(rels.items.len);
1148 for (rels.items) |*reloc| {1148 for (rels.items) |*reloc| {
1149 if (reloc.isResolvable(self)) relocs.appendAssumeCapacity(reloc);1149 if (reloc.isResolvable(self) and reloc.dirty) {
1150 relocs.appendAssumeCapacity(reloc);
1151 }
1150 }1152 }
1151 }1153 }
11521154
...@@ -1332,18 +1334,33 @@ fn markRelocsDirtyByTarget(self: *MachO, target: SymbolWithLoc) void {...@@ -1332,18 +1334,33 @@ fn markRelocsDirtyByTarget(self: *MachO, target: SymbolWithLoc) void {
13321334
1333fn markRelocsDirtyByAddress(self: *MachO, addr: u64) void {1335fn markRelocsDirtyByAddress(self: *MachO, addr: u64) void {
1334 log.debug("marking relocs dirty by address: {x}", .{addr});1336 log.debug("marking relocs dirty by address: {x}", .{addr});
1337
1338 const got_moved = blk: {
1339 const sect_id = self.got_section_index orelse break :blk false;
1340 break :blk self.sections.items(.header)[sect_id].addr > addr;
1341 };
1342 const stubs_moved = blk: {
1343 const sect_id = self.stubs_section_index orelse break :blk false;
1344 break :blk self.sections.items(.header)[sect_id].addr > addr;
1345 };
1346
1335 for (self.relocs.values()) |*relocs| {1347 for (self.relocs.values()) |*relocs| {
1336 for (relocs.items) |*reloc| {1348 for (relocs.items) |*reloc| {
1337 const target_addr = reloc.getTargetBaseAddress(self) orelse continue;1349 if (reloc.isGotIndirection()) {
1338 if (target_addr < addr) continue;1350 reloc.dirty = reloc.dirty or got_moved;
1339 reloc.dirty = true;1351 } else if (reloc.isStubTrampoline(self)) {
1352 reloc.dirty = reloc.dirty or stubs_moved;
1353 } else {
1354 const target_addr = reloc.getTargetBaseAddress(self) orelse continue;
1355 if (target_addr > addr) reloc.dirty = true;
1356 }
1340 }1357 }
1341 }1358 }
13421359
1343 // TODO: dirty only really affected GOT cells1360 // TODO: dirty only really affected GOT cells
1344 for (self.got_table.entries.items) |entry| {1361 for (self.got_table.entries.items) |entry| {
1345 const target_addr = self.getSymbol(entry).n_value;1362 const target_addr = self.getSymbol(entry).n_value;
1346 if (target_addr >= addr) {1363 if (target_addr > addr) {
1347 self.got_table_contents_dirty = true;1364 self.got_table_contents_dirty = true;
1348 break;1365 break;
1349 }1366 }
...@@ -1353,7 +1370,7 @@ fn markRelocsDirtyByAddress(self: *MachO, addr: u64) void {...@@ -1353,7 +1370,7 @@ fn markRelocsDirtyByAddress(self: *MachO, addr: u64) void {
1353 const stubs_addr = self.getSegment(self.stubs_section_index.?).vmaddr;1370 const stubs_addr = self.getSegment(self.stubs_section_index.?).vmaddr;
1354 const stub_helper_addr = self.getSegment(self.stub_helper_section_index.?).vmaddr;1371 const stub_helper_addr = self.getSegment(self.stub_helper_section_index.?).vmaddr;
1355 const laptr_addr = self.getSegment(self.la_symbol_ptr_section_index.?).vmaddr;1372 const laptr_addr = self.getSegment(self.la_symbol_ptr_section_index.?).vmaddr;
1356 if (stubs_addr >= addr or stub_helper_addr >= addr or laptr_addr >= addr)1373 if (stubs_addr > addr or stub_helper_addr > addr or laptr_addr > addr)
1357 self.stub_table_contents_dirty = true;1374 self.stub_table_contents_dirty = true;
1358 }1375 }
1359}1376}
...@@ -2794,7 +2811,7 @@ fn growSection(self: *MachO, sect_id: u8, needed_size: u64) !void {...@@ -2794,7 +2811,7 @@ fn growSection(self: *MachO, sect_id: u8, needed_size: u64) !void {
27942811
2795 const sect_vm_capacity = self.allocatedVirtualSize(segment.vmaddr);2812 const sect_vm_capacity = self.allocatedVirtualSize(segment.vmaddr);
2796 if (needed_size > sect_vm_capacity) {2813 if (needed_size > sect_vm_capacity) {
2797 self.markRelocsDirtyByAddress(segment.vmaddr + needed_size);2814 self.markRelocsDirtyByAddress(segment.vmaddr + segment.vmsize);
2798 try self.growSectionVirtualMemory(sect_id, needed_size);2815 try self.growSectionVirtualMemory(sect_id, needed_size);
2799 }2816 }
28002817
...@@ -4067,11 +4084,12 @@ pub fn findFirst(comptime T: type, haystack: []align(1) const T, start: usize, p...@@ -4067,11 +4084,12 @@ pub fn findFirst(comptime T: type, haystack: []align(1) const T, start: usize, p
4067pub fn logSections(self: *MachO) void {4084pub fn logSections(self: *MachO) void {
4068 log.debug("sections:", .{});4085 log.debug("sections:", .{});
4069 for (self.sections.items(.header), 0..) |header, i| {4086 for (self.sections.items(.header), 0..) |header, i| {
4070 log.debug(" sect({d}): {s},{s} @{x}, sizeof({x})", .{4087 log.debug(" sect({d}): {s},{s} @{x} ({x}), sizeof({x})", .{
4071 i + 1,4088 i + 1,
4072 header.segName(),4089 header.segName(),
4073 header.sectName(),4090 header.sectName(),
4074 header.offset,4091 header.offset,
4092 header.addr,
4075 header.size,4093 header.size,
4076 });4094 });
4077 }4095 }
src/link/MachO/Relocation.zig+27-14
...@@ -37,14 +37,33 @@ pub const Type = enum {...@@ -37,14 +37,33 @@ pub const Type = enum {
37 tlv_initializer,37 tlv_initializer,
38};38};
3939
40/// Returns true if and only if the reloc is dirty AND the target address is available.40/// Returns true if and only if the reloc can be resolved.
41pub fn isResolvable(self: Relocation, macho_file: *MachO) bool {41pub fn isResolvable(self: Relocation, macho_file: *MachO) bool {
42 const addr = self.getTargetBaseAddress(macho_file) orelse return false;42 _ = self.getTargetBaseAddress(macho_file) orelse return false;
43 if (addr == 0) return false;43 return true;
44 return self.dirty;44}
45
46pub fn isGotIndirection(self: Relocation) bool {
47 return switch (self.type) {
48 .got, .got_page, .got_pageoff => true,
49 else => false,
50 };
51}
52
53pub fn isStubTrampoline(self: Relocation, macho_file: *MachO) bool {
54 return switch (self.type) {
55 .branch => macho_file.getSymbol(self.target).undf(),
56 else => false,
57 };
45}58}
4659
47pub fn getTargetBaseAddress(self: Relocation, macho_file: *MachO) ?u64 {60pub fn getTargetBaseAddress(self: Relocation, macho_file: *MachO) ?u64 {
61 if (self.isStubTrampoline(macho_file)) {
62 const index = macho_file.stub_table.lookup.get(self.target) orelse return null;
63 const header = macho_file.sections.items(.header)[macho_file.stubs_section_index.?];
64 return header.addr +
65 index * @import("stubs.zig").calcStubEntrySize(macho_file.base.options.target.cpu.arch);
66 }
48 switch (self.type) {67 switch (self.type) {
49 .got, .got_page, .got_pageoff => {68 .got, .got_page, .got_pageoff => {
50 const got_index = macho_file.got_table.lookup.get(self.target) orelse return null;69 const got_index = macho_file.got_table.lookup.get(self.target) orelse return null;
...@@ -56,17 +75,11 @@ pub fn getTargetBaseAddress(self: Relocation, macho_file: *MachO) ?u64 {...@@ -56,17 +75,11 @@ pub fn getTargetBaseAddress(self: Relocation, macho_file: *MachO) ?u64 {
56 const atom = macho_file.getAtom(atom_index);75 const atom = macho_file.getAtom(atom_index);
57 return atom.getSymbol(macho_file).n_value;76 return atom.getSymbol(macho_file).n_value;
58 },77 },
59 .branch => {78 else => {
60 if (macho_file.stub_table.lookup.get(self.target)) |index| {79 const target_atom_index = macho_file.getAtomIndexForSymbol(self.target) orelse return null;
61 const header = macho_file.sections.items(.header)[macho_file.stubs_section_index.?];80 const target_atom = macho_file.getAtom(target_atom_index);
62 return header.addr +81 return target_atom.getSymbol(macho_file).n_value;
63 index * @import("stubs.zig").calcStubEntrySize(macho_file.base.options.target.cpu.arch);
64 }
65 const atom_index = macho_file.getAtomIndexForSymbol(self.target) orelse return null;
66 const atom = macho_file.getAtom(atom_index);
67 return atom.getSymbol(macho_file).n_value;
68 },82 },
69 else => return macho_file.getSymbol(self.target).n_value,
70 }83 }
71}84}
7285
test/behavior/align.zig+4-1
...@@ -491,7 +491,10 @@ test "read 128-bit field from default aligned struct in global memory" {...@@ -491,7 +491,10 @@ test "read 128-bit field from default aligned struct in global memory" {
491}491}
492492
493test "struct field explicit alignment" {493test "struct field explicit alignment" {
494 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;494 if (builtin.zig_backend == .stage2_x86_64) {
495 // Careful enabling this test, fails randomly.
496 return error.SkipZigTest;
497 }
495 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;498 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
496 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;499 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
497 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO500 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/basic.zig-1
...@@ -785,7 +785,6 @@ fn manyptrConcat(comptime s: [*:0]const u8) [*:0]const u8 {...@@ -785,7 +785,6 @@ fn manyptrConcat(comptime s: [*:0]const u8) [*:0]const u8 {
785test "comptime manyptr concatenation" {785test "comptime manyptr concatenation" {
786 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;786 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
787 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;787 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
788 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
789 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO788 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
790789
791 const s = "epic";790 const s = "epic";
test/behavior/bugs/12571.zig-1
...@@ -12,7 +12,6 @@ const Entry = packed struct {...@@ -12,7 +12,6 @@ const Entry = packed struct {
12};12};
1313
14test {14test {
15 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
16 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO15 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
17 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO16 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1817
test/behavior/bugs/12776.zig+4-1
...@@ -32,7 +32,10 @@ test {...@@ -32,7 +32,10 @@ test {
32 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;32 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
33 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;33 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
34 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;34 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
35 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;35 if (builtin.zig_backend == .stage2_x86_64) {
36 // Careful enabling this test, fails randomly.
37 return error.SkipZigTest;
38 }
3639
37 var ram = try RAM.new();40 var ram = try RAM.new();
38 var cpu = try CPU.new(&ram);41 var cpu = try CPU.new(&ram);
test/behavior/bugs/13069.zig-1
...@@ -4,7 +4,6 @@ const expect = std.testing.expect;...@@ -4,7 +4,6 @@ const expect = std.testing.expect;
44
5test {5test {
6 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO6 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
7 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO7 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO8 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
109
test/behavior/eval.zig-1
...@@ -532,7 +532,6 @@ test "@tagName of @typeInfo" {...@@ -532,7 +532,6 @@ test "@tagName of @typeInfo" {
532}532}
533533
534test "static eval list init" {534test "static eval list init" {
535 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
536 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO535 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
537 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO536 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
538 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO537 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/fn.zig-2
...@@ -335,7 +335,6 @@ fn numberLiteralArg(a: anytype) !void {...@@ -335,7 +335,6 @@ fn numberLiteralArg(a: anytype) !void {
335}335}
336336
337test "function call with anon list literal" {337test "function call with anon list literal" {
338 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
339 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO338 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
340 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO339 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
341 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO340 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -356,7 +355,6 @@ test "function call with anon list literal" {...@@ -356,7 +355,6 @@ test "function call with anon list literal" {
356}355}
357356
358test "function call with anon list literal - 2D" {357test "function call with anon list literal - 2D" {
359 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
360 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO358 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
361 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO359 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
362 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO360 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/math.zig-2
...@@ -734,7 +734,6 @@ test "small int addition" {...@@ -734,7 +734,6 @@ test "small int addition" {
734test "basic @mulWithOverflow" {734test "basic @mulWithOverflow" {
735 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO735 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
736 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO736 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
737 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
738737
739 {738 {
740 var a: u8 = 86;739 var a: u8 = 86;
...@@ -1126,7 +1125,6 @@ test "allow signed integer division/remainder when values are comptime-known and...@@ -1126,7 +1125,6 @@ test "allow signed integer division/remainder when values are comptime-known and
11261125
1127test "quad hex float literal parsing accurate" {1126test "quad hex float literal parsing accurate" {
1128 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO1127 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1129 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1130 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1128 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1131 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1129 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1132 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO1130 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/packed-struct.zig-2
...@@ -232,7 +232,6 @@ test "nested packed structs" {...@@ -232,7 +232,6 @@ test "nested packed structs" {
232}232}
233233
234test "regular in irregular packed struct" {234test "regular in irregular packed struct" {
235 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
236 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;235 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
237 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;236 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
238 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO237 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -621,7 +620,6 @@ test "store undefined to packed result location" {...@@ -621,7 +620,6 @@ test "store undefined to packed result location" {
621 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;620 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
622 if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;621 if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
623 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;622 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
624 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
625623
626 var x: u4 = 0;624 var x: u4 = 0;
627 var s = packed struct { x: u4, y: u4 }{ .x = x, .y = if (x > 0) x else undefined };625 var s = packed struct { x: u4, y: u4 }{ .x = x, .y = if (x > 0) x else undefined };
test/behavior/pointers.zig-1
...@@ -343,7 +343,6 @@ test "pointer sentinel with optional element" {...@@ -343,7 +343,6 @@ test "pointer sentinel with optional element" {
343test "pointer sentinel with +inf" {343test "pointer sentinel with +inf" {
344 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;344 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
345 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO345 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
346 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
347 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO346 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
348 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO347 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
349348
test/behavior/popcount.zig-1
...@@ -4,7 +4,6 @@ const expect = std.testing.expect;...@@ -4,7 +4,6 @@ const expect = std.testing.expect;
4const expectEqual = std.testing.expectEqual;4const expectEqual = std.testing.expectEqual;
55
6test "@popCount integers" {6test "@popCount integers" {
7 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO7 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO8 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO9 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/union.zig-1
...@@ -26,7 +26,6 @@ fn setFloat(foo: *FooWithFloats, x: f64) void {...@@ -26,7 +26,6 @@ fn setFloat(foo: *FooWithFloats, x: f64) void {
26}26}
2727
28test "init union with runtime value - floats" {28test "init union with runtime value - floats" {
29 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
30 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;29 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
31 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;30 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
32 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO31 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/vector.zig+4-4
...@@ -318,7 +318,6 @@ test "vector @splat" {...@@ -318,7 +318,6 @@ test "vector @splat" {
318}318}
319319
320test "load vector elements via comptime index" {320test "load vector elements via comptime index" {
321 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
322 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO321 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
323 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO322 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
324 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO323 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -340,7 +339,6 @@ test "load vector elements via comptime index" {...@@ -340,7 +339,6 @@ test "load vector elements via comptime index" {
340}339}
341340
342test "store vector elements via comptime index" {341test "store vector elements via comptime index" {
343 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
344 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO342 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
345 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO343 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
346 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO344 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -368,7 +366,6 @@ test "store vector elements via comptime index" {...@@ -368,7 +366,6 @@ test "store vector elements via comptime index" {
368}366}
369367
370test "load vector elements via runtime index" {368test "load vector elements via runtime index" {
371 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
372 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO369 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
373 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO370 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
374 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO371 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -390,7 +387,6 @@ test "load vector elements via runtime index" {...@@ -390,7 +387,6 @@ test "load vector elements via runtime index" {
390}387}
391388
392test "store vector elements via runtime index" {389test "store vector elements via runtime index" {
393 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
394 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO390 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
395 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO391 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
396 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO392 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -1100,6 +1096,7 @@ test "loading the second vector from a slice of vectors" {...@@ -1100,6 +1096,7 @@ test "loading the second vector from a slice of vectors" {
1100 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1096 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1101 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1097 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1102 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO1098 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1099 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
11031100
1104 @setRuntimeSafety(false);1101 @setRuntimeSafety(false);
1105 var small_bases = [2]@Vector(2, u8){1102 var small_bases = [2]@Vector(2, u8){
...@@ -1200,6 +1197,7 @@ test "zero multiplicand" {...@@ -1200,6 +1197,7 @@ test "zero multiplicand" {
1200 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1197 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1201 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1198 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1202 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO1199 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1200 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
12031201
1204 const zeros = @Vector(2, u32){ 0.0, 0.0 };1202 const zeros = @Vector(2, u32){ 0.0, 0.0 };
1205 var ones = @Vector(2, u32){ 1.0, 1.0 };1203 var ones = @Vector(2, u32){ 1.0, 1.0 };
...@@ -1259,6 +1257,7 @@ test "load packed vector element" {...@@ -1259,6 +1257,7 @@ test "load packed vector element" {
1259 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1257 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1260 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO1258 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1261 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO1259 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1260 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
12621261
1263 var x: @Vector(2, u15) = .{ 1, 4 };1262 var x: @Vector(2, u15) = .{ 1, 4 };
1264 try expect((&x[0]).* == 1);1263 try expect((&x[0]).* == 1);
...@@ -1297,6 +1296,7 @@ test "store to vector in slice" {...@@ -1297,6 +1296,7 @@ test "store to vector in slice" {
1297test "addition of vectors represented as strings" {1296test "addition of vectors represented as strings" {
1298 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1297 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1299 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1298 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1299 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
13001300
1301 const V = @Vector(3, u8);1301 const V = @Vector(3, u8);
1302 const foo: V = "foo".*;1302 const foo: V = "foo".*;
test/src/Cases.zig+7
...@@ -390,6 +390,13 @@ fn addFromDirInner(...@@ -390,6 +390,13 @@ fn addFromDirInner(
390 // Cross-product to get all possible test combinations390 // Cross-product to get all possible test combinations
391 for (backends) |backend| {391 for (backends) |backend| {
392 for (targets) |target| {392 for (targets) |target| {
393 if (backend == .stage2 and
394 target.getCpuArch() != .wasm32 and target.getCpuArch() != .x86_64)
395 {
396 // Other backends don't support new liveness format
397 continue;
398 }
399
393 const next = ctx.cases.items.len;400 const next = ctx.cases.items.len;
394 try ctx.cases.append(.{401 try ctx.cases.append(.{
395 .name = std.fs.path.stem(filename),402 .name = std.fs.path.stem(filename),