| ... | @@ -163,8 +163,12 @@ const MCValue = union(enum) { | ... | @@ -163,8 +163,12 @@ const MCValue = union(enum) { |
| 163 | immediate: u64, | 163 | immediate: u64, |
| 164 | /// The value doesn't exist in memory yet. | 164 | /// The value doesn't exist in memory yet. |
| 165 | load_symbol: SymbolOffset, | 165 | load_symbol: SymbolOffset, |
| | 166 | /// A TLV value. |
| | 167 | load_tlv: u32, |
| 166 | /// The address of the memory location not-yet-allocated by the linker. | 168 | /// The address of the memory location not-yet-allocated by the linker. |
| 167 | lea_symbol: SymbolOffset, | 169 | lea_symbol: SymbolOffset, |
| | 170 | /// The address of a TLV value. |
| | 171 | lea_tlv: u32, |
| 168 | /// The value is in a target-specific register. | 172 | /// The value is in a target-specific register. |
| 169 | register: Register, | 173 | register: Register, |
| 170 | /// The value is split across two registers | 174 | /// The value is split across two registers |
| ... | @@ -221,6 +225,7 @@ const MCValue = union(enum) { | ... | @@ -221,6 +225,7 @@ const MCValue = union(enum) { |
| 221 | .lea_frame, | 225 | .lea_frame, |
| 222 | .undef, | 226 | .undef, |
| 223 | .lea_symbol, | 227 | .lea_symbol, |
| | 228 | .lea_tlv, |
| 224 | .air_ref, | 229 | .air_ref, |
| 225 | .reserved_frame, | 230 | .reserved_frame, |
| 226 | => false, | 231 | => false, |
| ... | @@ -230,6 +235,7 @@ const MCValue = union(enum) { | ... | @@ -230,6 +235,7 @@ const MCValue = union(enum) { |
| 230 | .register_offset, | 235 | .register_offset, |
| 231 | .load_frame, | 236 | .load_frame, |
| 232 | .load_symbol, | 237 | .load_symbol, |
| | 238 | .load_tlv, |
| 233 | .indirect, | 239 | .indirect, |
| 234 | => true, | 240 | => true, |
| 235 | }; | 241 | }; |
| ... | @@ -248,10 +254,12 @@ const MCValue = union(enum) { | ... | @@ -248,10 +254,12 @@ const MCValue = union(enum) { |
| 248 | .undef, | 254 | .undef, |
| 249 | .air_ref, | 255 | .air_ref, |
| 250 | .lea_symbol, | 256 | .lea_symbol, |
| | 257 | .lea_tlv, |
| 251 | .reserved_frame, | 258 | .reserved_frame, |
| 252 | => unreachable, // not in memory | 259 | => unreachable, // not in memory |
| 253 | | 260 | |
| 254 | .load_symbol => |sym_off| .{ .lea_symbol = sym_off }, | 261 | .load_symbol => |sym_off| .{ .lea_symbol = sym_off }, |
| | 262 | .load_tlv => |sym| .{ .lea_tlv = sym }, |
| 255 | .memory => |addr| .{ .immediate = addr }, | 263 | .memory => |addr| .{ .immediate = addr }, |
| 256 | .load_frame => |off| .{ .lea_frame = off }, | 264 | .load_frame => |off| .{ .lea_frame = off }, |
| 257 | .indirect => |reg_off| switch (reg_off.off) { | 265 | .indirect => |reg_off| switch (reg_off.off) { |
| ... | @@ -270,17 +278,19 @@ const MCValue = union(enum) { | ... | @@ -270,17 +278,19 @@ const MCValue = union(enum) { |
| 270 | .indirect, | 278 | .indirect, |
| 271 | .undef, | 279 | .undef, |
| 272 | .air_ref, | 280 | .air_ref, |
| 273 | .load_frame, | | |
| 274 | .register_pair, | 281 | .register_pair, |
| | 282 | .load_frame, |
| 275 | .load_symbol, | 283 | .load_symbol, |
| | 284 | .load_tlv, |
| 276 | .reserved_frame, | 285 | .reserved_frame, |
| 277 | => unreachable, // not a pointer | 286 | => unreachable, // not a pointer |
| 278 | | 287 | |
| 279 | .immediate => |addr| .{ .memory = addr }, | 288 | .immediate => |addr| .{ .memory = addr }, |
| 280 | .lea_frame => |off| .{ .load_frame = off }, | | |
| 281 | .register => |reg| .{ .indirect = .{ .reg = reg } }, | 289 | .register => |reg| .{ .indirect = .{ .reg = reg } }, |
| 282 | .register_offset => |reg_off| .{ .indirect = reg_off }, | 290 | .register_offset => |reg_off| .{ .indirect = reg_off }, |
| | 291 | .lea_frame => |off| .{ .load_frame = off }, |
| 283 | .lea_symbol => |sym_off| .{ .load_symbol = sym_off }, | 292 | .lea_symbol => |sym_off| .{ .load_symbol = sym_off }, |
| | 293 | .lea_tlv => |sym| .{ .load_tlv = sym }, |
| 284 | }; | 294 | }; |
| 285 | } | 295 | } |
| 286 | | 296 | |
| ... | @@ -298,6 +308,8 @@ const MCValue = union(enum) { | ... | @@ -298,6 +308,8 @@ const MCValue = union(enum) { |
| 298 | .indirect, | 308 | .indirect, |
| 299 | .load_symbol, | 309 | .load_symbol, |
| 300 | .lea_symbol, | 310 | .lea_symbol, |
| | 311 | .lea_tlv, |
| | 312 | .load_tlv, |
| 301 | => switch (off) { | 313 | => switch (off) { |
| 302 | 0 => mcv, | 314 | 0 => mcv, |
| 303 | else => unreachable, | 315 | else => unreachable, |
| ... | @@ -355,6 +367,8 @@ const InstTracking = struct { | ... | @@ -355,6 +367,8 @@ const InstTracking = struct { |
| 355 | .memory, | 367 | .memory, |
| 356 | .load_frame, | 368 | .load_frame, |
| 357 | .lea_frame, | 369 | .lea_frame, |
| | 370 | .load_tlv, |
| | 371 | .lea_tlv, |
| 358 | .load_symbol, | 372 | .load_symbol, |
| 359 | .lea_symbol, | 373 | .lea_symbol, |
| 360 | => result, | 374 | => result, |
| ... | @@ -410,6 +424,8 @@ const InstTracking = struct { | ... | @@ -410,6 +424,8 @@ const InstTracking = struct { |
| 410 | .lea_frame, | 424 | .lea_frame, |
| 411 | .load_symbol, | 425 | .load_symbol, |
| 412 | .lea_symbol, | 426 | .lea_symbol, |
| | 427 | .load_tlv, |
| | 428 | .lea_tlv, |
| 413 | => inst_tracking.long, | 429 | => inst_tracking.long, |
| 414 | .dead, | 430 | .dead, |
| 415 | .register, | 431 | .register, |
| ... | @@ -438,6 +454,8 @@ const InstTracking = struct { | ... | @@ -438,6 +454,8 @@ const InstTracking = struct { |
| 438 | .lea_frame, | 454 | .lea_frame, |
| 439 | .load_symbol, | 455 | .load_symbol, |
| 440 | .lea_symbol, | 456 | .lea_symbol, |
| | 457 | .load_tlv, |
| | 458 | .lea_tlv, |
| 441 | => assert(std.meta.eql(inst_tracking.long, target.long)), | 459 | => assert(std.meta.eql(inst_tracking.long, target.long)), |
| 442 | .load_frame, | 460 | .load_frame, |
| 443 | .reserved_frame, | 461 | .reserved_frame, |
| ... | @@ -3510,17 +3528,19 @@ fn airWrapOptional(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -3510,17 +3528,19 @@ fn airWrapOptional(func: *Func, inst: Air.Inst.Index) !void { |
| 3510 | defer if (pl_lock) |lock| func.register_manager.unlockReg(lock); | 3528 | defer if (pl_lock) |lock| func.register_manager.unlockReg(lock); |
| 3511 | | 3529 | |
| 3512 | const opt_mcv = try func.allocRegOrMem(opt_ty, inst, true); | 3530 | const opt_mcv = try func.allocRegOrMem(opt_ty, inst, true); |
| 3513 | try func.genCopy(pl_ty, opt_mcv, pl_mcv); | | |
| 3514 | | 3531 | |
| 3515 | if (!same_repr) { | 3532 | if (!same_repr) { |
| 3516 | const pl_abi_size: i32 = @intCast(pl_ty.abiSize(pt)); | 3533 | const pl_abi_size: i32 = @intCast(pl_ty.abiSize(pt)); |
| 3517 | switch (opt_mcv) { | 3534 | switch (opt_mcv) { |
| 3518 | .load_frame => |frame_addr| try func.genSetMem( | 3535 | .load_frame => |frame_addr| { |
| 3519 | .{ .frame = frame_addr.index }, | 3536 | try func.genCopy(pl_ty, opt_mcv, pl_mcv); |
| 3520 | frame_addr.off + pl_abi_size, | 3537 | try func.genSetMem( |
| 3521 | Type.u8, | 3538 | .{ .frame = frame_addr.index }, |
| 3522 | .{ .immediate = 1 }, | 3539 | frame_addr.off + pl_abi_size, |
| 3523 | ), | 3540 | Type.u8, |
| | 3541 | .{ .immediate = 1 }, |
| | 3542 | ); |
| | 3543 | }, |
| 3524 | | 3544 | |
| 3525 | .register => |opt_reg| { | 3545 | .register => |opt_reg| { |
| 3526 | try func.genBinOp( | 3546 | try func.genBinOp( |
| ... | @@ -3531,6 +3551,7 @@ fn airWrapOptional(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -3531,6 +3551,7 @@ fn airWrapOptional(func: *Func, inst: Air.Inst.Index) !void { |
| 3531 | Type.u64, | 3551 | Type.u64, |
| 3532 | opt_reg, | 3552 | opt_reg, |
| 3533 | ); | 3553 | ); |
| | 3554 | try func.genCopy(pl_ty, opt_mcv, pl_mcv); |
| 3534 | }, | 3555 | }, |
| 3535 | else => unreachable, | 3556 | else => unreachable, |
| 3536 | } | 3557 | } |
| ... | @@ -4457,12 +4478,14 @@ fn load(func: *Func, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerErro | ... | @@ -4457,12 +4478,14 @@ fn load(func: *Func, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerErro |
| 4457 | .register_offset, | 4478 | .register_offset, |
| 4458 | .lea_frame, | 4479 | .lea_frame, |
| 4459 | .lea_symbol, | 4480 | .lea_symbol, |
| | 4481 | .lea_tlv, |
| 4460 | => try func.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()), | 4482 | => try func.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()), |
| 4461 | | 4483 | |
| 4462 | .memory, | 4484 | .memory, |
| 4463 | .indirect, | 4485 | .indirect, |
| 4464 | .load_symbol, | 4486 | .load_symbol, |
| 4465 | .load_frame, | 4487 | .load_frame, |
| | 4488 | .load_tlv, |
| 4466 | => { | 4489 | => { |
| 4467 | const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv); | 4490 | const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv); |
| 4468 | const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg); | 4491 | const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg); |
| ... | @@ -4509,12 +4532,14 @@ fn store(func: *Func, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type) !void { | ... | @@ -4509,12 +4532,14 @@ fn store(func: *Func, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type) !void { |
| 4509 | .register_offset, | 4532 | .register_offset, |
| 4510 | .lea_symbol, | 4533 | .lea_symbol, |
| 4511 | .lea_frame, | 4534 | .lea_frame, |
| | 4535 | .lea_tlv, |
| 4512 | => try func.genCopy(src_ty, ptr_mcv.deref(), src_mcv), | 4536 | => try func.genCopy(src_ty, ptr_mcv.deref(), src_mcv), |
| 4513 | | 4537 | |
| 4514 | .memory, | 4538 | .memory, |
| 4515 | .indirect, | 4539 | .indirect, |
| 4516 | .load_symbol, | 4540 | .load_symbol, |
| 4517 | .load_frame, | 4541 | .load_frame, |
| | 4542 | .load_tlv, |
| 4518 | => { | 4543 | => { |
| 4519 | const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv); | 4544 | const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv); |
| 4520 | const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg); | 4545 | const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg); |
| ... | @@ -5989,7 +6014,14 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -5989,7 +6014,14 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { |
| 5989 | try func.register_manager.getReg(reg, null); | 6014 | try func.register_manager.getReg(reg, null); |
| 5990 | try func.genSetReg(ty, reg, input_mcv); | 6015 | try func.genSetReg(ty, reg, input_mcv); |
| 5991 | break :arg .{ .register = reg }; | 6016 | break :arg .{ .register = reg }; |
| 5992 | } else return func.fail("invalid constraint: '{s}'", .{constraint}); | 6017 | } else if (mem.eql(u8, constraint, "r")) arg: { |
| | 6018 | switch (input_mcv) { |
| | 6019 | .register => break :arg input_mcv, |
| | 6020 | else => {}, |
| | 6021 | } |
| | 6022 | const temp_reg = try func.copyToTmpRegister(ty, input_mcv); |
| | 6023 | break :arg .{ .register = temp_reg }; |
| | 6024 | } else return func.fail("invalid input constraint: '{s}'", .{constraint}); |
| 5993 | if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { | 6025 | if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { |
| 5994 | _ = func.register_manager.lockReg(reg); | 6026 | _ = func.register_manager.lockReg(reg); |
| 5995 | }; | 6027 | }; |
| ... | @@ -6071,6 +6103,10 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -6071,6 +6103,10 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { |
| 6071 | assert(sym_off.off == 0); | 6103 | assert(sym_off.off == 0); |
| 6072 | break :blk .{ .sym = sym_off }; | 6104 | break :blk .{ .sym = sym_off }; |
| 6073 | } else return func.fail("invalid modifier: '{s}'", .{modifier}), | 6105 | } else return func.fail("invalid modifier: '{s}'", .{modifier}), |
| | 6106 | .register => |reg| if (modifier.len == 0) |
| | 6107 | .{ .reg = reg } |
| | 6108 | else |
| | 6109 | return func.fail("invalid modified '{s}'", .{modifier}), |
| 6074 | else => return func.fail("invalid constraint: '{s}'", .{op_str}), | 6110 | else => return func.fail("invalid constraint: '{s}'", .{op_str}), |
| 6075 | }; | 6111 | }; |
| 6076 | } else return func.fail("invalid operand: '{s}'", .{op_str}); | 6112 | } else return func.fail("invalid operand: '{s}'", .{op_str}); |
| ... | @@ -6253,6 +6289,13 @@ fn genCopy(func: *Func, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { | ... | @@ -6253,6 +6289,13 @@ fn genCopy(func: *Func, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { |
| 6253 | ty, | 6289 | ty, |
| 6254 | src_mcv, | 6290 | src_mcv, |
| 6255 | ), | 6291 | ), |
| | 6292 | .load_tlv => { |
| | 6293 | const addr_reg, const addr_lock = try func.allocReg(.int); |
| | 6294 | defer func.register_manager.unlockReg(addr_lock); |
| | 6295 | |
| | 6296 | try func.genSetReg(ty, addr_reg, dst_mcv.address()); |
| | 6297 | try func.genCopy(ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv); |
| | 6298 | }, |
| 6256 | .memory => return func.fail("TODO: genCopy memory", .{}), | 6299 | .memory => return func.fail("TODO: genCopy memory", .{}), |
| 6257 | .register_pair => |dst_regs| { | 6300 | .register_pair => |dst_regs| { |
| 6258 | const src_info: ?struct { addr_reg: Register, addr_lock: ?RegisterLock } = switch (src_mcv) { | 6301 | const src_info: ?struct { addr_reg: Register, addr_lock: ?RegisterLock } = switch (src_mcv) { |
| ... | @@ -6774,6 +6817,25 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! | ... | @@ -6774,6 +6817,25 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! |
| 6774 | try func.genSetReg(ty, addr_reg, src_mcv.address()); | 6817 | try func.genSetReg(ty, addr_reg, src_mcv.address()); |
| 6775 | try func.genSetReg(ty, reg, .{ .indirect = .{ .reg = addr_reg } }); | 6818 | try func.genSetReg(ty, reg, .{ .indirect = .{ .reg = addr_reg } }); |
| 6776 | }, | 6819 | }, |
| | 6820 | .lea_tlv => |sym| { |
| | 6821 | const atom_index = try func.owner.getSymbolIndex(func); |
| | 6822 | |
| | 6823 | _ = try func.addInst(.{ |
| | 6824 | .tag = .pseudo_load_tlv, |
| | 6825 | .data = .{ .reloc = .{ |
| | 6826 | .register = reg, |
| | 6827 | .atom_index = atom_index, |
| | 6828 | .sym_index = sym, |
| | 6829 | } }, |
| | 6830 | }); |
| | 6831 | }, |
| | 6832 | .load_tlv => { |
| | 6833 | const addr_reg, const addr_lock = try func.allocReg(.int); |
| | 6834 | defer func.register_manager.unlockReg(addr_lock); |
| | 6835 | |
| | 6836 | try func.genSetReg(ty, addr_reg, src_mcv.address()); |
| | 6837 | try func.genSetReg(ty, reg, .{ .indirect = .{ .reg = addr_reg } }); |
| | 6838 | }, |
| 6777 | .air_ref => |ref| try func.genSetReg(ty, reg, try func.resolveInst(ref)), | 6839 | .air_ref => |ref| try func.genSetReg(ty, reg, try func.resolveInst(ref)), |
| 6778 | else => return func.fail("TODO: genSetReg {s}", .{@tagName(src_mcv)}), | 6840 | else => return func.fail("TODO: genSetReg {s}", .{@tagName(src_mcv)}), |
| 6779 | } | 6841 | } |
| ... | @@ -6793,7 +6855,6 @@ fn genSetMem( | ... | @@ -6793,7 +6855,6 @@ fn genSetMem( |
| 6793 | const dst_ptr_mcv: MCValue = switch (base) { | 6855 | const dst_ptr_mcv: MCValue = switch (base) { |
| 6794 | .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } }, | 6856 | .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } }, |
| 6795 | .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } }, | 6857 | .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } }, |
| 6796 | .reloc => |base_symbol| .{ .lea_symbol = .{ .sym = base_symbol.sym_index, .off = disp } }, | | |
| 6797 | }; | 6858 | }; |
| 6798 | switch (src_mcv) { | 6859 | switch (src_mcv) { |
| 6799 | .none, | 6860 | .none, |
| ... | @@ -6940,6 +7001,7 @@ fn genSetMem( | ... | @@ -6940,6 +7001,7 @@ fn genSetMem( |
| 6940 | return func.genSetMem(base, disp, ty, .{ .register = reg }); | 7001 | return func.genSetMem(base, disp, ty, .{ .register = reg }); |
| 6941 | }, | 7002 | }, |
| 6942 | .air_ref => |src_ref| try func.genSetMem(base, disp, ty, try func.resolveInst(src_ref)), | 7003 | .air_ref => |src_ref| try func.genSetMem(base, disp, ty, try func.resolveInst(src_ref)), |
| | 7004 | else => return func.fail("TODO: genSetMem {s}", .{@tagName(src_mcv)}), |
| 6943 | } | 7005 | } |
| 6944 | } | 7006 | } |
| 6945 | | 7007 | |
| ... | @@ -7761,9 +7823,10 @@ fn genTypedValue(func: *Func, val: Value) InnerError!MCValue { | ... | @@ -7761,9 +7823,10 @@ fn genTypedValue(func: *Func, val: Value) InnerError!MCValue { |
| 7761 | .none => .none, | 7823 | .none => .none, |
| 7762 | .undef => unreachable, | 7824 | .undef => unreachable, |
| 7763 | .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } }, | 7825 | .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } }, |
| | 7826 | .load_tlv => |sym_index| .{ .lea_tlv = sym_index }, |
| 7764 | .immediate => |imm| .{ .immediate = imm }, | 7827 | .immediate => |imm| .{ .immediate = imm }, |
| 7765 | .memory => |addr| .{ .memory = addr }, | 7828 | .memory => |addr| .{ .memory = addr }, |
| 7766 | .load_got, .load_direct, .load_tlv => { | 7829 | .load_got, .load_direct => { |
| 7767 | return func.fail("TODO: genTypedValue {s}", .{@tagName(mcv)}); | 7830 | return func.fail("TODO: genTypedValue {s}", .{@tagName(mcv)}); |
| 7768 | }, | 7831 | }, |
| 7769 | }, | 7832 | }, |