| ... | @@ -20,7 +20,6 @@ const InternPool = @import("../../InternPool.zig"); | ... | @@ -20,7 +20,6 @@ const InternPool = @import("../../InternPool.zig"); |
| 20 | const Compilation = @import("../../Compilation.zig"); | 20 | const Compilation = @import("../../Compilation.zig"); |
| 21 | const trace = @import("../../tracy.zig").trace; | 21 | const trace = @import("../../tracy.zig").trace; |
| 22 | const codegen = @import("../../codegen.zig"); | 22 | const codegen = @import("../../codegen.zig"); |
| 23 | const Mnemonic = @import("mnem.zig").Mnemonic; | | |
| 24 | | 23 | |
| 25 | const ErrorMsg = Zcu.ErrorMsg; | 24 | const ErrorMsg = Zcu.ErrorMsg; |
| 26 | const Target = std.Target; | 25 | const Target = std.Target; |
| ... | @@ -38,6 +37,10 @@ const DebugInfoOutput = codegen.DebugInfoOutput; | ... | @@ -38,6 +37,10 @@ const DebugInfoOutput = codegen.DebugInfoOutput; |
| 38 | const bits = @import("bits.zig"); | 37 | const bits = @import("bits.zig"); |
| 39 | const abi = @import("abi.zig"); | 38 | const abi = @import("abi.zig"); |
| 40 | const Lower = @import("Lower.zig"); | 39 | const Lower = @import("Lower.zig"); |
| | 40 | const mnem_import = @import("mnem.zig"); |
| | 41 | const Mnemonic = mnem_import.Mnemonic; |
| | 42 | const Pseudo = mnem_import.Pseudo; |
| | 43 | const encoding = @import("encoding.zig"); |
| 41 | | 44 | |
| 42 | const Register = bits.Register; | 45 | const Register = bits.Register; |
| 43 | const CSR = bits.CSR; | 46 | const CSR = bits.CSR; |
| ... | @@ -46,6 +49,7 @@ const Memory = bits.Memory; | ... | @@ -46,6 +49,7 @@ const Memory = bits.Memory; |
| 46 | const FrameIndex = bits.FrameIndex; | 49 | const FrameIndex = bits.FrameIndex; |
| 47 | const RegisterManager = abi.RegisterManager; | 50 | const RegisterManager = abi.RegisterManager; |
| 48 | const RegisterLock = RegisterManager.RegisterLock; | 51 | const RegisterLock = RegisterManager.RegisterLock; |
| | 52 | const Instruction = encoding.Instruction; |
| 49 | | 53 | |
| 50 | const InnerError = CodeGenError || error{OutOfRegisters}; | 54 | const InnerError = CodeGenError || error{OutOfRegisters}; |
| 51 | | 55 | |
| ... | @@ -3858,8 +3862,55 @@ fn airArrayElemVal(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -3858,8 +3862,55 @@ fn airArrayElemVal(func: *Func, inst: Air.Inst.Index) !void { |
| 3858 | | 3862 | |
| 3859 | fn airPtrElemVal(func: *Func, inst: Air.Inst.Index) !void { | 3863 | fn airPtrElemVal(func: *Func, inst: Air.Inst.Index) !void { |
| 3860 | const is_volatile = false; // TODO | 3864 | const is_volatile = false; // TODO |
| | 3865 | const pt = func.pt; |
| | 3866 | const zcu = pt.zcu; |
| 3861 | const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | 3867 | const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 3862 | const result: MCValue = if (!is_volatile and func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement ptr_elem_val for {}", .{func.target.cpu.arch}); | 3868 | const base_ptr_ty = func.typeOf(bin_op.lhs); |
| | 3869 | |
| | 3870 | const result: MCValue = if (!is_volatile and func.liveness.isUnused(inst)) .unreach else result: { |
| | 3871 | const elem_ty = base_ptr_ty.elemType2(zcu); |
| | 3872 | if (!elem_ty.hasRuntimeBitsIgnoreComptime(pt)) break :result .none; |
| | 3873 | |
| | 3874 | const base_ptr_mcv = try func.resolveInst(bin_op.lhs); |
| | 3875 | const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) { |
| | 3876 | .register => |reg| func.register_manager.lockRegAssumeUnused(reg), |
| | 3877 | else => null, |
| | 3878 | }; |
| | 3879 | defer if (base_ptr_lock) |lock| func.register_manager.unlockReg(lock); |
| | 3880 | |
| | 3881 | const index_mcv = try func.resolveInst(bin_op.rhs); |
| | 3882 | const index_lock: ?RegisterLock = switch (index_mcv) { |
| | 3883 | .register => |reg| func.register_manager.lockRegAssumeUnused(reg), |
| | 3884 | else => null, |
| | 3885 | }; |
| | 3886 | defer if (index_lock) |lock| func.register_manager.unlockReg(lock); |
| | 3887 | |
| | 3888 | const elem_ptr_reg = if (base_ptr_mcv.isRegister() and func.liveness.operandDies(inst, 0)) |
| | 3889 | base_ptr_mcv.register |
| | 3890 | else |
| | 3891 | try func.copyToTmpRegister(base_ptr_ty, base_ptr_mcv); |
| | 3892 | const elem_ptr_lock = func.register_manager.lockRegAssumeUnused(elem_ptr_reg); |
| | 3893 | defer func.register_manager.unlockReg(elem_ptr_lock); |
| | 3894 | |
| | 3895 | try func.genBinOp( |
| | 3896 | .ptr_add, |
| | 3897 | base_ptr_mcv, |
| | 3898 | base_ptr_ty, |
| | 3899 | index_mcv, |
| | 3900 | Type.u64, |
| | 3901 | elem_ptr_reg, |
| | 3902 | ); |
| | 3903 | |
| | 3904 | const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true); |
| | 3905 | const dst_lock = switch (dst_mcv) { |
| | 3906 | .register => |reg| func.register_manager.lockRegAssumeUnused(reg), |
| | 3907 | else => null, |
| | 3908 | }; |
| | 3909 | defer if (dst_lock) |lock| func.register_manager.unlockReg(lock); |
| | 3910 | |
| | 3911 | try func.load(dst_mcv, .{ .register = elem_ptr_reg }, base_ptr_ty); |
| | 3912 | break :result dst_mcv; |
| | 3913 | }; |
| 3863 | return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); | 3914 | return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 3864 | } | 3915 | } |
| 3865 | | 3916 | |
| ... | @@ -3873,13 +3924,6 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -3873,13 +3924,6 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void { |
| 3873 | const elem_ptr_ty = func.typeOfIndex(inst); | 3924 | const elem_ptr_ty = func.typeOfIndex(inst); |
| 3874 | const base_ptr_ty = func.typeOf(extra.lhs); | 3925 | const base_ptr_ty = func.typeOf(extra.lhs); |
| 3875 | | 3926 | |
| 3876 | const base_ptr_mcv = try func.resolveInst(extra.lhs); | | |
| 3877 | const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) { | | |
| 3878 | .register => |reg| func.register_manager.lockRegAssumeUnused(reg), | | |
| 3879 | else => null, | | |
| 3880 | }; | | |
| 3881 | defer if (base_ptr_lock) |lock| func.register_manager.unlockReg(lock); | | |
| 3882 | | | |
| 3883 | if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) { | 3927 | if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) { |
| 3884 | // break :result if (func.reuseOperand(inst, extra.lhs, 0, base_ptr_mcv)) | 3928 | // break :result if (func.reuseOperand(inst, extra.lhs, 0, base_ptr_mcv)) |
| 3885 | // base_ptr_mcv | 3929 | // base_ptr_mcv |
| ... | @@ -3888,6 +3932,13 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -3888,6 +3932,13 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void { |
| 3888 | @panic("audit"); | 3932 | @panic("audit"); |
| 3889 | } | 3933 | } |
| 3890 | | 3934 | |
| | 3935 | const base_ptr_mcv = try func.resolveInst(extra.lhs); |
| | 3936 | const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) { |
| | 3937 | .register => |reg| func.register_manager.lockRegAssumeUnused(reg), |
| | 3938 | else => null, |
| | 3939 | }; |
| | 3940 | defer if (base_ptr_lock) |lock| func.register_manager.unlockReg(lock); |
| | 3941 | |
| 3891 | const index_mcv = try func.resolveInst(extra.rhs); | 3942 | const index_mcv = try func.resolveInst(extra.rhs); |
| 3892 | const index_lock: ?RegisterLock = switch (index_mcv) { | 3943 | const index_lock: ?RegisterLock = switch (index_mcv) { |
| 3893 | .register => |reg| func.register_manager.lockRegAssumeUnused(reg), | 3944 | .register => |reg| func.register_manager.lockRegAssumeUnused(reg), |
| ... | @@ -4392,15 +4443,16 @@ fn airStore(func: *Func, inst: Air.Inst.Index, safety: bool) !void { | ... | @@ -4392,15 +4443,16 @@ fn airStore(func: *Func, inst: Air.Inst.Index, safety: bool) !void { |
| 4392 | const ptr = try func.resolveInst(bin_op.lhs); | 4443 | const ptr = try func.resolveInst(bin_op.lhs); |
| 4393 | const value = try func.resolveInst(bin_op.rhs); | 4444 | const value = try func.resolveInst(bin_op.rhs); |
| 4394 | const ptr_ty = func.typeOf(bin_op.lhs); | 4445 | const ptr_ty = func.typeOf(bin_op.lhs); |
| 4395 | const value_ty = func.typeOf(bin_op.rhs); | | |
| 4396 | | 4446 | |
| 4397 | try func.store(ptr, value, ptr_ty, value_ty); | 4447 | try func.store(ptr, value, ptr_ty); |
| 4398 | | 4448 | |
| 4399 | return func.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); | 4449 | return func.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 4400 | } | 4450 | } |
| 4401 | | 4451 | |
| 4402 | /// Loads `value` into the "payload" of `pointer`. | 4452 | /// Loads `value` into the "payload" of `pointer`. |
| 4403 | fn store(func: *Func, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type, src_ty: Type) !void { | 4453 | fn store(func: *Func, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type) !void { |
| | 4454 | const zcu = func.pt.zcu; |
| | 4455 | const src_ty = ptr_ty.childType(zcu); |
| 4404 | log.debug("storing {}:{} in {}:{}", .{ src_mcv, src_ty.fmt(func.pt), ptr_mcv, ptr_ty.fmt(func.pt) }); | 4456 | log.debug("storing {}:{} in {}:{}", .{ src_mcv, src_ty.fmt(func.pt), ptr_mcv, ptr_ty.fmt(func.pt) }); |
| 4405 | | 4457 | |
| 4406 | switch (ptr_mcv) { | 4458 | switch (ptr_mcv) { |
| ... | @@ -4429,7 +4481,7 @@ fn store(func: *Func, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type, src_ty: | ... | @@ -4429,7 +4481,7 @@ fn store(func: *Func, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type, src_ty: |
| 4429 | | 4481 | |
| 4430 | try func.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv); | 4482 | try func.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv); |
| 4431 | }, | 4483 | }, |
| 4432 | .air_ref => |ptr_ref| try func.store(try func.resolveInst(ptr_ref), src_mcv, ptr_ty, src_ty), | 4484 | .air_ref => |ptr_ref| try func.store(try func.resolveInst(ptr_ref), src_mcv, ptr_ty), |
| 4433 | } | 4485 | } |
| 4434 | } | 4486 | } |
| 4435 | | 4487 | |
| ... | @@ -5795,7 +5847,6 @@ fn airBoolOp(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -5795,7 +5847,6 @@ fn airBoolOp(func: *Func, inst: Air.Inst.Index) !void { |
| 5795 | fn airAsm(func: *Func, inst: Air.Inst.Index) !void { | 5847 | fn airAsm(func: *Func, inst: Air.Inst.Index) !void { |
| 5796 | const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | 5848 | const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 5797 | const extra = func.air.extraData(Air.Asm, ty_pl.payload); | 5849 | const extra = func.air.extraData(Air.Asm, ty_pl.payload); |
| 5798 | const is_volatile = @as(u1, @truncate(extra.data.flags >> 31)) != 0; | | |
| 5799 | const clobbers_len: u31 = @truncate(extra.data.flags); | 5850 | const clobbers_len: u31 = @truncate(extra.data.flags); |
| 5800 | var extra_i: usize = extra.end; | 5851 | var extra_i: usize = extra.end; |
| 5801 | const outputs: []const Air.Inst.Ref = | 5852 | const outputs: []const Air.Inst.Ref = |
| ... | @@ -5804,86 +5855,300 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -5804,86 +5855,300 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { |
| 5804 | const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra[extra_i..][0..extra.data.inputs_len]); | 5855 | const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra[extra_i..][0..extra.data.inputs_len]); |
| 5805 | extra_i += inputs.len; | 5856 | extra_i += inputs.len; |
| 5806 | | 5857 | |
| 5807 | const dead = !is_volatile and func.liveness.isUnused(inst); | 5858 | var result: MCValue = .none; |
| 5808 | const result: MCValue = if (dead) .unreach else result: { | 5859 | var args = std.ArrayList(MCValue).init(func.gpa); |
| 5809 | if (outputs.len > 1) { | 5860 | try args.ensureTotalCapacity(outputs.len + inputs.len); |
| 5810 | return func.fail("TODO implement codegen for asm with more than 1 output", .{}); | 5861 | defer { |
| 5811 | } | 5862 | for (args.items) |arg| if (arg.getReg()) |reg| func.register_manager.unlockReg(.{ |
| 5812 | | 5863 | .tracked_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue, |
| 5813 | const output_constraint: ?[]const u8 = for (outputs) |output| { | 5864 | }); |
| 5814 | if (output != .none) { | 5865 | args.deinit(); |
| 5815 | return func.fail("TODO implement codegen for non-expr asm", .{}); | 5866 | } |
| 5816 | } | 5867 | var arg_map = std.StringHashMap(u8).init(func.gpa); |
| 5817 | const extra_bytes = std.mem.sliceAsBytes(func.air.extra[extra_i..]); | 5868 | try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len)); |
| 5818 | const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(func.air.extra[extra_i..]), 0); | 5869 | defer arg_map.deinit(); |
| 5819 | const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); | 5870 | |
| 5820 | // This equation accounts for the fact that even if we have exactly 4 bytes | 5871 | var outputs_extra_i = extra_i; |
| 5821 | // for the string, we still use the next u32 for the null terminator. | 5872 | for (outputs) |output| { |
| 5822 | extra_i += (constraint.len + name.len + (2 + 3)) / 4; | 5873 | const extra_bytes = mem.sliceAsBytes(func.air.extra[extra_i..]); |
| | 5874 | const constraint = mem.sliceTo(mem.sliceAsBytes(func.air.extra[extra_i..]), 0); |
| | 5875 | const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); |
| | 5876 | // This equation accounts for the fact that even if we have exactly 4 bytes |
| | 5877 | // for the string, we still use the next u32 for the null terminator. |
| | 5878 | extra_i += (constraint.len + name.len + (2 + 3)) / 4; |
| | 5879 | |
| | 5880 | const is_read = switch (constraint[0]) { |
| | 5881 | '=' => false, |
| | 5882 | '+' => read: { |
| | 5883 | if (output == .none) return func.fail( |
| | 5884 | "read-write constraint unsupported for asm result: '{s}'", |
| | 5885 | .{constraint}, |
| | 5886 | ); |
| | 5887 | break :read true; |
| | 5888 | }, |
| | 5889 | else => return func.fail("invalid constraint: '{s}'", .{constraint}), |
| | 5890 | }; |
| | 5891 | const is_early_clobber = constraint[1] == '&'; |
| | 5892 | const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..]; |
| | 5893 | const arg_mcv: MCValue = arg_mcv: { |
| | 5894 | const arg_maybe_reg: ?Register = if (mem.eql(u8, rest, "m")) |
| | 5895 | if (output != .none) null else return func.fail( |
| | 5896 | "memory constraint unsupported for asm result: '{s}'", |
| | 5897 | .{constraint}, |
| | 5898 | ) |
| | 5899 | else if (mem.startsWith(u8, rest, "{") and mem.endsWith(u8, rest, "}")) |
| | 5900 | parseRegName(rest["{".len .. rest.len - "}".len]) orelse |
| | 5901 | return func.fail("invalid register constraint: '{s}'", .{constraint}) |
| | 5902 | else if (rest.len == 1 and std.ascii.isDigit(rest[0])) { |
| | 5903 | const index = std.fmt.charToDigit(rest[0], 10) catch unreachable; |
| | 5904 | if (index >= args.items.len) return func.fail("constraint out of bounds: '{s}'", .{ |
| | 5905 | constraint, |
| | 5906 | }); |
| | 5907 | break :arg_mcv args.items[index]; |
| | 5908 | } else return func.fail("invalid constraint: '{s}'", .{constraint}); |
| | 5909 | break :arg_mcv if (arg_maybe_reg) |reg| .{ .register = reg } else arg: { |
| | 5910 | const ptr_mcv = try func.resolveInst(output); |
| | 5911 | switch (ptr_mcv) { |
| | 5912 | .immediate => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_| |
| | 5913 | break :arg ptr_mcv.deref(), |
| | 5914 | .register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(), |
| | 5915 | else => {}, |
| | 5916 | } |
| | 5917 | break :arg .{ .indirect = .{ .reg = try func.copyToTmpRegister(Type.usize, ptr_mcv) } }; |
| | 5918 | }; |
| | 5919 | }; |
| | 5920 | if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { |
| | 5921 | _ = func.register_manager.lockReg(reg); |
| | 5922 | }; |
| | 5923 | if (!mem.eql(u8, name, "_")) |
| | 5924 | arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); |
| | 5925 | args.appendAssumeCapacity(arg_mcv); |
| | 5926 | if (output == .none) result = arg_mcv; |
| | 5927 | if (is_read) try func.load(arg_mcv, .{ .air_ref = output }, func.typeOf(output)); |
| | 5928 | } |
| 5823 | | 5929 | |
| 5824 | break constraint; | 5930 | for (inputs) |input| { |
| 5825 | } else null; | 5931 | const input_bytes = mem.sliceAsBytes(func.air.extra[extra_i..]); |
| | 5932 | const constraint = mem.sliceTo(input_bytes, 0); |
| | 5933 | const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); |
| | 5934 | // This equation accounts for the fact that even if we have exactly 4 bytes |
| | 5935 | // for the string, we still use the next u32 for the null terminator. |
| | 5936 | extra_i += (constraint.len + name.len + (2 + 3)) / 4; |
| | 5937 | |
| | 5938 | const ty = func.typeOf(input); |
| | 5939 | const input_mcv = try func.resolveInst(input); |
| | 5940 | const arg_mcv: MCValue = if (mem.eql(u8, constraint, "X")) |
| | 5941 | input_mcv |
| | 5942 | else if (mem.startsWith(u8, constraint, "{") and mem.endsWith(u8, constraint, "}")) arg: { |
| | 5943 | const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse |
| | 5944 | return func.fail("invalid register constraint: '{s}'", .{constraint}); |
| | 5945 | try func.register_manager.getReg(reg, null); |
| | 5946 | try func.genSetReg(ty, reg, input_mcv); |
| | 5947 | break :arg .{ .register = reg }; |
| | 5948 | } else return func.fail("invalid constraint: '{s}'", .{constraint}); |
| | 5949 | if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { |
| | 5950 | _ = func.register_manager.lockReg(reg); |
| | 5951 | }; |
| | 5952 | if (!mem.eql(u8, name, "_")) |
| | 5953 | arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); |
| | 5954 | args.appendAssumeCapacity(arg_mcv); |
| | 5955 | } |
| 5826 | | 5956 | |
| 5827 | for (inputs) |input| { | 5957 | { |
| 5828 | const input_bytes = std.mem.sliceAsBytes(func.air.extra[extra_i..]); | 5958 | var clobber_i: u32 = 0; |
| 5829 | const constraint = std.mem.sliceTo(input_bytes, 0); | 5959 | while (clobber_i < clobbers_len) : (clobber_i += 1) { |
| 5830 | const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); | 5960 | const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(func.air.extra[extra_i..]), 0); |
| 5831 | // This equation accounts for the fact that even if we have exactly 4 bytes | 5961 | // This equation accounts for the fact that even if we have exactly 4 bytes |
| 5832 | // for the string, we still use the next u32 for the null terminator. | 5962 | // for the string, we still use the next u32 for the null terminator. |
| 5833 | extra_i += (constraint.len + name.len + (2 + 3)) / 4; | 5963 | extra_i += clobber.len / 4 + 1; |
| 5834 | | 5964 | |
| 5835 | if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') { | 5965 | if (std.mem.eql(u8, clobber, "") or std.mem.eql(u8, clobber, "memory")) { |
| 5836 | return func.fail("unrecognized asm input constraint: '{s}'", .{constraint}); | 5966 | // nothing really to do |
| | 5967 | } else { |
| | 5968 | try func.register_manager.getReg(parseRegName(clobber) orelse |
| | 5969 | return func.fail("invalid clobber: '{s}'", .{clobber}), null); |
| 5837 | } | 5970 | } |
| 5838 | const reg_name = constraint[1 .. constraint.len - 1]; | | |
| 5839 | const reg = parseRegName(reg_name) orelse | | |
| 5840 | return func.fail("unrecognized register: '{s}'", .{reg_name}); | | |
| 5841 | | | |
| 5842 | const arg_mcv = try func.resolveInst(input); | | |
| 5843 | try func.register_manager.getReg(reg, null); | | |
| 5844 | try func.genSetReg(func.typeOf(input), reg, arg_mcv); | | |
| 5845 | } | 5971 | } |
| | 5972 | } |
| 5846 | | 5973 | |
| 5847 | { | 5974 | const asm_source = std.mem.sliceAsBytes(func.air.extra[extra_i..])[0..extra.data.source_len]; |
| 5848 | var clobber_i: u32 = 0; | 5975 | var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;"); |
| 5849 | while (clobber_i < clobbers_len) : (clobber_i += 1) { | 5976 | next_line: while (line_it.next()) |line| { |
| 5850 | const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(func.air.extra[extra_i..]), 0); | 5977 | var mnem_it = mem.tokenizeAny(u8, line, " \t"); |
| 5851 | // This equation accounts for the fact that even if we have exactly 4 bytes | 5978 | const instruction: union(enum) { mnem: Mnemonic, pseudo: Pseudo } = while (mnem_it.next()) |mnem_str| { |
| 5852 | // for the string, we still use the next u32 for the null terminator. | 5979 | if (mem.startsWith(u8, mnem_str, "#")) continue :next_line; |
| 5853 | extra_i += clobber.len / 4 + 1; | 5980 | if (mem.startsWith(u8, mnem_str, "//")) continue :next_line; |
| | 5981 | if (std.meta.stringToEnum(Mnemonic, mnem_str)) |mnem| { |
| | 5982 | break .{ .mnem = mnem }; |
| | 5983 | } else if (std.meta.stringToEnum(Pseudo, mnem_str)) |pseudo| { |
| | 5984 | break .{ .pseudo = pseudo }; |
| | 5985 | } else return func.fail("TODO: airAsm labels, found '{s}'", .{mnem_str}); |
| | 5986 | } else continue; |
| | 5987 | |
| | 5988 | const Operand = union(enum) { |
| | 5989 | none, |
| | 5990 | reg: Register, |
| | 5991 | imm: Immediate, |
| | 5992 | sym: SymbolOffset, |
| | 5993 | }; |
| 5854 | | 5994 | |
| 5855 | if (std.mem.eql(u8, clobber, "") or std.mem.eql(u8, clobber, "memory")) { | 5995 | var ops: [4]Operand = .{.none} ** 4; |
| 5856 | // nothing really to do | 5996 | var last_op = false; |
| 5857 | } else { | 5997 | var op_it = mem.splitScalar(u8, mnem_it.rest(), ','); |
| 5858 | try func.register_manager.getReg(parseRegName(clobber) orelse | 5998 | next_op: for (&ops) |*op| { |
| 5859 | return func.fail("invalid clobber: '{s}'", .{clobber}), null); | 5999 | const op_str = while (!last_op) { |
| 5860 | } | 6000 | const full_str = op_it.next() orelse break :next_op; |
| 5861 | } | 6001 | const code_str = if (mem.indexOfScalar(u8, full_str, '#') orelse |
| 5862 | } | 6002 | mem.indexOf(u8, full_str, "//")) |comment| |
| | 6003 | code: { |
| | 6004 | last_op = true; |
| | 6005 | break :code full_str[0..comment]; |
| | 6006 | } else full_str; |
| | 6007 | const trim_str = mem.trim(u8, code_str, " \t*"); |
| | 6008 | if (trim_str.len > 0) break trim_str; |
| | 6009 | } else break; |
| | 6010 | |
| | 6011 | if (parseRegName(op_str)) |reg| { |
| | 6012 | op.* = .{ .reg = reg }; |
| | 6013 | } else if (std.fmt.parseInt(i12, op_str, 10)) |int| { |
| | 6014 | op.* = .{ .imm = Immediate.s(int) }; |
| | 6015 | } else |_| if (mem.startsWith(u8, op_str, "%[")) { |
| | 6016 | const mod_index = mem.indexOf(u8, op_str, "]@"); |
| | 6017 | const modifier = if (mod_index) |index| |
| | 6018 | op_str[index + "]@".len ..] |
| | 6019 | else |
| | 6020 | ""; |
| | 6021 | |
| | 6022 | op.* = switch (args.items[ |
| | 6023 | arg_map.get(op_str["%[".len .. mod_index orelse op_str.len - "]".len]) orelse |
| | 6024 | return func.fail("no matching constraint: '{s}'", .{op_str}) |
| | 6025 | ]) { |
| | 6026 | .load_symbol => |sym_off| if (mem.eql(u8, modifier, "plt")) blk: { |
| | 6027 | assert(sym_off.off == 0); |
| | 6028 | break :blk .{ .sym = sym_off }; |
| | 6029 | } else return func.fail("invalid modifier: '{s}'", .{modifier}), |
| | 6030 | else => return func.fail("invalid constraint: '{s}'", .{op_str}), |
| | 6031 | }; |
| | 6032 | } else return func.fail("invalid operand: '{s}'", .{op_str}); |
| | 6033 | } else if (op_it.next()) |op_str| return func.fail("extra operand: '{s}'", .{op_str}); |
| 5863 | | 6034 | |
| 5864 | const asm_source = std.mem.sliceAsBytes(func.air.extra[extra_i..])[0..extra.data.source_len]; | 6035 | switch (instruction) { |
| | 6036 | .mnem => |mnem| { |
| | 6037 | _ = (switch (ops[0]) { |
| | 6038 | .none => try func.addInst(.{ |
| | 6039 | .tag = mnem, |
| | 6040 | .data = .none, |
| | 6041 | }), |
| | 6042 | .reg => |reg1| switch (ops[1]) { |
| | 6043 | .reg => |reg2| switch (ops[2]) { |
| | 6044 | .imm => |imm1| try func.addInst(.{ |
| | 6045 | .tag = mnem, |
| | 6046 | .data = .{ .i_type = .{ |
| | 6047 | .rd = reg1, |
| | 6048 | .rs1 = reg2, |
| | 6049 | .imm12 = imm1, |
| | 6050 | } }, |
| | 6051 | }), |
| | 6052 | else => error.InvalidInstruction, |
| | 6053 | }, |
| | 6054 | else => error.InvalidInstruction, |
| | 6055 | }, |
| | 6056 | else => error.InvalidInstruction, |
| | 6057 | }) catch |err| { |
| | 6058 | switch (err) { |
| | 6059 | error.InvalidInstruction => return func.fail( |
| | 6060 | "invalid instruction: {s} {s} {s} {s} {s}", |
| | 6061 | .{ |
| | 6062 | @tagName(mnem), |
| | 6063 | @tagName(ops[0]), |
| | 6064 | @tagName(ops[1]), |
| | 6065 | @tagName(ops[2]), |
| | 6066 | @tagName(ops[3]), |
| | 6067 | }, |
| | 6068 | ), |
| | 6069 | else => |e| return e, |
| | 6070 | } |
| | 6071 | }; |
| | 6072 | }, |
| | 6073 | .pseudo => |pseudo| { |
| | 6074 | (@as(error{InvalidInstruction}!void, switch (pseudo) { |
| | 6075 | .li => blk: { |
| | 6076 | if (ops[0] != .reg or ops[1] != .imm) { |
| | 6077 | break :blk error.InvalidInstruction; |
| | 6078 | } |
| 5865 | | 6079 | |
| 5866 | if (std.meta.stringToEnum(Mnemonic, asm_source)) |tag| { | 6080 | const reg = ops[0].reg; |
| 5867 | _ = try func.addInst(.{ | 6081 | const imm = ops[1].imm; |
| 5868 | .tag = tag, | | |
| 5869 | .data = .none, | | |
| 5870 | }); | | |
| 5871 | } else { | | |
| 5872 | return func.fail("TODO: asm_source {s}", .{asm_source}); | | |
| 5873 | } | | |
| 5874 | | 6082 | |
| 5875 | if (output_constraint) |output| { | 6083 | try func.genSetReg(Type.usize, reg, .{ .immediate = imm.asBits(u64) }); |
| 5876 | if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') { | 6084 | }, |
| 5877 | return func.fail("unrecognized asm output constraint: '{s}'", .{output}); | 6085 | .mv => blk: { |
| 5878 | } | 6086 | if (ops[0] != .reg or ops[1] != .reg) { |
| 5879 | const reg_name = output[2 .. output.len - 1]; | 6087 | break :blk error.InvalidInstruction; |
| 5880 | const reg = parseRegName(reg_name) orelse | 6088 | } |
| 5881 | return func.fail("unrecognized register: '{s}'", .{reg_name}); | 6089 | |
| 5882 | break :result .{ .register = reg }; | 6090 | const dst = ops[0].reg; |
| 5883 | } else { | 6091 | const src = ops[1].reg; |
| 5884 | break :result .{ .none = {} }; | 6092 | |
| | 6093 | if (dst.class() != .int or src.class() != .int) { |
| | 6094 | return func.fail("pseudo instruction 'mv' only works on integer registers", .{}); |
| | 6095 | } |
| | 6096 | |
| | 6097 | try func.genSetReg(Type.usize, dst, .{ .register = src }); |
| | 6098 | }, |
| | 6099 | .tail => blk: { |
| | 6100 | if (ops[0] != .sym) { |
| | 6101 | break :blk error.InvalidInstruction; |
| | 6102 | } |
| | 6103 | |
| | 6104 | const sym_offset = ops[0].sym; |
| | 6105 | assert(sym_offset.off == 0); |
| | 6106 | |
| | 6107 | const random_link_reg, const lock = try func.allocReg(.int); |
| | 6108 | defer func.register_manager.unlockReg(lock); |
| | 6109 | |
| | 6110 | _ = try func.addInst(.{ |
| | 6111 | .tag = .pseudo_extern_fn_reloc, |
| | 6112 | .data = .{ .reloc = .{ |
| | 6113 | .register = random_link_reg, |
| | 6114 | .atom_index = try func.owner.getSymbolIndex(func), |
| | 6115 | .sym_index = sym_offset.sym, |
| | 6116 | } }, |
| | 6117 | }); |
| | 6118 | }, |
| | 6119 | })) catch |err| { |
| | 6120 | switch (err) { |
| | 6121 | error.InvalidInstruction => return func.fail( |
| | 6122 | "invalid instruction: {s} {s} {s} {s} {s}", |
| | 6123 | .{ |
| | 6124 | @tagName(pseudo), |
| | 6125 | @tagName(ops[0]), |
| | 6126 | @tagName(ops[1]), |
| | 6127 | @tagName(ops[2]), |
| | 6128 | @tagName(ops[3]), |
| | 6129 | }, |
| | 6130 | ), |
| | 6131 | else => |e| return e, |
| | 6132 | } |
| | 6133 | }; |
| | 6134 | }, |
| 5885 | } | 6135 | } |
| 5886 | }; | 6136 | } |
| | 6137 | |
| | 6138 | for (outputs, args.items[0..outputs.len]) |output, arg_mcv| { |
| | 6139 | const extra_bytes = mem.sliceAsBytes(func.air.extra[outputs_extra_i..]); |
| | 6140 | const constraint = |
| | 6141 | mem.sliceTo(mem.sliceAsBytes(func.air.extra[outputs_extra_i..]), 0); |
| | 6142 | const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); |
| | 6143 | // This equation accounts for the fact that even if we have exactly 4 bytes |
| | 6144 | // for the string, we still use the next u32 for the null terminator. |
| | 6145 | outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4; |
| | 6146 | |
| | 6147 | if (output == .none) continue; |
| | 6148 | if (arg_mcv != .register) continue; |
| | 6149 | if (constraint.len == 2 and std.ascii.isDigit(constraint[1])) continue; |
| | 6150 | try func.store(.{ .air_ref = output }, arg_mcv, func.typeOf(output)); |
| | 6151 | } |
| 5887 | | 6152 | |
| 5888 | simple: { | 6153 | simple: { |
| 5889 | var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); | 6154 | var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); |
| ... | @@ -6867,7 +7132,7 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -6867,7 +7132,7 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void { |
| 6867 | } | 7132 | } |
| 6868 | | 7133 | |
| 6869 | switch (val_size) { | 7134 | switch (val_size) { |
| 6870 | 1, 2 => return func.fail("TODO: airAtomicRmw Int {}", .{val_size}), | 7135 | 1, 2 => return func.fail("TODO: airAtomicRmw {s} Int {}", .{ @tagName(op), val_size }), |
| 6871 | 4, 8 => {}, | 7136 | 4, 8 => {}, |
| 6872 | else => unreachable, | 7137 | else => unreachable, |
| 6873 | } | 7138 | } |
| ... | @@ -6997,7 +7262,7 @@ fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOr | ... | @@ -6997,7 +7262,7 @@ fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOr |
| 6997 | else => unreachable, | 7262 | else => unreachable, |
| 6998 | } | 7263 | } |
| 6999 | | 7264 | |
| 7000 | try func.store(ptr_mcv, val_mcv, ptr_ty, val_ty); | 7265 | try func.store(ptr_mcv, val_mcv, ptr_ty); |
| 7001 | return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); | 7266 | return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 7002 | } | 7267 | } |
| 7003 | | 7268 | |
| ... | @@ -7061,7 +7326,7 @@ fn airMemset(func: *Func, inst: Air.Inst.Index, safety: bool) !void { | ... | @@ -7061,7 +7326,7 @@ fn airMemset(func: *Func, inst: Air.Inst.Index, safety: bool) !void { |
| 7061 | const len = dst_ptr_ty.childType(zcu).arrayLen(zcu); | 7326 | const len = dst_ptr_ty.childType(zcu).arrayLen(zcu); |
| 7062 | | 7327 | |
| 7063 | assert(len != 0); // prevented by Sema | 7328 | assert(len != 0); // prevented by Sema |
| 7064 | try func.store(dst_ptr, src_val, elem_ptr_ty, elem_ty); | 7329 | try func.store(dst_ptr, src_val, elem_ptr_ty); |
| 7065 | | 7330 | |
| 7066 | const second_elem_ptr_reg, const second_elem_ptr_lock = try func.allocReg(.int); | 7331 | const second_elem_ptr_reg, const second_elem_ptr_lock = try func.allocReg(.int); |
| 7067 | defer func.register_manager.unlockReg(second_elem_ptr_lock); | 7332 | defer func.register_manager.unlockReg(second_elem_ptr_lock); |
| ... | @@ -7108,6 +7373,10 @@ fn airMemcpy(func: *Func, inst: Air.Inst.Index) !void { | ... | @@ -7108,6 +7373,10 @@ fn airMemcpy(func: *Func, inst: Air.Inst.Index) !void { |
| 7108 | ); | 7373 | ); |
| 7109 | break :len .{ .register = len_reg }; | 7374 | break :len .{ .register = len_reg }; |
| 7110 | }, | 7375 | }, |
| | 7376 | .One => len: { |
| | 7377 | const array_ty = dst_ty.childType(zcu); |
| | 7378 | break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(pt) }; |
| | 7379 | }, |
| 7111 | else => |size| return func.fail("TODO: airMemcpy size {s}", .{@tagName(size)}), | 7380 | else => |size| return func.fail("TODO: airMemcpy size {s}", .{@tagName(size)}), |
| 7112 | }; | 7381 | }; |
| 7113 | const len_lock: ?RegisterLock = switch (len_mcv) { | 7382 | const len_lock: ?RegisterLock = switch (len_mcv) { |