authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-10-07 06:17:54-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-10-07 16:02:01-04:00
logb19fd485b185743907c71cedc0cc3a87ffcce27b
treec9e798991112c855563c815072e5bd84df8a49a6
parent7436f3efc9fd3337431dce16ca12393819394dc1

x86_64: improve inline assembly support

* instruction prefixes * mnemonic fixes * labels * memory operands * read-write constraint modifier * register and memory alternative constraint

2 files changed, 248 insertions(+), 72 deletions(-)

src/arch/x86_64/CodeGen.zig+246-68
...@@ -32,10 +32,10 @@ const Target = std.Target;...@@ -32,10 +32,10 @@ const Target = std.Target;
32const Type = @import("../../type.zig").Type;32const Type = @import("../../type.zig").Type;
33const TypedValue = @import("../../TypedValue.zig");33const TypedValue = @import("../../TypedValue.zig");
34const Value = @import("../../value.zig").Value;34const Value = @import("../../value.zig").Value;
35const Instruction = @import("encoder.zig").Instruction;
3536
36const abi = @import("abi.zig");37const abi = @import("abi.zig");
37const bits = @import("bits.zig");38const bits = @import("bits.zig");
38const encoder = @import("encoder.zig");
39const errUnionErrorOffset = codegen.errUnionErrorOffset;39const errUnionErrorOffset = codegen.errUnionErrorOffset;
40const errUnionPayloadOffset = codegen.errUnionPayloadOffset;40const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
4141
...@@ -1275,6 +1275,17 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins...@@ -1275,6 +1275,17 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins
1275 });1275 });
1276}1276}
12771277
1278fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
1279 _ = try self.addInst(.{
1280 .tag = tag[1],
1281 .ops = .inst,
1282 .data = .{ .inst = .{
1283 .fixes = tag[0],
1284 .inst = target,
1285 } },
1286 });
1287}
1288
1278fn asmPlaceholder(self: *Self) !Mir.Inst.Index {1289fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
1279 return self.addInst(.{1290 return self.addInst(.{
1280 .tag = .pseudo,1291 .tag = .pseudo,
...@@ -2174,7 +2185,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {...@@ -2174,7 +2185,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
2174 const ret_reg = param_regs[0];2185 const ret_reg = param_regs[0];
2175 const enum_mcv = MCValue{ .register = param_regs[1] };2186 const enum_mcv = MCValue{ .register = param_regs[1] };
21762187
2177 var exitlude_jump_relocs = try self.gpa.alloc(u32, enum_ty.enumFieldCount(mod));2188 var exitlude_jump_relocs = try self.gpa.alloc(Mir.Inst.Index, enum_ty.enumFieldCount(mod));
2178 defer self.gpa.free(exitlude_jump_relocs);2189 defer self.gpa.free(exitlude_jump_relocs);
21792190
2180 const data_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);2191 const data_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
...@@ -5887,7 +5898,7 @@ fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerErr...@@ -5887,7 +5898,7 @@ fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerErr
58875898
5888 try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);5899 try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
5889 },5900 },
5890 .air_ref => |ptr_ref| try self.store(ptr_ty, ptr_mcv, try self.resolveInst(ptr_ref)),5901 .air_ref => |ptr_ref| try self.store(ptr_ty, try self.resolveInst(ptr_ref), src_mcv),
5891 }5902 }
5892}5903}
58935904
...@@ -9313,7 +9324,7 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {...@@ -9313,7 +9324,7 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
9313 return self.finishAir(inst, .unreach, .{ operand, .none, .none });9324 return self.finishAir(inst, .unreach, .{ operand, .none, .none });
9314}9325}
93159326
9316fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {9327fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
9317 const mod = self.bin_file.options.module.?;9328 const mod = self.bin_file.options.module.?;
9318 const abi_size = ty.abiSize(mod);9329 const abi_size = ty.abiSize(mod);
9319 switch (mcv) {9330 switch (mcv) {
...@@ -9695,7 +9706,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {...@@ -9695,7 +9706,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
9695 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;9706 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
9696 const loop = self.air.extraData(Air.Block, ty_pl.payload);9707 const loop = self.air.extraData(Air.Block, ty_pl.payload);
9697 const body = self.air.extra[loop.end..][0..loop.data.body_len];9708 const body = self.air.extra[loop.end..][0..loop.data.body_len];
9698 const jmp_target: u32 = @intCast(self.mir_instructions.len);9709 const jmp_target: Mir.Inst.Index = @intCast(self.mir_instructions.len);
96999710
9700 self.scope_generation += 1;9711 self.scope_generation += 1;
9701 const state = try self.saveState();9712 const state = try self.saveState();
...@@ -9772,7 +9783,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -9772,7 +9783,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
9772 const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];9783 const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];
9773 extra_index = case.end + items.len + case_body.len;9784 extra_index = case.end + items.len + case_body.len;
97749785
9775 var relocs = try self.gpa.alloc(u32, items.len);9786 var relocs = try self.gpa.alloc(Mir.Inst.Index, items.len);
9776 defer self.gpa.free(relocs);9787 defer self.gpa.free(relocs);
97779788
9778 try self.spillEflagsIfOccupied();9789 try self.spillEflagsIfOccupied();
...@@ -9929,22 +9940,35 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -9929,22 +9940,35 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9929 .none => self.typeOfIndex(inst),9940 .none => self.typeOfIndex(inst),
9930 else => self.typeOf(output).childType(mod),9941 else => self.typeOf(output).childType(mod),
9931 };9942 };
9932 const arg_maybe_reg: ?Register = if (mem.eql(u8, constraint, "=r"))9943 const is_read = switch (constraint[0]) {
9944 '=' => false,
9945 '+' => read: {
9946 if (output == .none) return self.fail(
9947 "read-write constraint unsupported for asm result: '{s}'",
9948 .{constraint},
9949 );
9950 break :read true;
9951 },
9952 else => return self.fail("invalid constraint: '{s}'", .{constraint}),
9953 };
9954 const arg_maybe_reg: ?Register = if (mem.eql(u8, constraint[1..], "r"))
9933 self.register_manager.tryAllocReg(maybe_inst, self.regClassForType(ty)) orelse9955 self.register_manager.tryAllocReg(maybe_inst, self.regClassForType(ty)) orelse
9934 return self.fail("ran out of registers lowering inline asm", .{})9956 return self.fail("ran out of registers lowering inline asm", .{})
9935 else if (mem.eql(u8, constraint, "=m"))9957 else if (mem.eql(u8, constraint[1..], "m"))
9936 if (output != .none) null else return self.fail(9958 if (output != .none) null else return self.fail(
9937 "memory constraint unsupported for asm result",9959 "memory constraint unsupported for asm result: '{s}'",
9938 .{},9960 .{constraint},
9939 )9961 )
9940 else if (mem.eql(u8, constraint, "=g"))9962 else if (mem.eql(u8, constraint[1..], "g") or
9963 mem.eql(u8, constraint[1..], "rm") or mem.eql(u8, constraint[1..], "mr") or
9964 mem.eql(u8, constraint[1..], "r,m") or mem.eql(u8, constraint[1..], "m,r"))
9941 self.register_manager.tryAllocReg(maybe_inst, self.regClassForType(ty)) orelse9965 self.register_manager.tryAllocReg(maybe_inst, self.regClassForType(ty)) orelse
9942 if (output != .none) null else return self.fail(9966 if (output != .none)
9943 "ran out of register lowering inline asm",9967 null
9944 .{},9968 else
9945 )9969 return self.fail("ran out of registers lowering inline asm", .{})
9946 else if (mem.startsWith(u8, constraint, "={") and mem.endsWith(u8, constraint, "}"))9970 else if (mem.startsWith(u8, constraint[1..], "{") and mem.endsWith(u8, constraint[1..], "}"))
9947 parseRegName(constraint["={".len .. constraint.len - "}".len]) orelse9971 parseRegName(constraint[1 + "{".len .. constraint.len - "}".len]) orelse
9948 return self.fail("invalid register constraint: '{s}'", .{constraint})9972 return self.fail("invalid register constraint: '{s}'", .{constraint})
9949 else9973 else
9950 return self.fail("invalid constraint: '{s}'", .{constraint});9974 return self.fail("invalid constraint: '{s}'", .{constraint});
...@@ -9965,6 +9989,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -9965,6 +9989,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9965 arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));9989 arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
9966 args.appendAssumeCapacity(arg_mcv);9990 args.appendAssumeCapacity(arg_mcv);
9967 if (output == .none) result = arg_mcv;9991 if (output == .none) result = arg_mcv;
9992 if (is_read) try self.load(arg_mcv, self.typeOf(output), .{ .air_ref = output });
9968 }9993 }
99699994
9970 for (inputs) |input| {9995 for (inputs) |input| {
...@@ -9999,7 +10024,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -9999,7 +10024,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9999 };10024 };
10000 try self.genSetReg(addr_reg, Type.usize, input_mcv.address());10025 try self.genSetReg(addr_reg, Type.usize, input_mcv.address());
10001 break :arg .{ .indirect = .{ .reg = addr_reg } };10026 break :arg .{ .indirect = .{ .reg = addr_reg } };
10002 } else if (mem.eql(u8, constraint, "g")) arg: {10027 } else if (mem.eql(u8, constraint, "g") or
10028 mem.eql(u8, constraint, "rm") or mem.eql(u8, constraint, "mr") or
10029 mem.eql(u8, constraint, "r,m") or mem.eql(u8, constraint, "m,r"))
10030 arg: {
10003 switch (input_mcv) {10031 switch (input_mcv) {
10004 .register, .indirect, .load_frame => break :arg input_mcv,10032 .register, .indirect, .load_frame => break :arg input_mcv,
10005 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_|10033 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_|
...@@ -10038,44 +10066,141 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -10038,44 +10066,141 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
10038 }10066 }
10039 }10067 }
1004010068
10069 const Label = struct {
10070 target: Mir.Inst.Index = undefined,
10071 pending_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{},
10072
10073 const Kind = enum { definition, reference };
10074
10075 fn isValid(kind: Kind, name: []const u8) bool {
10076 for (name, 0..) |c, i| switch (c) {
10077 else => return false,
10078 '$' => if (i == 0) return false,
10079 '.' => {},
10080 '0'...'9' => if (i == 0) switch (kind) {
10081 .definition => if (name.len != 1) return false,
10082 .reference => {
10083 if (name.len != 2) return false;
10084 switch (name[1]) {
10085 else => return false,
10086 'B', 'F', 'b', 'f' => {},
10087 }
10088 },
10089 },
10090 '@', 'A'...'Z', '_', 'a'...'z' => {},
10091 };
10092 return name.len > 0;
10093 }
10094 };
10095 var labels: std.StringHashMapUnmanaged(Label) = .{};
10096 defer {
10097 var label_it = labels.valueIterator();
10098 while (label_it.next()) |label| label.pending_relocs.deinit(self.gpa);
10099 labels.deinit(self.gpa);
10100 }
10101
10041 const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];10102 const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
10042 var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");10103 var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");
10043 while (line_it.next()) |line| {10104 next_line: while (line_it.next()) |line| {
10044 var mnem_it = mem.tokenizeAny(u8, line, " \t");10105 var mnem_it = mem.tokenizeAny(u8, line, " \t");
10045 const mnem_str = mnem_it.next() orelse continue;10106 var prefix: Instruction.Prefix = .none;
10046 if (mem.startsWith(u8, mnem_str, "#")) continue;10107 const mnem_str = while (mnem_it.next()) |mnem_str| {
1004710108 if (mem.startsWith(u8, mnem_str, "#")) continue :next_line;
10048 const mnem_size: ?Memory.PtrSize = if (mem.endsWith(u8, mnem_str, "b"))10109 if (std.meta.stringToEnum(Instruction.Prefix, mnem_str)) |pre| {
10049 .byte10110 if (prefix != .none) return self.fail("extra prefix: '{s}'", .{mnem_str});
10050 else if (mem.endsWith(u8, mnem_str, "w"))10111 prefix = pre;
10051 .word10112 continue;
10052 else if (mem.endsWith(u8, mnem_str, "l"))
10053 .dword
10054 else if (mem.endsWith(u8, mnem_str, "q"))
10055 .qword
10056 else
10057 null;
10058 const mnem_tag = Mir.Inst.FixedTag{ ._, mnem: {
10059 if (mnem_size) |_| {
10060 if (std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])) |mnem| {
10061 break :mnem mnem;
10062 }
10063 }10113 }
10064 break :mnem std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse10114 if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str;
10065 return self.fail("invalid mnemonic: '{s}'", .{mnem_str});10115 const label_name = mnem_str[0 .. mnem_str.len - ":".len];
10066 } };10116 if (!Label.isValid(.definition, label_name))
1006710117 return self.fail("invalid label: '{s}'", .{label_name});
10068 var op_it = mem.tokenizeScalar(u8, mnem_it.rest(), ',');10118 const label_gop = try labels.getOrPut(self.gpa, label_name);
10069 var ops = [1]encoder.Instruction.Operand{.none} ** 4;10119 if (!label_gop.found_existing) label_gop.value_ptr.* = .{} else {
10070 for (&ops) |*op| {10120 const anon = std.ascii.isDigit(label_name[0]);
10071 const op_str = mem.trim(u8, op_it.next() orelse break, " \t");10121 if (!anon and label_gop.value_ptr.pending_relocs.items.len == 0)
10072 if (mem.startsWith(u8, op_str, "#")) break;10122 return self.fail("redefined label: '{s}'", .{label_name});
10123 for (label_gop.value_ptr.pending_relocs.items) |pending_reloc|
10124 try self.performReloc(pending_reloc);
10125 if (anon)
10126 label_gop.value_ptr.pending_relocs.clearRetainingCapacity()
10127 else
10128 label_gop.value_ptr.pending_relocs.clearAndFree(self.gpa);
10129 }
10130 label_gop.value_ptr.target = @intCast(self.mir_instructions.len);
10131 } else continue;
10132
10133 var mnem_size: ?Memory.PtrSize = null;
10134 const mnem_tag = mnem: {
10135 mnem_size = if (mem.endsWith(u8, mnem_str, "b"))
10136 .byte
10137 else if (mem.endsWith(u8, mnem_str, "w"))
10138 .word
10139 else if (mem.endsWith(u8, mnem_str, "l"))
10140 .dword
10141 else if (mem.endsWith(u8, mnem_str, "q"))
10142 .qword
10143 else if (mem.endsWith(u8, mnem_str, "t"))
10144 .tbyte
10145 else
10146 break :mnem null;
10147 break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str[0 .. mnem_str.len - 1]);
10148 } orelse mnem: {
10149 mnem_size = null;
10150 break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str);
10151 } orelse return self.fail("invalid mnemonic: '{s}'", .{mnem_str});
10152 const mnem_name = @tagName(mnem_tag);
10153 const mnem_fixed_tag: Mir.Inst.FixedTag = for (std.enums.values(Mir.Inst.Fixes)) |fixes| {
10154 const fixes_name = @tagName(fixes);
10155 const space_i = mem.indexOfScalar(u8, fixes_name, ' ');
10156 const fixes_prefix = if (space_i) |i|
10157 std.meta.stringToEnum(Instruction.Prefix, fixes_name[0..i]).?
10158 else
10159 .none;
10160 if (fixes_prefix != prefix) continue;
10161 const pattern = fixes_name[if (space_i) |i| i + " ".len else 0..];
10162 const wildcard_i = mem.indexOfScalar(u8, pattern, '_').?;
10163 const mnem_prefix = pattern[0..wildcard_i];
10164 const mnem_suffix = pattern[wildcard_i + "_".len ..];
10165 if (!mem.startsWith(u8, mnem_name, mnem_prefix)) continue;
10166 if (!mem.endsWith(u8, mnem_name, mnem_suffix)) continue;
10167 break .{ fixes, std.meta.stringToEnum(
10168 Mir.Inst.Tag,
10169 mnem_name[mnem_prefix.len .. mnem_name.len - mnem_suffix.len],
10170 ) orelse continue };
10171 } else {
10172 assert(prefix != .none);
10173 return self.fail("invalid prefix for mnemonic: '{s} {s}'", .{
10174 @tagName(prefix), mnem_str,
10175 });
10176 };
10177
10178 const Operand = union(enum) {
10179 none,
10180 reg: Register,
10181 mem: Memory,
10182 imm: Immediate,
10183 inst: Mir.Inst.Index,
10184 };
10185 var ops: [4]Operand = .{.none} ** 4;
10186
10187 var last_op = false;
10188 var op_it = mem.splitScalar(u8, mnem_it.rest(), ',');
10189 next_op: for (&ops) |*op| {
10190 const op_str = while (!last_op) {
10191 const full_str = op_it.next() orelse break :next_op;
10192 const trim_str = mem.trim(u8, if (mem.indexOfScalar(u8, full_str, '#')) |hash| hash: {
10193 last_op = true;
10194 break :hash full_str[0..hash];
10195 } else full_str, " \t");
10196 if (trim_str.len > 0) break trim_str;
10197 } else break;
10073 if (mem.startsWith(u8, op_str, "%%")) {10198 if (mem.startsWith(u8, op_str, "%%")) {
10074 const colon = mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');10199 const colon = mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');
10075 const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse10200 const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse
10076 return self.fail("invalid register: '{s}'", .{op_str});10201 return self.fail("invalid register: '{s}'", .{op_str});
10077 if (colon) |colon_pos| {10202 if (colon) |colon_pos| {
10078 const disp = std.fmt.parseInt(i32, op_str[colon_pos + 1 ..], 0) catch10203 const disp = std.fmt.parseInt(i32, op_str[colon_pos + ":".len ..], 0) catch
10079 return self.fail("invalid displacement: '{s}'", .{op_str});10204 return self.fail("invalid displacement: '{s}'", .{op_str});
10080 op.* = .{ .mem = Memory.sib(10205 op.* = .{ .mem = Memory.sib(
10081 mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),10206 mnem_size orelse return self.fail("unknown size: '{s}'", .{op_str}),
...@@ -10089,7 +10214,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -10089,7 +10214,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
10089 } else if (mem.startsWith(u8, op_str, "%[") and mem.endsWith(u8, op_str, "]")) {10214 } else if (mem.startsWith(u8, op_str, "%[") and mem.endsWith(u8, op_str, "]")) {
10090 const colon = mem.indexOfScalarPos(u8, op_str, "%[".len, ':');10215 const colon = mem.indexOfScalarPos(u8, op_str, "%[".len, ':');
10091 const modifier = if (colon) |colon_pos|10216 const modifier = if (colon) |colon_pos|
10092 op_str[colon_pos + 1 .. op_str.len - "]".len]10217 op_str[colon_pos + ":".len .. op_str.len - "]".len]
10093 else10218 else
10094 "";10219 "";
10095 op.* = switch (args.items[10220 op.* = switch (args.items[
...@@ -10140,64 +10265,113 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -10140,64 +10265,113 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
10140 }10265 }
10141 op.* = .{ .imm = Immediate.u(u) };10266 op.* = .{ .imm = Immediate.u(u) };
10142 } else |_| return self.fail("invalid immediate: '{s}'", .{op_str});10267 } else |_| return self.fail("invalid immediate: '{s}'", .{op_str});
10268 } else if (mem.endsWith(u8, op_str, ")")) {
10269 const open = mem.indexOfScalar(u8, op_str, '(') orelse
10270 return self.fail("invalid operand: '{s}'", .{op_str});
10271 var sib_it = mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ',');
10272 const base_str = sib_it.next() orelse
10273 return self.fail("invalid memory operand: '{s}'", .{op_str});
10274 if (base_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))
10275 return self.fail("invalid memory operand: '{s}'", .{op_str});
10276 const index_str = sib_it.next() orelse "";
10277 if (index_str.len > 0 and !mem.startsWith(u8, base_str, "%%"))
10278 return self.fail("invalid memory operand: '{s}'", .{op_str});
10279 const scale_str = sib_it.next() orelse "";
10280 if (index_str.len == 0 and scale_str.len > 0)
10281 return self.fail("invalid memory operand: '{s}'", .{op_str});
10282 const scale = if (scale_str.len > 0) switch (std.fmt.parseInt(u4, scale_str, 10) catch
10283 return self.fail("invalid scale: '{s}'", .{op_str})) {
10284 1, 2, 4, 8 => |scale| scale,
10285 else => return self.fail("invalid scale: '{s}'", .{op_str}),
10286 } else 1;
10287 if (sib_it.next()) |_| return self.fail("invalid memory operand: '{s}'", .{op_str});
10288 op.* = .{ .mem = Memory.sib(mnem_size orelse
10289 return self.fail("unknown size: '{s}'", .{op_str}), .{
10290 .disp = if (open > 0) std.fmt.parseInt(i32, op_str[0..open], 0) catch
10291 return self.fail("invalid displacement: '{s}'", .{op_str}) else 0,
10292 .base = if (base_str.len > 0) .{ .reg = parseRegName(base_str["%%".len..]) orelse
10293 return self.fail("invalid base register: '{s}'", .{base_str}) } else .none,
10294 .scale_index = if (index_str.len > 0) .{
10295 .index = parseRegName(index_str["%%".len..]) orelse
10296 return self.fail("invalid index register: '{s}'", .{op_str}),
10297 .scale = scale,
10298 } else null,
10299 }) };
10300 } else if (Label.isValid(.reference, op_str)) {
10301 const anon = std.ascii.isDigit(op_str[0]);
10302 const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else op_str.len]);
10303 if (!label_gop.found_existing) label_gop.value_ptr.* = .{};
10304 if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing)
10305 return self.fail("undefined label: '{s}'", .{op_str});
10306 const pending_relocs = &label_gop.value_ptr.pending_relocs;
10307 if (if (anon)
10308 op_str[1] == 'f' or op_str[1] == 'F'
10309 else
10310 !label_gop.found_existing or pending_relocs.items.len > 0)
10311 try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len));
10312 op.* = .{ .inst = label_gop.value_ptr.target };
10143 } else return self.fail("invalid operand: '{s}'", .{op_str});10313 } else return self.fail("invalid operand: '{s}'", .{op_str});
10144 } else if (op_it.next()) |op_str| return self.fail("extra operand: '{s}'", .{op_str});10314 } else if (op_it.next()) |op_str| return self.fail("extra operand: '{s}'", .{op_str});
1014510315
10146 (switch (ops[0]) {10316 (switch (ops[0]) {
10147 .none => self.asmOpOnly(mnem_tag),10317 .none => self.asmOpOnly(mnem_fixed_tag),
10148 .reg => |reg0| switch (ops[1]) {10318 .reg => |reg0| switch (ops[1]) {
10149 .none => self.asmRegister(mnem_tag, reg0),10319 .none => self.asmRegister(mnem_fixed_tag, reg0),
10150 .reg => |reg1| switch (ops[2]) {10320 .reg => |reg1| switch (ops[2]) {
10151 .none => self.asmRegisterRegister(mnem_tag, reg1, reg0),10321 .none => self.asmRegisterRegister(mnem_fixed_tag, reg1, reg0),
10152 .reg => |reg2| switch (ops[3]) {10322 .reg => |reg2| switch (ops[3]) {
10153 .none => self.asmRegisterRegisterRegister(mnem_tag, reg2, reg1, reg0),10323 .none => self.asmRegisterRegisterRegister(mnem_fixed_tag, reg2, reg1, reg0),
10154 else => error.InvalidInstruction,10324 else => error.InvalidInstruction,
10155 },10325 },
10156 .mem => |mem2| switch (ops[3]) {10326 .mem => |mem2| switch (ops[3]) {
10157 .none => self.asmMemoryRegisterRegister(mnem_tag, mem2, reg1, reg0),10327 .none => self.asmMemoryRegisterRegister(mnem_fixed_tag, mem2, reg1, reg0),
10158 else => error.InvalidInstruction,10328 else => error.InvalidInstruction,
10159 },10329 },
10160 else => error.InvalidInstruction,10330 else => error.InvalidInstruction,
10161 },10331 },
10162 .mem => |mem1| switch (ops[2]) {10332 .mem => |mem1| switch (ops[2]) {
10163 .none => self.asmMemoryRegister(mnem_tag, mem1, reg0),10333 .none => self.asmMemoryRegister(mnem_fixed_tag, mem1, reg0),
10164 else => error.InvalidInstruction,10334 else => error.InvalidInstruction,
10165 },10335 },
10166 else => error.InvalidInstruction,10336 else => error.InvalidInstruction,
10167 },10337 },
10168 .mem => |mem0| switch (ops[1]) {10338 .mem => |mem0| switch (ops[1]) {
10169 .none => self.asmMemory(mnem_tag, mem0),10339 .none => self.asmMemory(mnem_fixed_tag, mem0),
10170 .reg => |reg1| switch (ops[2]) {10340 .reg => |reg1| switch (ops[2]) {
10171 .none => self.asmRegisterMemory(mnem_tag, reg1, mem0),10341 .none => self.asmRegisterMemory(mnem_fixed_tag, reg1, mem0),
10172 else => error.InvalidInstruction,10342 else => error.InvalidInstruction,
10173 },10343 },
10174 else => error.InvalidInstruction,10344 else => error.InvalidInstruction,
10175 },10345 },
10176 .imm => |imm0| switch (ops[1]) {10346 .imm => |imm0| switch (ops[1]) {
10177 .none => self.asmImmediate(mnem_tag, imm0),10347 .none => self.asmImmediate(mnem_fixed_tag, imm0),
10178 .reg => |reg1| switch (ops[2]) {10348 .reg => |reg1| switch (ops[2]) {
10179 .none => self.asmRegisterImmediate(mnem_tag, reg1, imm0),10349 .none => self.asmRegisterImmediate(mnem_fixed_tag, reg1, imm0),
10180 .reg => |reg2| switch (ops[3]) {10350 .reg => |reg2| switch (ops[3]) {
10181 .none => self.asmRegisterRegisterImmediate(mnem_tag, reg2, reg1, imm0),10351 .none => self.asmRegisterRegisterImmediate(mnem_fixed_tag, reg2, reg1, imm0),
10182 else => error.InvalidInstruction,10352 else => error.InvalidInstruction,
10183 },10353 },
10184 .mem => |mem2| switch (ops[3]) {10354 .mem => |mem2| switch (ops[3]) {
10185 .none => self.asmMemoryRegisterImmediate(mnem_tag, mem2, reg1, imm0),10355 .none => self.asmMemoryRegisterImmediate(mnem_fixed_tag, mem2, reg1, imm0),
10186 else => error.InvalidInstruction,10356 else => error.InvalidInstruction,
10187 },10357 },
10188 else => error.InvalidInstruction,10358 else => error.InvalidInstruction,
10189 },10359 },
10190 .mem => |mem1| switch (ops[2]) {10360 .mem => |mem1| switch (ops[2]) {
10191 .none => self.asmMemoryImmediate(mnem_tag, mem1, imm0),10361 .none => self.asmMemoryImmediate(mnem_fixed_tag, mem1, imm0),
10192 else => error.InvalidInstruction,10362 else => error.InvalidInstruction,
10193 },10363 },
10194 else => error.InvalidInstruction,10364 else => error.InvalidInstruction,
10195 },10365 },
10366 .inst => |inst0| switch (ops[1]) {
10367 .none => self.asmReloc(mnem_fixed_tag, inst0),
10368 else => error.InvalidInstruction,
10369 },
10196 }) catch |err| switch (err) {10370 }) catch |err| switch (err) {
10197 error.InvalidInstruction => return self.fail(10371 error.InvalidInstruction => return self.fail(
10198 "Invalid instruction: '{s} {s} {s} {s} {s}'",10372 "invalid instruction: '{s} {s} {s} {s} {s}'",
10199 .{10373 .{
10200 @tagName(mnem_tag[1]),10374 mnem_str,
10201 @tagName(ops[0]),10375 @tagName(ops[0]),
10202 @tagName(ops[1]),10376 @tagName(ops[1]),
10203 @tagName(ops[2]),10377 @tagName(ops[2]),
...@@ -10208,7 +10382,11 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -10208,7 +10382,11 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
10208 };10382 };
10209 }10383 }
1021010384
10211 for (outputs, args.items[0..outputs.len]) |output, mcv| {10385 var label_it = labels.iterator();
10386 while (label_it.next()) |label| if (label.value_ptr.pending_relocs.items.len > 0)
10387 return self.fail("undefined label: '{s}'", .{label.key_ptr.*});
10388
10389 for (outputs, args.items[0..outputs.len]) |output, arg_mcv| {
10212 const extra_bytes = mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);10390 const extra_bytes = mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);
10213 const constraint =10391 const constraint =
10214 mem.sliceTo(mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);10392 mem.sliceTo(mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);
...@@ -10218,8 +10396,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -10218,8 +10396,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
10218 outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;10396 outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
1021910397
10220 if (output == .none) continue;10398 if (output == .none) continue;
10221 if (mcv != .register) continue;10399 if (arg_mcv != .register) continue;
10222 try self.store(self.typeOf(output), try self.resolveInst(output), mcv);10400 try self.store(self.typeOf(output), .{ .air_ref = output }, arg_mcv);
10223 }10401 }
1022410402
10225 simple: {10403 simple: {
...@@ -11510,7 +11688,7 @@ fn atomicOp(...@@ -11510,7 +11688,7 @@ fn atomicOp(
11510 defer self.register_manager.unlockReg(tmp_lock);11688 defer self.register_manager.unlockReg(tmp_lock);
1151111689
11512 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);11690 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);
11513 const loop: u32 = @intCast(self.mir_instructions.len);11691 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
11514 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {11692 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
11515 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });11693 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
11516 }11694 }
...@@ -11584,7 +11762,7 @@ fn atomicOp(...@@ -11584,7 +11762,7 @@ fn atomicOp(
11584 .scale_index = ptr_mem.scaleIndex(),11762 .scale_index = ptr_mem.scaleIndex(),
11585 .disp = ptr_mem.sib.disp + 8,11763 .disp = ptr_mem.sib.disp + 8,
11586 }));11764 }));
11587 const loop: u32 = @intCast(self.mir_instructions.len);11765 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
11588 const val_mem_mcv: MCValue = switch (val_mcv) {11766 const val_mem_mcv: MCValue = switch (val_mcv) {
11589 .memory, .indirect, .load_frame => val_mcv,11767 .memory, .indirect, .load_frame => val_mcv,
11590 else => .{ .indirect = .{11768 else => .{ .indirect = .{
test/behavior/asm.zig+2-4
...@@ -42,7 +42,6 @@ test "module level assembly" {...@@ -42,7 +42,6 @@ test "module level assembly" {
4242
43test "output constraint modifiers" {43test "output constraint modifiers" {
44 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO44 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
45 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
46 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO45 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
47 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO46 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
48 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO47 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -66,7 +65,6 @@ test "output constraint modifiers" {...@@ -66,7 +65,6 @@ test "output constraint modifiers" {
6665
67test "alternative constraints" {66test "alternative constraints" {
68 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO67 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
69 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
70 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO68 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
71 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO69 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
72 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO70 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -163,8 +161,8 @@ export fn derp() i32 {...@@ -163,8 +161,8 @@ export fn derp() i32 {
163}161}
164162
165test "rw constraint (x86_64)" {163test "rw constraint (x86_64)" {
166 if (builtin.target.cpu.arch != .x86_64 or builtin.zig_backend != .stage2_llvm)164 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
167 return error.SkipZigTest;165 if (builtin.target.cpu.arch != .x86_64) return error.SkipZigTest;
168166
169 var res: i32 = 5;167 var res: i32 = 5;
170 asm ("addl %[b], %[a]"168 asm ("addl %[b], %[a]"