authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-08 06:50:18-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-08 07:36:20-04:00
log6c14eb2863c7c00f809c5e447ceb8186b55f2eef
tree14d9948d53fb661206d2a8b71ba4af73e9431de1
parent6778da4516e68c271cb50fe9c252ab4084daf16b

x86_64: optimize mir tag usage

This moves all pseudo-instructions to a single `Mir.Inst.Tag` tag and prepares to start coalescing similar mnemonics. 239 tags left in use.

6 files changed, 839 insertions(+), 864 deletions(-)

src/arch/x86_64/CodeGen.zig+251-152
...@@ -973,14 +973,14 @@ fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {...@@ -973,14 +973,14 @@ fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
973 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);973 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
974 const result_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);974 const result_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);
975 self.mir_instructions.appendAssumeCapacity(inst);975 self.mir_instructions.appendAssumeCapacity(inst);
976 switch (inst.tag) {976 if (inst.tag != .pseudo or switch (inst.ops) {
977 else => wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)}),977 else => true,
978 .dbg_line,978 .pseudo_dbg_prologue_end_none,
979 .dbg_prologue_end,979 .pseudo_dbg_line_line_column,
980 .dbg_epilogue_begin,980 .pseudo_dbg_epilogue_begin_none,
981 .dead,981 .pseudo_dead_none,
982 => {},982 => false,
983 }983 }) wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});
984 return result_index;984 return result_index;
985}985}
986986
...@@ -1003,35 +1003,57 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {...@@ -1003,35 +1003,57 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1003 return result;1003 return result;
1004}1004}
10051005
1006fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {1006/// A `cc` of `.z_and_np` clobbers `reg2`!
1007fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
1007 _ = try self.addInst(.{1008 _ = try self.addInst(.{
1008 .tag = .setcc,1009 .tag = switch (cc) {
1009 .ops = .r_cc,1010 else => .cmov,
1010 .data = .{ .r_cc = .{1011 .z_and_np, .nz_or_p => .pseudo,
1011 .r = reg,1012 },
1012 .scratch = if (cc == .z_and_np or cc == .nz_or_p)1013 .ops = switch (cc) {
1013 (try self.register_manager.allocReg(null, gp)).to8()1014 else => .rr,
1014 else1015 .z_and_np => .pseudo_cmov_z_and_np_rr,
1015 .none,1016 .nz_or_p => .pseudo_cmov_nz_or_p_rr,
1016 .cc = cc,1017 },
1018 .data = .{ .rr = .{
1019 .fixes = switch (cc) {
1020 else => Mir.Inst.Fixes.fromCondition(cc),
1021 .z_and_np, .nz_or_p => ._,
1022 },
1023 .r1 = reg1,
1024 .r2 = reg2,
1017 } },1025 } },
1018 });1026 });
1019}1027}
10201028
1021fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {1029/// A `cc` of `.z_and_np` is not supported by this encoding!
1030fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
1022 _ = try self.addInst(.{1031 _ = try self.addInst(.{
1023 .tag = .setcc,1032 .tag = switch (cc) {
1024 .ops = switch (m) {1033 else => .cmov,
1025 .sib => .m_sib_cc,1034 .z_and_np => unreachable,
1026 .rip => .m_rip_cc,1035 .nz_or_p => .pseudo,
1027 else => unreachable,
1028 },1036 },
1029 .data = .{ .x_cc = .{1037 .ops = switch (cc) {
1030 .scratch = if (cc == .z_and_np or cc == .nz_or_p)1038 else => switch (m) {
1031 (try self.register_manager.allocReg(null, gp)).to8()1039 .sib => .rm_sib,
1032 else1040 .rip => .rm_rip,
1033 .none,1041 else => unreachable,
1034 .cc = cc,1042 },
1043 .z_and_np => unreachable,
1044 .nz_or_p => switch (m) {
1045 .sib => .pseudo_cmov_nz_or_p_rm_sib,
1046 .rip => .pseudo_cmov_nz_or_p_rm_rip,
1047 else => unreachable,
1048 },
1049 },
1050 .data = .{ .rx = .{
1051 .fixes = switch (cc) {
1052 else => Mir.Inst.Fixes.fromCondition(cc),
1053 .z_and_np => unreachable,
1054 .nz_or_p => ._,
1055 },
1056 .r1 = reg,
1035 .payload = switch (m) {1057 .payload = switch (m) {
1036 .sib => try self.addExtra(Mir.MemorySib.encode(m)),1058 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
1037 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),1059 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
...@@ -1041,60 +1063,106 @@ fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {...@@ -1041,60 +1063,106 @@ fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
1041 });1063 });
1042}1064}
10431065
1044/// A `cc` of `.z_and_np` clobbers `reg2`!1066fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
1045fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
1046 _ = try self.addInst(.{1067 _ = try self.addInst(.{
1047 .tag = .cmovcc,1068 .tag = switch (cc) {
1048 .ops = .rr_cc,1069 else => .set,
1049 .data = .{ .rr_cc = .{1070 .z_and_np, .nz_or_p => .pseudo,
1050 .r1 = reg1,1071 },
1051 .r2 = reg2,1072 .ops = switch (cc) {
1052 .cc = cc,1073 else => .r,
1053 } },1074 .z_and_np => .pseudo_set_z_and_np_r,
1075 .nz_or_p => .pseudo_set_nz_or_p_r,
1076 },
1077 .data = switch (cc) {
1078 else => .{ .r = .{
1079 .fixes = Mir.Inst.Fixes.fromCondition(cc),
1080 .r1 = reg,
1081 } },
1082 .z_and_np, .nz_or_p => .{ .r_scratch = .{
1083 .r1 = reg,
1084 .scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
1085 } },
1086 },
1054 });1087 });
1055}1088}
10561089
1057fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {1090fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
1058 assert(cc != .z_and_np); // not supported1091 const payload = switch (m) {
1092 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
1093 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1094 else => unreachable,
1095 };
1059 _ = try self.addInst(.{1096 _ = try self.addInst(.{
1060 .tag = .cmovcc,1097 .tag = switch (cc) {
1061 .ops = switch (m) {1098 else => .set,
1062 .sib => .rm_sib_cc,1099 .z_and_np, .nz_or_p => .pseudo,
1063 .rip => .rm_rip_cc,
1064 else => unreachable,
1065 },1100 },
1066 .data = .{ .rx_cc = .{1101 .ops = switch (cc) {
1067 .r = reg,1102 else => switch (m) {
1068 .cc = cc,1103 .sib => .m_sib,
1069 .payload = switch (m) {1104 .rip => .m_rip,
1070 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
1071 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1072 else => unreachable,1105 else => unreachable,
1073 },1106 },
1074 } },1107 .z_and_np => switch (m) {
1108 .sib => .pseudo_set_z_and_np_m_sib,
1109 .rip => .pseudo_set_z_and_np_m_rip,
1110 else => unreachable,
1111 },
1112 .nz_or_p => switch (m) {
1113 .sib => .pseudo_set_nz_or_p_m_sib,
1114 .rip => .pseudo_set_nz_or_p_m_rip,
1115 else => unreachable,
1116 },
1117 },
1118 .data = switch (cc) {
1119 else => .{ .x = .{
1120 .fixes = Mir.Inst.Fixes.fromCondition(cc),
1121 .payload = payload,
1122 } },
1123 .z_and_np, .nz_or_p => .{ .x_scratch = .{
1124 .scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
1125 .payload = payload,
1126 } },
1127 },
1075 });1128 });
1076}1129}
10771130
1078fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {1131fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
1079 return self.addInst(.{1132 return self.addInst(.{
1080 .tag = .jmp_reloc,1133 .tag = .jmp,
1081 .ops = undefined,1134 .ops = .inst,
1082 .data = .{ .inst = target },1135 .data = .{ .inst = .{
1136 .inst = target,
1137 } },
1083 });1138 });
1084}1139}
10851140
1086fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Inst.Index {1141fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Inst.Index {
1087 return self.addInst(.{1142 return self.addInst(.{
1088 .tag = .jcc,1143 .tag = switch (cc) {
1089 .ops = .inst_cc,1144 else => .j,
1090 .data = .{ .inst_cc = .{ .inst = target, .cc = cc } },1145 .z_and_np, .nz_or_p => .pseudo,
1146 },
1147 .ops = switch (cc) {
1148 else => .inst,
1149 .z_and_np => .pseudo_j_z_and_np_inst,
1150 .nz_or_p => .pseudo_j_nz_or_p_inst,
1151 },
1152 .data = .{ .inst = .{
1153 .fixes = switch (cc) {
1154 else => Mir.Inst.Fixes.fromCondition(cc),
1155 .z_and_np, .nz_or_p => ._,
1156 },
1157 .inst = target,
1158 } },
1091 });1159 });
1092}1160}
10931161
1094fn asmPlaceholder(self: *Self) !Mir.Inst.Index {1162fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
1095 return self.addInst(.{1163 return self.addInst(.{
1096 .tag = .dead,1164 .tag = .pseudo,
1097 .ops = undefined,1165 .ops = .pseudo_dead_none,
1098 .data = undefined,1166 .data = undefined,
1099 });1167 });
1100}1168}
...@@ -1107,11 +1175,19 @@ fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {...@@ -1107,11 +1175,19 @@ fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
1107 });1175 });
1108}1176}
11091177
1178fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
1179 _ = try self.addInst(.{
1180 .tag = .pseudo,
1181 .ops = ops,
1182 .data = undefined,
1183 });
1184}
1185
1110fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {1186fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {
1111 _ = try self.addInst(.{1187 _ = try self.addInst(.{
1112 .tag = tag,1188 .tag = tag,
1113 .ops = .r,1189 .ops = .r,
1114 .data = .{ .r = reg },1190 .data = .{ .r = .{ .r1 = reg } },
1115 });1191 });
1116}1192}
11171193
...@@ -1122,9 +1198,11 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {...@@ -1122,9 +1198,11 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
1122 .signed => .i_s,1198 .signed => .i_s,
1123 .unsigned => .i_u,1199 .unsigned => .i_u,
1124 },1200 },
1125 .data = .{ .i = switch (imm) {1201 .data = .{ .i = .{
1126 .signed => |s| @bitCast(u32, s),1202 .i = switch (imm) {
1127 .unsigned => |u| @intCast(u32, u),1203 .signed => |s| @bitCast(u32, s),
1204 .unsigned => |u| @intCast(u32, u),
1205 },
1128 } },1206 } },
1129 });1207 });
1130}1208}
...@@ -1147,14 +1225,14 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme...@@ -1147,14 +1225,14 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
1147 .ops = ops,1225 .ops = ops,
1148 .data = switch (ops) {1226 .data = switch (ops) {
1149 .ri_s, .ri_u => .{ .ri = .{1227 .ri_s, .ri_u => .{ .ri = .{
1150 .r = reg,1228 .r1 = reg,
1151 .i = switch (imm) {1229 .i = switch (imm) {
1152 .signed => |s| @bitCast(u32, s),1230 .signed => |s| @bitCast(u32, s),
1153 .unsigned => |u| @intCast(u32, u),1231 .unsigned => |u| @intCast(u32, u),
1154 },1232 },
1155 } },1233 } },
1156 .ri64 => .{ .rx = .{1234 .ri64 => .{ .rx = .{
1157 .r = reg,1235 .r1 = reg,
1158 .payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),1236 .payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
1159 } },1237 } },
1160 else => unreachable,1238 else => unreachable,
...@@ -1249,10 +1327,12 @@ fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {...@@ -1249,10 +1327,12 @@ fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
1249 .rip => .m_rip,1327 .rip => .m_rip,
1250 else => unreachable,1328 else => unreachable,
1251 },1329 },
1252 .data = .{ .payload = switch (m) {1330 .data = .{ .x = .{
1253 .sib => try self.addExtra(Mir.MemorySib.encode(m)),1331 .payload = switch (m) {
1254 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),1332 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
1255 else => unreachable,1333 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1334 else => unreachable,
1335 },
1256 } },1336 } },
1257 });1337 });
1258}1338}
...@@ -1266,7 +1346,7 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !...@@ -1266,7 +1346,7 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !
1266 else => unreachable,1346 else => unreachable,
1267 },1347 },
1268 .data = .{ .rx = .{1348 .data = .{ .rx = .{
1269 .r = reg,1349 .r1 = reg,
1270 .payload = switch (m) {1350 .payload = switch (m) {
1271 .sib => try self.addExtra(Mir.MemorySib.encode(m)),1351 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
1272 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),1352 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
...@@ -1291,7 +1371,7 @@ fn asmRegisterMemoryImmediate(...@@ -1291,7 +1371,7 @@ fn asmRegisterMemoryImmediate(
1291 else => unreachable,1371 else => unreachable,
1292 },1372 },
1293 .data = .{ .rix = .{1373 .data = .{ .rix = .{
1294 .r = reg,1374 .r1 = reg,
1295 .i = @intCast(u8, imm.unsigned),1375 .i = @intCast(u8, imm.unsigned),
1296 .payload = switch (m) {1376 .payload = switch (m) {
1297 .sib => try self.addExtra(Mir.MemorySib.encode(m)),1377 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
...@@ -1339,7 +1419,7 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !...@@ -1339,7 +1419,7 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !
1339 else => unreachable,1419 else => unreachable,
1340 },1420 },
1341 .data = .{ .rx = .{1421 .data = .{ .rx = .{
1342 .r = reg,1422 .r1 = reg,
1343 .payload = switch (m) {1423 .payload = switch (m) {
1344 .sib => try self.addExtra(Mir.MemorySib.encode(m)),1424 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
1345 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),1425 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
...@@ -1413,11 +1493,15 @@ fn asmMemoryRegisterImmediate(...@@ -1413,11 +1493,15 @@ fn asmMemoryRegisterImmediate(
1413 .rip => .mri_rip,1493 .rip => .mri_rip,
1414 else => unreachable,1494 else => unreachable,
1415 },1495 },
1416 .data = .{ .rix = .{ .r = reg, .i = @intCast(u8, imm.unsigned), .payload = switch (m) {1496 .data = .{ .rix = .{
1417 .sib => try self.addExtra(Mir.MemorySib.encode(m)),1497 .r1 = reg,
1418 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),1498 .i = @intCast(u8, imm.unsigned),
1419 else => unreachable,1499 .payload = switch (m) {
1420 } } },1500 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
1501 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1502 else => unreachable,
1503 },
1504 } },
1421 });1505 });
1422}1506}
14231507
...@@ -1450,7 +1534,7 @@ fn gen(self: *Self) InnerError!void {...@@ -1450,7 +1534,7 @@ fn gen(self: *Self) InnerError!void {
1450 else => unreachable,1534 else => unreachable,
1451 }1535 }
14521536
1453 try self.asmOpOnly(.dbg_prologue_end);1537 try self.asmPseudo(.pseudo_dbg_prologue_end_none);
14541538
1455 try self.genBody(self.air.getMainBody());1539 try self.genBody(self.air.getMainBody());
14561540
...@@ -1462,11 +1546,11 @@ fn gen(self: *Self) InnerError!void {...@@ -1462,11 +1546,11 @@ fn gen(self: *Self) InnerError!void {
1462 // }1546 // }
1463 // Eliding the reloc will cause a miscompilation in this case.1547 // Eliding the reloc will cause a miscompilation in this case.
1464 for (self.exitlude_jump_relocs.items) |jmp_reloc| {1548 for (self.exitlude_jump_relocs.items) |jmp_reloc| {
1465 self.mir_instructions.items(.data)[jmp_reloc].inst =1549 self.mir_instructions.items(.data)[jmp_reloc].inst.inst =
1466 @intCast(u32, self.mir_instructions.len);1550 @intCast(u32, self.mir_instructions.len);
1467 }1551 }
14681552
1469 try self.asmOpOnly(.dbg_epilogue_begin);1553 try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
1470 const backpatch_stack_dealloc = try self.asmPlaceholder();1554 const backpatch_stack_dealloc = try self.asmPlaceholder();
1471 const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();1555 const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
1472 try self.asmRegister(.pop, .rbp);1556 try self.asmRegister(.pop, .rbp);
...@@ -1480,46 +1564,54 @@ fn gen(self: *Self) InnerError!void {...@@ -1480,46 +1564,54 @@ fn gen(self: *Self) InnerError!void {
1480 self.mir_instructions.set(backpatch_frame_align, .{1564 self.mir_instructions.set(backpatch_frame_align, .{
1481 .tag = .@"and",1565 .tag = .@"and",
1482 .ops = .ri_s,1566 .ops = .ri_s,
1483 .data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_mask } },1567 .data = .{ .ri = .{
1568 .r1 = .rsp,
1569 .i = frame_layout.stack_mask,
1570 } },
1484 });1571 });
1485 }1572 }
1486 if (need_stack_adjust) {1573 if (need_stack_adjust) {
1487 self.mir_instructions.set(backpatch_stack_alloc, .{1574 self.mir_instructions.set(backpatch_stack_alloc, .{
1488 .tag = .sub,1575 .tag = .sub,
1489 .ops = .ri_s,1576 .ops = .ri_s,
1490 .data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_adjust } },1577 .data = .{ .ri = .{
1578 .r1 = .rsp,
1579 .i = frame_layout.stack_adjust,
1580 } },
1491 });1581 });
1492 }1582 }
1493 if (need_frame_align or need_stack_adjust) {1583 if (need_frame_align or need_stack_adjust) {
1494 self.mir_instructions.set(backpatch_stack_dealloc, .{1584 self.mir_instructions.set(backpatch_stack_dealloc, .{
1495 .tag = .mov,1585 .tag = .mov,
1496 .ops = .rr,1586 .ops = .rr,
1497 .data = .{ .rr = .{ .r1 = .rsp, .r2 = .rbp } },1587 .data = .{ .rr = .{
1588 .r1 = .rsp,
1589 .r2 = .rbp,
1590 } },
1498 });1591 });
1499 }1592 }
1500 if (need_save_reg) {1593 if (need_save_reg) {
1501 const save_reg_list = frame_layout.save_reg_list.asInt();
1502 self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{1594 self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
1503 .tag = .push_regs,1595 .tag = .pseudo,
1504 .ops = undefined,1596 .ops = .pseudo_push_reg_list,
1505 .data = .{ .payload = save_reg_list },1597 .data = .{ .reg_list = frame_layout.save_reg_list },
1506 });1598 });
1507 self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{1599 self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{
1508 .tag = .pop_regs,1600 .tag = .pseudo,
1509 .ops = undefined,1601 .ops = .pseudo_pop_reg_list,
1510 .data = .{ .payload = save_reg_list },1602 .data = .{ .reg_list = frame_layout.save_reg_list },
1511 });1603 });
1512 }1604 }
1513 } else {1605 } else {
1514 try self.asmOpOnly(.dbg_prologue_end);1606 try self.asmPseudo(.pseudo_dbg_prologue_end_none);
1515 try self.genBody(self.air.getMainBody());1607 try self.genBody(self.air.getMainBody());
1516 try self.asmOpOnly(.dbg_epilogue_begin);1608 try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
1517 }1609 }
15181610
1519 // Drop them off at the rbrace.1611 // Drop them off at the rbrace.
1520 _ = try self.addInst(.{1612 _ = try self.addInst(.{
1521 .tag = .dbg_line,1613 .tag = .pseudo,
1522 .ops = undefined,1614 .ops = .pseudo_dbg_line_line_column,
1523 .data = .{ .line_column = .{1615 .data = .{ .line_column = .{
1524 .line = self.end_di_line,1616 .line = self.end_di_line,
1525 .column = self.end_di_column,1617 .column = self.end_di_column,
...@@ -2446,11 +2538,11 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -2446,11 +2538,11 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2446 .register => |dst_reg| {2538 .register => |dst_reg| {
2447 const min_abi_size = @min(dst_abi_size, src_abi_size);2539 const min_abi_size = @min(dst_abi_size, src_abi_size);
2448 const tag: Mir.Inst.Tag = switch (signedness) {2540 const tag: Mir.Inst.Tag = switch (signedness) {
2449 .signed => .movsx,2541 .signed => if (min_abi_size >= 4) .movsxd else .movsx,
2450 .unsigned => if (min_abi_size > 2) .mov else .movzx,2542 .unsigned => if (min_abi_size >= 4) .mov else .movzx,
2451 };2543 };
2452 const dst_alias = switch (tag) {2544 const dst_alias = switch (tag) {
2453 .movsx => dst_reg.to64(),2545 .movsx, .movsxd => dst_reg.to64(),
2454 .mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),2546 .mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),
2455 else => unreachable,2547 else => unreachable,
2456 };2548 };
...@@ -5247,7 +5339,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -5247,7 +5339,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5247 const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));5339 const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));
5248 if (signedness == .signed and field_byte_size < 8) {5340 if (signedness == .signed and field_byte_size < 8) {
5249 try self.asmRegisterRegister(5341 try self.asmRegisterRegister(
5250 .movsx,5342 if (field_byte_size >= 4) .movsxd else .movsx,
5251 dst_mcv.register,5343 dst_mcv.register,
5252 registerAlias(dst_mcv.register, field_byte_size),5344 registerAlias(dst_mcv.register, field_byte_size),
5253 );5345 );
...@@ -7194,10 +7286,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -7194,10 +7286,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7194 const atom_index = try self.owner.getSymbolIndex(self);7286 const atom_index = try self.owner.getSymbolIndex(self);
7195 const sym_index = try coff_file.getGlobalSymbol(decl_name, lib_name);7287 const sym_index = try coff_file.getGlobalSymbol(decl_name, lib_name);
7196 _ = try self.addInst(.{7288 _ = try self.addInst(.{
7197 .tag = .mov_linker,7289 .tag = .mov,
7198 .ops = .import_reloc,7290 .ops = .import_reloc,
7199 .data = .{ .rx = .{7291 .data = .{ .rx = .{
7200 .r = .rax,7292 .r1 = .rax,
7201 .payload = try self.addExtra(Mir.Reloc{7293 .payload = try self.addExtra(Mir.Reloc{
7202 .atom_index = atom_index,7294 .atom_index = atom_index,
7203 .sym_index = sym_index,7295 .sym_index = sym_index,
...@@ -7209,9 +7301,9 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -7209,9 +7301,9 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
7209 const atom_index = try self.owner.getSymbolIndex(self);7301 const atom_index = try self.owner.getSymbolIndex(self);
7210 const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);7302 const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);
7211 _ = try self.addInst(.{7303 _ = try self.addInst(.{
7212 .tag = .call_extern,7304 .tag = .call,
7213 .ops = undefined,7305 .ops = .extern_fn_reloc,
7214 .data = .{ .relocation = .{7306 .data = .{ .reloc = .{
7215 .atom_index = atom_index,7307 .atom_index = atom_index,
7216 .sym_index = sym_index,7308 .sym_index = sym_index,
7217 } },7309 } },
...@@ -7489,8 +7581,8 @@ fn genTry(...@@ -7489,8 +7581,8 @@ fn genTry(
7489fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {7581fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
7490 const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;7582 const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
7491 _ = try self.addInst(.{7583 _ = try self.addInst(.{
7492 .tag = .dbg_line,7584 .tag = .pseudo,
7493 .ops = undefined,7585 .ops = .pseudo_dbg_line_line_column,
7494 .data = .{ .line_column = .{7586 .data = .{ .line_column = .{
7495 .line = dbg_stmt.line,7587 .line = dbg_stmt.line,
7496 .column = dbg_stmt.column,7588 .column = dbg_stmt.column,
...@@ -8021,14 +8113,14 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -8021,14 +8113,14 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
8021fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {8113fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
8022 const next_inst = @intCast(u32, self.mir_instructions.len);8114 const next_inst = @intCast(u32, self.mir_instructions.len);
8023 switch (self.mir_instructions.items(.tag)[reloc]) {8115 switch (self.mir_instructions.items(.tag)[reloc]) {
8024 .jcc => {8116 .j, .jmp => {},
8025 self.mir_instructions.items(.data)[reloc].inst_cc.inst = next_inst;8117 .pseudo => switch (self.mir_instructions.items(.ops)[reloc]) {
8026 },8118 .pseudo_j_z_and_np_inst, .pseudo_j_nz_or_p_inst => {},
8027 .jmp_reloc => {8119 else => unreachable,
8028 self.mir_instructions.items(.data)[reloc].inst = next_inst;
8029 },8120 },
8030 else => unreachable,8121 else => unreachable,
8031 }8122 }
8123 self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;
8032}8124}
80338125
8034fn airBr(self: *Self, inst: Air.Inst.Index) !void {8126fn airBr(self: *Self, inst: Air.Inst.Index) !void {
...@@ -8577,10 +8669,10 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -8577,10 +8669,10 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8577 .load_direct => |sym_index| if (!ty.isRuntimeFloat()) {8669 .load_direct => |sym_index| if (!ty.isRuntimeFloat()) {
8578 const atom_index = try self.owner.getSymbolIndex(self);8670 const atom_index = try self.owner.getSymbolIndex(self);
8579 _ = try self.addInst(.{8671 _ = try self.addInst(.{
8580 .tag = .mov_linker,8672 .tag = .mov,
8581 .ops = .direct_reloc,8673 .ops = .direct_reloc,
8582 .data = .{ .rx = .{8674 .data = .{ .rx = .{
8583 .r = dst_reg.to64(),8675 .r1 = dst_reg.to64(),
8584 .payload = try self.addExtra(Mir.Reloc{8676 .payload = try self.addExtra(Mir.Reloc{
8585 .atom_index = atom_index,8677 .atom_index = atom_index,
8586 .sym_index = sym_index,8678 .sym_index = sym_index,
...@@ -8618,8 +8710,8 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -8618,8 +8710,8 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8618 const atom_index = try self.owner.getSymbolIndex(self);8710 const atom_index = try self.owner.getSymbolIndex(self);
8619 _ = try self.addInst(.{8711 _ = try self.addInst(.{
8620 .tag = switch (src_mcv) {8712 .tag = switch (src_mcv) {
8621 .lea_direct => .lea_linker,8713 .lea_direct => .lea,
8622 .lea_got => .mov_linker,8714 .lea_got => .mov,
8623 else => unreachable,8715 else => unreachable,
8624 },8716 },
8625 .ops = switch (src_mcv) {8717 .ops = switch (src_mcv) {
...@@ -8628,7 +8720,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -8628,7 +8720,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8628 else => unreachable,8720 else => unreachable,
8629 },8721 },
8630 .data = .{ .rx = .{8722 .data = .{ .rx = .{
8631 .r = dst_reg.to64(),8723 .r1 = dst_reg.to64(),
8632 .payload = try self.addExtra(Mir.Reloc{8724 .payload = try self.addExtra(Mir.Reloc{
8633 .atom_index = atom_index,8725 .atom_index = atom_index,
8634 .sym_index = sym_index,8726 .sym_index = sym_index,
...@@ -8640,10 +8732,10 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -8640,10 +8732,10 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
8640 const atom_index = try self.owner.getSymbolIndex(self);8732 const atom_index = try self.owner.getSymbolIndex(self);
8641 if (self.bin_file.cast(link.File.MachO)) |_| {8733 if (self.bin_file.cast(link.File.MachO)) |_| {
8642 _ = try self.addInst(.{8734 _ = try self.addInst(.{
8643 .tag = .lea_linker,8735 .tag = .lea,
8644 .ops = .tlv_reloc,8736 .ops = .tlv_reloc,
8645 .data = .{ .rx = .{8737 .data = .{ .rx = .{
8646 .r = .rdi,8738 .r1 = .rdi,
8647 .payload = try self.addExtra(Mir.Reloc{8739 .payload = try self.addExtra(Mir.Reloc{
8648 .atom_index = atom_index,8740 .atom_index = atom_index,
8649 .sym_index = sym_index,8741 .sym_index = sym_index,
...@@ -8847,9 +8939,9 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue...@@ -8847,9 +8939,9 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue
8847 try self.genSetReg(.rsi, Type.usize, src_ptr);8939 try self.genSetReg(.rsi, Type.usize, src_ptr);
8848 try self.genSetReg(.rcx, Type.usize, len);8940 try self.genSetReg(.rcx, Type.usize, len);
8849 _ = try self.addInst(.{8941 _ = try self.addInst(.{
8850 .tag = .movs,8942 .tag = .mov,
8851 .ops = .string,8943 .ops = .none,
8852 .data = .{ .string = .{ .repeat = .rep, .width = .b } },8944 .data = .{ .none = .{ .fixes = .@"rep _sb" } },
8853 });8945 });
8854}8946}
88558947
...@@ -8859,9 +8951,9 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)...@@ -8859,9 +8951,9 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)
8859 try self.genSetReg(.al, Type.u8, value);8951 try self.genSetReg(.al, Type.u8, value);
8860 try self.genSetReg(.rcx, Type.usize, len);8952 try self.genSetReg(.rcx, Type.usize, len);
8861 _ = try self.addInst(.{8953 _ = try self.addInst(.{
8862 .tag = .stos,8954 .tag = .sto,
8863 .ops = .string,8955 .ops = .none,
8864 .data = .{ .string = .{ .repeat = .rep, .width = .b } },8956 .data = .{ .none = .{ .fixes = .@"rep _sb" } },
8865 });8957 });
8866}8958}
88678959
...@@ -9135,22 +9227,22 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -9135,22 +9227,22 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
9135 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);9227 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
91369228
9137 try self.spillEflagsIfOccupied();9229 try self.spillEflagsIfOccupied();
9138 if (val_abi_size <= 8) {9230 _ = try self.addInst(if (val_abi_size <= 8) .{
9139 _ = try self.addInst(.{9231 .tag = .cmpxchg,
9140 .tag = .cmpxchg,9232 .ops = .mr_sib,
9141 .ops = .lock_mr_sib,9233 .data = .{ .rx = .{
9142 .data = .{ .rx = .{9234 .fixes = .@"lock _",
9143 .r = registerAlias(new_reg.?, val_abi_size),9235 .r1 = registerAlias(new_reg.?, val_abi_size),
9144 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),9236 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9145 } },9237 } },
9146 });9238 } else .{
9147 } else {9239 .tag = .cmpxchg,
9148 _ = try self.addInst(.{9240 .ops = .m_sib,
9149 .tag = .cmpxchgb,9241 .data = .{ .x = .{
9150 .ops = .lock_m_sib,9242 .fixes = .@"lock _16b",
9151 .data = .{ .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)) },9243 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9152 });9244 } },
9153 }9245 });
91549246
9155 const result: MCValue = result: {9247 const result: MCValue = result: {
9156 if (self.liveness.isUnused(inst)) break :result .unreach;9248 if (self.liveness.isUnused(inst)) break :result .unreach;
...@@ -9252,13 +9344,14 @@ fn atomicOp(...@@ -9252,13 +9344,14 @@ fn atomicOp(
9252 }9344 }
9253 _ = try self.addInst(.{9345 _ = try self.addInst(.{
9254 .tag = tag,9346 .tag = tag,
9255 .ops = switch (tag) {9347 .ops = .mr_sib,
9256 .mov, .xchg => .mr_sib,
9257 .xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
9258 else => unreachable,
9259 },
9260 .data = .{ .rx = .{9348 .data = .{ .rx = .{
9261 .r = registerAlias(dst_reg, val_abi_size),9349 .fixes = switch (tag) {
9350 .mov, .xchg => ._,
9351 .xadd, .add, .sub, .@"and", .@"or", .xor => .@"lock _",
9352 else => unreachable,
9353 },
9354 .r1 = registerAlias(dst_reg, val_abi_size),
9262 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),9355 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9263 } },9356 } },
9264 });9357 });
...@@ -9330,9 +9423,10 @@ fn atomicOp(...@@ -9330,9 +9423,10 @@ fn atomicOp(
9330 };9423 };
9331 _ = try self.addInst(.{9424 _ = try self.addInst(.{
9332 .tag = .cmpxchg,9425 .tag = .cmpxchg,
9333 .ops = .lock_mr_sib,9426 .ops = .mr_sib,
9334 .data = .{ .rx = .{9427 .data = .{ .rx = .{
9335 .r = registerAlias(tmp_reg, val_abi_size),9428 .fixes = .@"lock _",
9429 .r1 = registerAlias(tmp_reg, val_abi_size),
9336 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),9430 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9337 } },9431 } },
9338 });9432 });
...@@ -9397,9 +9491,14 @@ fn atomicOp(...@@ -9397,9 +9491,14 @@ fn atomicOp(
9397 val_ty.fmt(self.bin_file.options.module.?), @tagName(op),9491 val_ty.fmt(self.bin_file.options.module.?), @tagName(op),
9398 }),9492 }),
9399 };9493 };
9400 _ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{9494 _ = try self.addInst(.{
9401 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),9495 .tag = .cmpxchg,
9402 } });9496 .ops = .m_sib,
9497 .data = .{ .x = .{
9498 .fixes = .@"lock _16b",
9499 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
9500 } },
9501 });
9403 _ = try self.asmJccReloc(loop, .ne);9502 _ = try self.asmJccReloc(loop, .ne);
94049503
9405 if (unused) return .unreach;9504 if (unused) return .unreach;
src/arch/x86_64/Emit.zig+34-31
...@@ -41,7 +41,7 @@ pub fn emitMir(emit: *Emit) Error!void {...@@ -41,7 +41,7 @@ pub fn emitMir(emit: *Emit) Error!void {
41 .offset = end_offset - 4,41 .offset = end_offset - 4,
42 .length = @intCast(u5, end_offset - start_offset),42 .length = @intCast(u5, end_offset - start_offset),
43 }),43 }),
44 .@"extern" => |symbol| if (emit.bin_file.cast(link.File.MachO)) |macho_file| {44 .linker_extern_fn => |symbol| if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
45 // Add relocation to the decl.45 // Add relocation to the decl.
46 const atom_index = macho_file.getAtomIndexForSymbol(46 const atom_index = macho_file.getAtomIndexForSymbol(
47 .{ .sym_index = symbol.atom_index, .file = null },47 .{ .sym_index = symbol.atom_index, .file = null },
...@@ -129,36 +129,39 @@ pub fn emitMir(emit: *Emit) Error!void {...@@ -129,36 +129,39 @@ pub fn emitMir(emit: *Emit) Error!void {
129 const mir_inst = emit.lower.mir.instructions.get(mir_index);129 const mir_inst = emit.lower.mir.instructions.get(mir_index);
130 switch (mir_inst.tag) {130 switch (mir_inst.tag) {
131 else => unreachable,131 else => unreachable,
132 .dead => {},132 .pseudo => switch (mir_inst.ops) {
133 .dbg_line => try emit.dbgAdvancePCAndLine(133 else => unreachable,
134 mir_inst.data.line_column.line,134 .pseudo_dbg_prologue_end_none => {
135 mir_inst.data.line_column.column,135 switch (emit.debug_output) {
136 ),136 .dwarf => |dw| {
137 .dbg_prologue_end => {137 try dw.setPrologueEnd();
138 switch (emit.debug_output) {138 log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
139 .dwarf => |dw| {139 emit.prev_di_line, emit.prev_di_column,
140 try dw.setPrologueEnd();140 });
141 log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{141 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
142 emit.prev_di_line, emit.prev_di_column,142 },
143 });143 .plan9 => {},
144 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);144 .none => {},
145 },145 }
146 .plan9 => {},146 },
147 .none => {},147 .pseudo_dbg_line_line_column => try emit.dbgAdvancePCAndLine(
148 }148 mir_inst.data.line_column.line,
149 },149 mir_inst.data.line_column.column,
150 .dbg_epilogue_begin => {150 ),
151 switch (emit.debug_output) {151 .pseudo_dbg_epilogue_begin_none => {
152 .dwarf => |dw| {152 switch (emit.debug_output) {
153 try dw.setEpilogueBegin();153 .dwarf => |dw| {
154 log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{154 try dw.setEpilogueBegin();
155 emit.prev_di_line, emit.prev_di_column,155 log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
156 });156 emit.prev_di_line, emit.prev_di_column,
157 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);157 });
158 },158 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
159 .plan9 => {},159 },
160 .none => {},160 .plan9 => {},
161 }161 .none => {},
162 }
163 },
164 .pseudo_dead_none => {},
162 },165 },
163 }166 }
164 }167 }
src/arch/x86_64/Encoding.zig+1-1
...@@ -705,7 +705,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op...@@ -705,7 +705,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
705}705}
706706
707const mnemonic_to_encodings_map = init: {707const mnemonic_to_encodings_map = init: {
708 @setEvalBranchQuota(100_000);708 @setEvalBranchQuota(20_000);
709 const encodings = @import("encodings.zig");709 const encodings = @import("encodings.zig");
710 var entries = encodings.table;710 var entries = encodings.table;
711 std.sort.sort(encodings.Entry, &entries, {}, struct {711 std.sort.sort(encodings.Entry, &entries, {}, struct {
src/arch/x86_64/Lower.zig+205-508
...@@ -35,7 +35,7 @@ pub const Reloc = struct {...@@ -35,7 +35,7 @@ pub const Reloc = struct {
3535
36 const Target = union(enum) {36 const Target = union(enum) {
37 inst: Mir.Inst.Index,37 inst: Mir.Inst.Index,
38 @"extern": Mir.Reloc,38 linker_extern_fn: Mir.Reloc,
39 linker_got: Mir.Reloc,39 linker_got: Mir.Reloc,
40 linker_direct: Mir.Reloc,40 linker_direct: Mir.Reloc,
41 linker_import: Mir.Reloc,41 linker_import: Mir.Reloc,
...@@ -59,280 +59,119 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -59,280 +59,119 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
5959
60 const inst = lower.mir.instructions.get(index);60 const inst = lower.mir.instructions.get(index);
61 switch (inst.tag) {61 switch (inst.tag) {
62 .adc,62 else => try lower.generic(inst),
63 .add,63 .pseudo => switch (inst.ops) {
64 .@"and",64 .pseudo_cmov_z_and_np_rr => {
65 .bsf,65 try lower.emit(.none, .cmovnz, &.{
66 .bsr,66 .{ .reg = inst.data.rr.r2 },
67 .bswap,67 .{ .reg = inst.data.rr.r1 },
68 .bt,68 });
69 .btc,69 try lower.emit(.none, .cmovnp, &.{
70 .btr,70 .{ .reg = inst.data.rr.r1 },
71 .bts,71 .{ .reg = inst.data.rr.r2 },
72 .call,72 });
73 .cbw,73 },
74 .cwde,74 .pseudo_cmov_nz_or_p_rr => {
75 .cdqe,75 try lower.emit(.none, .cmovnz, &.{
76 .cwd,76 .{ .reg = inst.data.rr.r1 },
77 .cdq,77 .{ .reg = inst.data.rr.r2 },
78 .cqo,78 });
79 .cmp,79 try lower.emit(.none, .cmovp, &.{
80 .cmpxchg,80 .{ .reg = inst.data.rr.r1 },
81 .div,81 .{ .reg = inst.data.rr.r2 },
82 .fisttp,82 });
83 .fld,83 },
84 .idiv,84 .pseudo_cmov_nz_or_p_rm_sib,
85 .imul,85 .pseudo_cmov_nz_or_p_rm_rip,
86 .int3,86 => {
87 .jmp,87 try lower.emit(.none, .cmovnz, &.{
88 .lea,88 .{ .reg = inst.data.rx.r1 },
89 .lfence,89 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
90 .lzcnt,90 });
91 .mfence,91 try lower.emit(.none, .cmovp, &.{
92 .mov,92 .{ .reg = inst.data.rx.r1 },
93 .movbe,93 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
94 .movd,94 });
95 .movq,95 },
96 .movzx,96 .pseudo_set_z_and_np_r => {
97 .mul,97 try lower.emit(.none, .setz, &.{
98 .neg,98 .{ .reg = inst.data.r_scratch.r1 },
99 .nop,99 });
100 .not,100 try lower.emit(.none, .setnp, &.{
101 .@"or",101 .{ .reg = inst.data.r_scratch.scratch_reg },
102 .pop,102 });
103 .popcnt,103 try lower.emit(.none, .@"and", &.{
104 .push,104 .{ .reg = inst.data.r_scratch.r1 },
105 .rcl,105 .{ .reg = inst.data.r_scratch.scratch_reg },
106 .rcr,106 });
107 .ret,107 },
108 .rol,108 .pseudo_set_z_and_np_m_sib,
109 .ror,109 .pseudo_set_z_and_np_m_rip,
110 .sal,110 => {
111 .sar,111 try lower.emit(.none, .setz, &.{
112 .sbb,112 .{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
113 .sfence,113 });
114 .shl,114 try lower.emit(.none, .setnp, &.{
115 .shld,115 .{ .reg = inst.data.x_scratch.scratch_reg },
116 .shr,116 });
117 .shrd,117 try lower.emit(.none, .@"and", &.{
118 .sub,118 .{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
119 .syscall,119 .{ .reg = inst.data.x_scratch.scratch_reg },
120 .@"test",120 });
121 .tzcnt,121 },
122 .ud2,122 .pseudo_set_nz_or_p_r => {
123 .xadd,123 try lower.emit(.none, .setnz, &.{
124 .xchg,124 .{ .reg = inst.data.r_scratch.r1 },
125 .xor,125 });
126126 try lower.emit(.none, .setp, &.{
127 .addps,127 .{ .reg = inst.data.r_scratch.scratch_reg },
128 .addss,128 });
129 .andnps,129 try lower.emit(.none, .@"or", &.{
130 .andps,130 .{ .reg = inst.data.r_scratch.r1 },
131 .cmpss,131 .{ .reg = inst.data.r_scratch.scratch_reg },
132 .cvtsi2ss,132 });
133 .divps,133 },
134 .divss,134 .pseudo_set_nz_or_p_m_sib,
135 .maxps,135 .pseudo_set_nz_or_p_m_rip,
136 .maxss,136 => {
137 .minps,137 try lower.emit(.none, .setnz, &.{
138 .minss,138 .{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
139 .movaps,139 });
140 .movhlps,140 try lower.emit(.none, .setp, &.{
141 .movss,141 .{ .reg = inst.data.x_scratch.scratch_reg },
142 .movups,142 });
143 .mulps,143 try lower.emit(.none, .@"or", &.{
144 .mulss,144 .{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
145 .orps,145 .{ .reg = inst.data.x_scratch.scratch_reg },
146 .pextrw,146 });
147 .pinsrw,147 },
148 .sqrtps,148 .pseudo_j_z_and_np_inst => {
149 .sqrtss,149 try lower.emit(.none, .jnz, &.{
150 .subps,150 .{ .imm = lower.reloc(.{ .inst = index + 1 }) },
151 .subss,151 });
152 .ucomiss,152 try lower.emit(.none, .jnp, &.{
153 .unpckhps,153 .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
154 .unpcklps,154 });
155 .xorps,155 },
156156 .pseudo_j_nz_or_p_inst => {
157 .addpd,157 try lower.emit(.none, .jnz, &.{
158 .addsd,158 .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
159 .andnpd,159 });
160 .andpd,160 try lower.emit(.none, .jp, &.{
161 .cmpsd,161 .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
162 .cvtsd2ss,162 });
163 .cvtsi2sd,163 },
164 .cvtss2sd,
165 .divpd,
166 .divsd,
167 .maxpd,
168 .maxsd,
169 .minpd,
170 .minsd,
171 .movsd,
172 .mulpd,
173 .mulsd,
174 .orpd,
175 .pshufhw,
176 .pshuflw,
177 .psrld,
178 .psrlq,
179 .psrlw,
180 .punpckhbw,
181 .punpckhdq,
182 .punpckhqdq,
183 .punpckhwd,
184 .punpcklbw,
185 .punpckldq,
186 .punpcklqdq,
187 .punpcklwd,
188 .sqrtpd,
189 .sqrtsd,
190 .subpd,
191 .subsd,
192 .ucomisd,
193 .unpckhpd,
194 .unpcklpd,
195 .xorpd,
196
197 .movddup,
198 .movshdup,
199 .movsldup,
200
201 .pextrb,
202 .pextrd,
203 .pextrq,
204 .pinsrb,
205 .pinsrd,
206 .pinsrq,
207 .roundpd,
208 .roundps,
209 .roundsd,
210 .roundss,
211
212 .vaddpd,
213 .vaddps,
214 .vaddsd,
215 .vaddss,
216 .vcvtsd2ss,
217 .vcvtsi2sd,
218 .vcvtsi2ss,
219 .vcvtss2sd,
220 .vdivpd,
221 .vdivps,
222 .vdivsd,
223 .vdivss,
224 .vmaxpd,
225 .vmaxps,
226 .vmaxsd,
227 .vmaxss,
228 .vminpd,
229 .vminps,
230 .vminsd,
231 .vminss,
232 .vmovapd,
233 .vmovaps,
234 .vmovddup,
235 .vmovhlps,
236 .vmovsd,
237 .vmovshdup,
238 .vmovsldup,
239 .vmovss,
240 .vmovupd,
241 .vmovups,
242 .vmulpd,
243 .vmulps,
244 .vmulsd,
245 .vmulss,
246 .vpextrb,
247 .vpextrd,
248 .vpextrq,
249 .vpextrw,
250 .vpinsrb,
251 .vpinsrd,
252 .vpinsrq,
253 .vpinsrw,
254 .vpshufhw,
255 .vpshuflw,
256 .vpsrld,
257 .vpsrlq,
258 .vpsrlw,
259 .vpunpckhbw,
260 .vpunpckhdq,
261 .vpunpckhqdq,
262 .vpunpckhwd,
263 .vpunpcklbw,
264 .vpunpckldq,
265 .vpunpcklqdq,
266 .vpunpcklwd,
267 .vroundpd,
268 .vroundps,
269 .vroundsd,
270 .vroundss,
271 .vsqrtpd,
272 .vsqrtps,
273 .vsqrtsd,
274 .vsqrtss,
275 .vsubpd,
276 .vsubps,
277 .vsubsd,
278 .vsubss,
279 .vunpckhpd,
280 .vunpckhps,
281 .vunpcklpd,
282 .vunpcklps,
283
284 .vcvtph2ps,
285 .vcvtps2ph,
286
287 .vfmadd132pd,
288 .vfmadd213pd,
289 .vfmadd231pd,
290 .vfmadd132ps,
291 .vfmadd213ps,
292 .vfmadd231ps,
293 .vfmadd132sd,
294 .vfmadd213sd,
295 .vfmadd231sd,
296 .vfmadd132ss,
297 .vfmadd213ss,
298 .vfmadd231ss,
299 => try lower.mirGeneric(inst),
300
301 .cmps,
302 .lods,
303 .movs,
304 .scas,
305 .stos,
306 => try lower.mirString(inst),
307
308 .cmpxchgb => try lower.mirCmpxchgBytes(inst),
309
310 .jmp_reloc => try lower.emitInstWithReloc(.none, .jmp, &.{
311 .{ .imm = Immediate.s(0) },
312 }, .{ .inst = inst.data.inst }),
313
314 .call_extern => try lower.emitInstWithReloc(.none, .call, &.{
315 .{ .imm = Immediate.s(0) },
316 }, .{ .@"extern" = inst.data.relocation }),
317
318 .lea_linker => try lower.mirLinker(.lea, inst),
319 .mov_linker => try lower.mirLinker(.mov, inst),
320
321 .mov_moffs => try lower.mirMovMoffs(inst),
322
323 .movsx => try lower.mirMovsx(inst),
324 .cmovcc => try lower.mirCmovcc(inst),
325 .setcc => try lower.mirSetcc(inst),
326 .jcc => try lower.mirJcc(index, inst),
327164
328 .push_regs => try lower.mirRegisterList(.push, inst),165 .pseudo_push_reg_list => try lower.pushPopRegList(.push, inst),
329 .pop_regs => try lower.mirRegisterList(.pop, inst),166 .pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst),
330167
331 .dbg_line,168 .pseudo_dbg_prologue_end_none,
332 .dbg_prologue_end,169 .pseudo_dbg_line_line_column,
333 .dbg_epilogue_begin,170 .pseudo_dbg_epilogue_begin_none,
334 .dead,171 .pseudo_dead_none,
335 => {},172 => {},
173 else => unreachable,
174 },
336 }175 }
337176
338 return .{177 return .{
...@@ -348,15 +187,6 @@ pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {...@@ -348,15 +187,6 @@ pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
348 return error.LowerFail;187 return error.LowerFail;
349}188}
350189
351fn mnem_cc(comptime base: @Type(.EnumLiteral), cc: bits.Condition) Mnemonic {
352 return switch (cc) {
353 inline else => |c| if (@hasField(Mnemonic, @tagName(base) ++ @tagName(c)))
354 @field(Mnemonic, @tagName(base) ++ @tagName(c))
355 else
356 unreachable,
357 };
358}
359
360fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {190fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
361 return switch (ops) {191 return switch (ops) {
362 .rri_s,192 .rri_s,
...@@ -364,8 +194,6 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {...@@ -364,8 +194,6 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
364 .i_s,194 .i_s,
365 .mi_sib_s,195 .mi_sib_s,
366 .mi_rip_s,196 .mi_rip_s,
367 .lock_mi_sib_s,
368 .lock_mi_rip_s,
369 => Immediate.s(@bitCast(i32, i)),197 => Immediate.s(@bitCast(i32, i)),
370198
371 .rrri,199 .rrri,
...@@ -374,8 +202,6 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {...@@ -374,8 +202,6 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
374 .i_u,202 .i_u,
375 .mi_sib_u,203 .mi_sib_u,
376 .mi_rip_u,204 .mi_rip_u,
377 .lock_mi_sib_u,
378 .lock_mi_rip_u,
379 .rmi_sib,205 .rmi_sib,
380 .rmi_rip,206 .rmi_rip,
381 .mri_sib,207 .mri_sib,
...@@ -395,10 +221,8 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {...@@ -395,10 +221,8 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
395fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {221fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
396 return lower.mir.resolveFrameLoc(switch (ops) {222 return lower.mir.resolveFrameLoc(switch (ops) {
397 .rm_sib,223 .rm_sib,
398 .rm_sib_cc,
399 .rmi_sib,224 .rmi_sib,
400 .m_sib,225 .m_sib,
401 .m_sib_cc,
402 .mi_sib_u,226 .mi_sib_u,
403 .mi_sib_s,227 .mi_sib_s,
404 .mr_sib,228 .mr_sib,
...@@ -406,17 +230,15 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {...@@ -406,17 +230,15 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
406 .mri_sib,230 .mri_sib,
407 .rrm_sib,231 .rrm_sib,
408 .rrmi_sib,232 .rrmi_sib,
409 .lock_m_sib,233
410 .lock_mi_sib_u,234 .pseudo_cmov_nz_or_p_rm_sib,
411 .lock_mi_sib_s,235 .pseudo_set_z_and_np_m_sib,
412 .lock_mr_sib,236 .pseudo_set_nz_or_p_m_sib,
413 => lower.mir.extraData(Mir.MemorySib, payload).data.decode(),237 => lower.mir.extraData(Mir.MemorySib, payload).data.decode(),
414238
415 .rm_rip,239 .rm_rip,
416 .rm_rip_cc,
417 .rmi_rip,240 .rmi_rip,
418 .m_rip,241 .m_rip,
419 .m_rip_cc,
420 .mi_rip_u,242 .mi_rip_u,
421 .mi_rip_s,243 .mi_rip_s,
422 .mr_rip,244 .mr_rip,
...@@ -424,66 +246,83 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {...@@ -424,66 +246,83 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
424 .mri_rip,246 .mri_rip,
425 .rrm_rip,247 .rrm_rip,
426 .rrmi_rip,248 .rrmi_rip,
427 .lock_m_rip,249
428 .lock_mi_rip_u,250 .pseudo_cmov_nz_or_p_rm_rip,
429 .lock_mi_rip_s,251 .pseudo_set_z_and_np_m_rip,
430 .lock_mr_rip,252 .pseudo_set_nz_or_p_m_rip,
431 => lower.mir.extraData(Mir.MemoryRip, payload).data.decode(),253 => lower.mir.extraData(Mir.MemoryRip, payload).data.decode(),
432254
433 .rax_moffs,255 .rax_moffs,
434 .moffs_rax,256 .moffs_rax,
435 .lock_moffs_rax,
436 => lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),257 => lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),
437258
438 else => unreachable,259 else => unreachable,
439 });260 });
440}261}
441262
442fn emitInst(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {263fn reloc(lower: *Lower, target: Reloc.Target) Immediate {
443 lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
444 lower.result_insts_len += 1;
445}
446
447fn emitInstWithReloc(
448 lower: *Lower,
449 prefix: Prefix,
450 mnemonic: Mnemonic,
451 ops: []const Operand,
452 target: Reloc.Target,
453) Error!void {
454 lower.result_relocs[lower.result_relocs_len] = .{264 lower.result_relocs[lower.result_relocs_len] = .{
455 .lowered_inst_index = lower.result_insts_len,265 .lowered_inst_index = lower.result_insts_len,
456 .target = target,266 .target = target,
457 };267 };
458 lower.result_relocs_len += 1;268 lower.result_relocs_len += 1;
459 try lower.emitInst(prefix, mnemonic, ops);269 return Immediate.s(0);
270}
271
272fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
273 lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
274 lower.result_insts_len += 1;
460}275}
461276
462fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {277fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
463 try lower.emitInst(switch (inst.ops) {278 const fixes = switch (inst.ops) {
464 else => .none,279 .none => inst.data.none.fixes,
465 .lock_m_sib,280 .inst => inst.data.inst.fixes,
466 .lock_m_rip,281 .i_s, .i_u => inst.data.i.fixes,
467 .lock_mi_sib_u,282 .r => inst.data.r.fixes,
468 .lock_mi_rip_u,283 .rr => inst.data.rr.fixes,
469 .lock_mi_sib_s,284 .rrr => inst.data.rrr.fixes,
470 .lock_mi_rip_s,285 .rrri => inst.data.rrri.fixes,
471 .lock_mr_sib,286 .rri_s, .rri_u => inst.data.rri.fixes,
472 .lock_mr_rip,287 .ri_s, .ri_u => inst.data.ri.fixes,
473 .lock_moffs_rax,288 .ri64, .rm_sib, .rm_rip, .mr_sib, .mr_rip => inst.data.rx.fixes,
474 => .lock,289 .mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => ._,
475 }, switch (inst.tag) {290 .mrr_sib, .mrr_rip, .rrm_sib, .rrm_rip => inst.data.rrx.fixes,
476 inline else => |tag| if (@hasField(Mnemonic, @tagName(tag)))291 .rmi_sib, .rmi_rip, .mri_sib, .mri_rip => inst.data.rix.fixes,
477 @field(Mnemonic, @tagName(tag))292 .rrmi_sib, .rrmi_rip => inst.data.rrix.fixes,
293 .m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes,
294 .extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._,
295 else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),
296 };
297 try lower.emit(switch (fixes) {
298 inline else => |tag| comptime if (std.mem.indexOfScalar(u8, @tagName(tag), ' ')) |space|
299 @field(Prefix, @tagName(tag)[0..space])
478 else300 else
479 unreachable,301 .none,
302 }, mnemonic: {
303 comptime var max_len = 0;
304 inline for (@typeInfo(Mnemonic).Enum.fields) |field| max_len = @max(field.name.len, max_len);
305 var buf: [max_len]u8 = undefined;
306
307 const fixes_name = @tagName(fixes);
308 const pattern = fixes_name[if (std.mem.indexOfScalar(u8, fixes_name, ' ')) |i| i + 1 else 0..];
309 const wildcard_i = std.mem.indexOfScalar(u8, pattern, '_').?;
310 const parts = .{ pattern[0..wildcard_i], @tagName(inst.tag), pattern[wildcard_i + 1 ..] };
311 const err_msg = "unsupported mnemonic: ";
312 const mnemonic = std.fmt.bufPrint(&buf, "{s}{s}{s}", parts) catch
313 return lower.fail(err_msg ++ "'{s}{s}{s}'", parts);
314 break :mnemonic std.meta.stringToEnum(Mnemonic, mnemonic) orelse
315 return lower.fail(err_msg ++ "'{s}'", .{mnemonic});
480 }, switch (inst.ops) {316 }, switch (inst.ops) {
481 .none => &.{},317 .none => &.{},
318 .inst => &.{
319 .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
320 },
482 .i_s, .i_u => &.{321 .i_s, .i_u => &.{
483 .{ .imm = lower.imm(inst.ops, inst.data.i) },322 .{ .imm = lower.imm(inst.ops, inst.data.i.i) },
484 },323 },
485 .r => &.{324 .r => &.{
486 .{ .reg = inst.data.r },325 .{ .reg = inst.data.r.r1 },
487 },326 },
488 .rr => &.{327 .rr => &.{
489 .{ .reg = inst.data.rr.r1 },328 .{ .reg = inst.data.rr.r1 },
...@@ -501,11 +340,11 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -501,11 +340,11 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
501 .{ .imm = lower.imm(inst.ops, inst.data.rrri.i) },340 .{ .imm = lower.imm(inst.ops, inst.data.rrri.i) },
502 },341 },
503 .ri_s, .ri_u => &.{342 .ri_s, .ri_u => &.{
504 .{ .reg = inst.data.ri.r },343 .{ .reg = inst.data.ri.r1 },
505 .{ .imm = lower.imm(inst.ops, inst.data.ri.i) },344 .{ .imm = lower.imm(inst.ops, inst.data.ri.i) },
506 },345 },
507 .ri64 => &.{346 .ri64 => &.{
508 .{ .reg = inst.data.rx.r },347 .{ .reg = inst.data.rx.r1 },
509 .{ .imm = lower.imm(inst.ops, inst.data.rx.payload) },348 .{ .imm = lower.imm(inst.ops, inst.data.rx.payload) },
510 },349 },
511 .rri_s, .rri_u => &.{350 .rri_s, .rri_u => &.{
...@@ -513,33 +352,25 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -513,33 +352,25 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
513 .{ .reg = inst.data.rri.r2 },352 .{ .reg = inst.data.rri.r2 },
514 .{ .imm = lower.imm(inst.ops, inst.data.rri.i) },353 .{ .imm = lower.imm(inst.ops, inst.data.rri.i) },
515 },354 },
516 .m_sib, .lock_m_sib, .m_rip, .lock_m_rip => &.{355 .m_sib, .m_rip => &.{
517 .{ .mem = lower.mem(inst.ops, inst.data.payload) },356 .{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
518 },357 },
519 .mi_sib_s,358 .mi_sib_s, .mi_sib_u, .mi_rip_u, .mi_rip_s => &.{
520 .lock_mi_sib_s,
521 .mi_sib_u,
522 .lock_mi_sib_u,
523 .mi_rip_u,
524 .lock_mi_rip_u,
525 .mi_rip_s,
526 .lock_mi_rip_s,
527 => &.{
528 .{ .mem = lower.mem(inst.ops, inst.data.ix.payload) },359 .{ .mem = lower.mem(inst.ops, inst.data.ix.payload) },
529 .{ .imm = lower.imm(inst.ops, inst.data.ix.i) },360 .{ .imm = lower.imm(inst.ops, inst.data.ix.i) },
530 },361 },
531 .rm_sib, .rm_rip => &.{362 .rm_sib, .rm_rip => &.{
532 .{ .reg = inst.data.rx.r },363 .{ .reg = inst.data.rx.r1 },
533 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },364 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
534 },365 },
535 .rmi_sib, .rmi_rip => &.{366 .rmi_sib, .rmi_rip => &.{
536 .{ .reg = inst.data.rix.r },367 .{ .reg = inst.data.rix.r1 },
537 .{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },368 .{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
538 .{ .imm = lower.imm(inst.ops, inst.data.rix.i) },369 .{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
539 },370 },
540 .mr_sib, .lock_mr_sib, .mr_rip, .lock_mr_rip => &.{371 .mr_sib, .mr_rip => &.{
541 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },372 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
542 .{ .reg = inst.data.rx.r },373 .{ .reg = inst.data.rx.r1 },
543 },374 },
544 .mrr_sib, .mrr_rip => &.{375 .mrr_sib, .mrr_rip => &.{
545 .{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },376 .{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
...@@ -548,7 +379,7 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -548,7 +379,7 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
548 },379 },
549 .mri_sib, .mri_rip => &.{380 .mri_sib, .mri_rip => &.{
550 .{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },381 .{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
551 .{ .reg = inst.data.rix.r },382 .{ .reg = inst.data.rix.r1 },
552 .{ .imm = lower.imm(inst.ops, inst.data.rix.i) },383 .{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
553 },384 },
554 .rrm_sib, .rrm_rip => &.{385 .rrm_sib, .rrm_rip => &.{
...@@ -562,180 +393,46 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -562,180 +393,46 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
562 .{ .mem = lower.mem(inst.ops, inst.data.rrix.payload) },393 .{ .mem = lower.mem(inst.ops, inst.data.rrix.payload) },
563 .{ .imm = lower.imm(inst.ops, inst.data.rrix.i) },394 .{ .imm = lower.imm(inst.ops, inst.data.rrix.i) },
564 },395 },
565 else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
566 });
567}
568
569fn mirString(lower: *Lower, inst: Mir.Inst) Error!void {
570 switch (inst.ops) {
571 .string => try lower.emitInst(switch (inst.data.string.repeat) {
572 inline else => |repeat| @field(Prefix, @tagName(repeat)),
573 }, switch (inst.tag) {
574 inline .cmps, .lods, .movs, .scas, .stos => |tag| switch (inst.data.string.width) {
575 inline else => |width| @field(Mnemonic, @tagName(tag) ++ @tagName(width)),
576 },
577 else => unreachable,
578 }, &.{}),
579 else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
580 }
581}
582
583fn mirCmpxchgBytes(lower: *Lower, inst: Mir.Inst) Error!void {
584 const ops: [1]Operand = switch (inst.ops) {
585 .m_sib, .lock_m_sib, .m_rip, .lock_m_rip => .{
586 .{ .mem = lower.mem(inst.ops, inst.data.payload) },
587 },
588 else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
589 };
590 try lower.emitInst(switch (inst.ops) {
591 .m_sib, .m_rip => .none,
592 .lock_m_sib, .lock_m_rip => .lock,
593 else => unreachable,
594 }, switch (@divExact(ops[0].bitSize(), 8)) {
595 8 => .cmpxchg8b,
596 16 => .cmpxchg16b,
597 else => return lower.fail("invalid operand for {s}", .{@tagName(inst.tag)}),
598 }, &ops);
599}
600
601fn mirMovMoffs(lower: *Lower, inst: Mir.Inst) Error!void {
602 try lower.emitInst(switch (inst.ops) {
603 .rax_moffs, .moffs_rax => .none,
604 .lock_moffs_rax => .lock,
605 else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
606 }, .mov, switch (inst.ops) {
607 .rax_moffs => &.{396 .rax_moffs => &.{
608 .{ .reg = .rax },397 .{ .reg = .rax },
609 .{ .mem = lower.mem(inst.ops, inst.data.payload) },398 .{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
610 },399 },
611 .moffs_rax, .lock_moffs_rax => &.{400 .moffs_rax => &.{
612 .{ .mem = lower.mem(inst.ops, inst.data.payload) },401 .{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
613 .{ .reg = .rax },402 .{ .reg = .rax },
614 },403 },
615 else => unreachable,404 .extern_fn_reloc => &.{
616 });405 .{ .imm = lower.reloc(.{ .linker_extern_fn = inst.data.reloc }) },
617}
618
619fn mirMovsx(lower: *Lower, inst: Mir.Inst) Error!void {
620 const ops: [2]Operand = switch (inst.ops) {
621 .rr => .{
622 .{ .reg = inst.data.rr.r1 },
623 .{ .reg = inst.data.rr.r2 },
624 },
625 .rm_sib, .rm_rip => .{
626 .{ .reg = inst.data.rx.r },
627 .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
628 },
629 else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
630 };
631 try lower.emitInst(.none, switch (ops[0].bitSize()) {
632 32, 64 => switch (ops[1].bitSize()) {
633 32 => .movsxd,
634 else => .movsx,
635 },
636 else => .movsx,
637 }, &ops);
638}
639
640fn mirCmovcc(lower: *Lower, inst: Mir.Inst) Error!void {
641 const data: struct { cc: bits.Condition, ops: [2]Operand } = switch (inst.ops) {
642 .rr_cc => .{ .cc = inst.data.rr_cc.cc, .ops = .{
643 .{ .reg = inst.data.rr_cc.r1 },
644 .{ .reg = inst.data.rr_cc.r2 },
645 } },
646 .rm_sib_cc, .rm_rip_cc => .{ .cc = inst.data.rx_cc.cc, .ops = .{
647 .{ .reg = inst.data.rx_cc.r },
648 .{ .mem = lower.mem(inst.ops, inst.data.rx_cc.payload) },
649 } },
650 else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
651 };
652 switch (data.cc) {
653 else => |cc| try lower.emitInst(.none, mnem_cc(.cmov, cc), &data.ops),
654 .z_and_np => {
655 try lower.emitInst(.none, mnem_cc(.cmov, .nz), &.{ data.ops[1], data.ops[0] });
656 try lower.emitInst(.none, mnem_cc(.cmov, .np), &data.ops);
657 },406 },
658 .nz_or_p => {407 .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ops: {
659 try lower.emitInst(.none, mnem_cc(.cmov, .nz), &data.ops);408 const reg = inst.data.rx.r1;
660 try lower.emitInst(.none, mnem_cc(.cmov, .p), &data.ops);409 const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
410 _ = lower.reloc(switch (inst.ops) {
411 .got_reloc => .{ .linker_got = extra },
412 .direct_reloc => .{ .linker_direct = extra },
413 .import_reloc => .{ .linker_import = extra },
414 .tlv_reloc => .{ .linker_tlv = extra },
415 else => unreachable,
416 });
417 break :ops &.{
418 .{ .reg = reg },
419 .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
420 };
661 },421 },
662 }
663}
664
665fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {
666 const data: struct { cc: bits.Condition, ops: [2]Operand } = switch (inst.ops) {
667 .r_cc => .{ .cc = inst.data.r_cc.cc, .ops = .{
668 .{ .reg = inst.data.r_cc.r },
669 .{ .reg = inst.data.r_cc.scratch },
670 } },
671 .m_sib_cc, .m_rip_cc => .{ .cc = inst.data.x_cc.cc, .ops = .{
672 .{ .mem = lower.mem(inst.ops, inst.data.x_cc.payload) },
673 .{ .reg = inst.data.x_cc.scratch },
674 } },
675 else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),422 else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
676 };423 });
677 switch (data.cc) {
678 else => |cc| try lower.emitInst(.none, mnem_cc(.set, cc), data.ops[0..1]),
679 .z_and_np => {
680 try lower.emitInst(.none, mnem_cc(.set, .z), data.ops[0..1]);
681 try lower.emitInst(.none, mnem_cc(.set, .np), data.ops[1..2]);
682 try lower.emitInst(.none, .@"and", data.ops[0..2]);
683 },
684 .nz_or_p => {
685 try lower.emitInst(.none, mnem_cc(.set, .nz), data.ops[0..1]);
686 try lower.emitInst(.none, mnem_cc(.set, .p), data.ops[1..2]);
687 try lower.emitInst(.none, .@"or", data.ops[0..2]);
688 },
689 }
690}
691
692fn mirJcc(lower: *Lower, index: Mir.Inst.Index, inst: Mir.Inst) Error!void {
693 switch (inst.data.inst_cc.cc) {
694 else => |cc| try lower.emitInstWithReloc(.none, mnem_cc(.j, cc), &.{
695 .{ .imm = Immediate.s(0) },
696 }, .{ .inst = inst.data.inst_cc.inst }),
697 .z_and_np => {
698 try lower.emitInstWithReloc(.none, mnem_cc(.j, .nz), &.{
699 .{ .imm = Immediate.s(0) },
700 }, .{ .inst = index + 1 });
701 try lower.emitInstWithReloc(.none, mnem_cc(.j, .np), &.{
702 .{ .imm = Immediate.s(0) },
703 }, .{ .inst = inst.data.inst_cc.inst });
704 },
705 .nz_or_p => {
706 try lower.emitInstWithReloc(.none, mnem_cc(.j, .nz), &.{
707 .{ .imm = Immediate.s(0) },
708 }, .{ .inst = inst.data.inst_cc.inst });
709 try lower.emitInstWithReloc(.none, mnem_cc(.j, .p), &.{
710 .{ .imm = Immediate.s(0) },
711 }, .{ .inst = inst.data.inst_cc.inst });
712 },
713 }
714}424}
715425
716fn mirRegisterList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void {426fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
717 const reg_list = Mir.RegisterList.fromInt(inst.data.payload);
718 const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);427 const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);
719 var it = reg_list.iterator(.{ .direction = switch (mnemonic) {428 var it = inst.data.reg_list.iterator(.{ .direction = switch (mnemonic) {
720 .push => .reverse,429 .push => .reverse,
721 .pop => .forward,430 .pop => .forward,
722 else => unreachable,431 else => unreachable,
723 } });432 } });
724 while (it.next()) |i| try lower.emitInst(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});433 while (it.next()) |i| try lower.emit(.none, mnemonic, &.{.{
725}434 .reg = callee_preserved_regs[i],
726435 }});
727fn mirLinker(lower: *Lower, mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
728 const reloc = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
729 try lower.emitInstWithReloc(.none, mnemonic, &.{
730 .{ .reg = inst.data.rx.r },
731 .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(inst.data.rx.r.bitSize()), 0) },
732 }, switch (inst.ops) {
733 .got_reloc => .{ .linker_got = reloc },
734 .direct_reloc => .{ .linker_direct = reloc },
735 .import_reloc => .{ .linker_import = reloc },
736 .tlv_reloc => .{ .linker_tlv = reloc },
737 else => unreachable,
738 });
739}436}
740437
741const abi = @import("abi.zig");438const abi = @import("abi.zig");
src/arch/x86_64/Mir.zig+348-169
...@@ -32,6 +32,210 @@ pub const Inst = struct {...@@ -32,6 +32,210 @@ pub const Inst = struct {
3232
33 pub const Index = u32;33 pub const Index = u32;
3434
35 pub const Fixes = enum(u8) {
36 /// ___
37 @"_",
38
39 /// ___ Above
40 _a,
41 /// ___ Above Or Equal
42 _ae,
43 /// ___ Below
44 _b,
45 /// ___ Below Or Equal
46 _be,
47 /// ___ Carry
48 _c,
49 /// ___ Equal
50 _e,
51 /// ___ Greater
52 _g,
53 /// ___ Greater Or Equal
54 _ge,
55 /// ___ Less
56 _l,
57 /// ___ Less Or Equal
58 _le,
59 /// ___ Not Above
60 _na,
61 /// ___ Not Above Or Equal
62 _nae,
63 /// ___ Not Below
64 _nb,
65 /// ___ Not Below Or Equal
66 _nbe,
67 /// ___ Not Carry
68 _nc,
69 /// ___ Not Equal
70 _ne,
71 /// ___ Not Greater
72 _ng,
73 /// ___ Not Greater Or Equal
74 _nge,
75 /// ___ Not Less
76 _nl,
77 /// ___ Not Less Or Equal
78 _nle,
79 /// ___ Not Overflow
80 _no,
81 /// ___ Not Parity
82 _np,
83 /// ___ Not Sign
84 _ns,
85 /// ___ Not Zero
86 _nz,
87 /// ___ Overflow
88 _o,
89 /// ___ Parity
90 _p,
91 /// ___ Parity Even
92 _pe,
93 /// ___ Parity Odd
94 _po,
95 /// ___ Sign
96 _s,
97 /// ___ Zero
98 _z,
99
100 /// ___ String
101 //_s,
102 /// ___ String Byte
103 _sb,
104 /// ___ String Word
105 _sw,
106 /// ___ String Doubleword
107 _sd,
108 /// ___ String Quadword
109 _sq,
110
111 /// Repeat ___ String
112 @"rep _s",
113 /// Repeat ___ String Byte
114 @"rep _sb",
115 /// Repeat ___ String Word
116 @"rep _sw",
117 /// Repeat ___ String Doubleword
118 @"rep _sd",
119 /// Repeat ___ String Quadword
120 @"rep _sq",
121
122 /// Repeat Equal ___ String
123 @"repe _s",
124 /// Repeat Equal ___ String Byte
125 @"repe _sb",
126 /// Repeat Equal ___ String Word
127 @"repe _sw",
128 /// Repeat Equal ___ String Doubleword
129 @"repe _sd",
130 /// Repeat Equal ___ String Quadword
131 @"repe _sq",
132
133 /// Repeat Not Equal ___ String
134 @"repne _s",
135 /// Repeat Not Equal ___ String Byte
136 @"repne _sb",
137 /// Repeat Not Equal ___ String Word
138 @"repne _sw",
139 /// Repeat Not Equal ___ String Doubleword
140 @"repne _sd",
141 /// Repeat Not Equal ___ String Quadword
142 @"repne _sq",
143
144 /// Repeat Not Zero ___ String
145 @"repnz _s",
146 /// Repeat Not Zero ___ String Byte
147 @"repnz _sb",
148 /// Repeat Not Zero ___ String Word
149 @"repnz _sw",
150 /// Repeat Not Zero ___ String Doubleword
151 @"repnz _sd",
152 /// Repeat Not Zero ___ String Quadword
153 @"repnz _sq",
154
155 /// Repeat Zero ___ String
156 @"repz _s",
157 /// Repeat Zero ___ String Byte
158 @"repz _sb",
159 /// Repeat Zero ___ String Word
160 @"repz _sw",
161 /// Repeat Zero ___ String Doubleword
162 @"repz _sd",
163 /// Repeat Zero ___ String Quadword
164 @"repz _sq",
165
166 /// Locked ___
167 @"lock _",
168 /// ___ 8 Bytes
169 _8b,
170 /// Locked ___ 8 Bytes
171 @"lock _8b",
172 /// ___ 16 Bytes
173 _16b,
174 /// Locked ___ 16 Bytes
175 @"lock _16b",
176
177 /// Packed ___
178 p_,
179 /// Packed ___ Byte
180 p_b,
181 /// Packed ___ Word
182 p_w,
183 /// Packed ___ Doubleword
184 p_d,
185 /// Packed ___ Quadword
186 p_q,
187 /// Packed ___ Double Quadword
188 p_dq,
189
190 /// ___ Scalar Single-Precision Values
191 _ss,
192 /// ___ Packed Single-Precision Values
193 _ps,
194 /// ___ Scalar Double-Precision Values
195 //_sd,
196 /// ___ Packed Double-Precision Values
197 _pd,
198
199 /// VEX-Encoded ___
200 v_,
201 /// VEX-Encoded Packed ___
202 vp_,
203 /// VEX-Encoded Packed ___ Byte
204 vp_b,
205 /// VEX-Encoded Packed ___ Word
206 vp_w,
207 /// VEX-Encoded Packed ___ Doubleword
208 vp_d,
209 /// VEX-Encoded Packed ___ Quadword
210 vp_q,
211 /// VEX-Encoded Packed ___ Double Quadword
212 vp_dq,
213 /// VEX-Encoded ___ Scalar Single-Precision Values
214 v_ss,
215 /// VEX-Encoded ___ Packed Single-Precision Values
216 v_ps,
217 /// VEX-Encoded ___ Scalar Double-Precision Values
218 v_sd,
219 /// VEX-Encoded ___ Packed Double-Precision Values
220 v_pd,
221
222 /// Mask ___ Byte
223 k_b,
224 /// Mask ___ Word
225 k_w,
226 /// Mask ___ Doubleword
227 k_d,
228 /// Mask ___ Quadword
229 k_q,
230
231 pub fn fromCondition(cc: bits.Condition) Fixes {
232 return switch (cc) {
233 inline else => |cc_tag| @field(Fixes, "_" ++ @tagName(cc_tag)),
234 .z_and_np, .nz_or_p => unreachable,
235 };
236 }
237 };
238
35 pub const Tag = enum(u8) {239 pub const Tag = enum(u8) {
36 /// Add with carry240 /// Add with carry
37 adc,241 adc,
...@@ -57,22 +261,24 @@ pub const Inst = struct {...@@ -57,22 +261,24 @@ pub const Inst = struct {
57 call,261 call,
58 /// Convert byte to word262 /// Convert byte to word
59 cbw,263 cbw,
60 /// Convert word to doubleword
61 cwde,
62 /// Convert doubleword to quadword
63 cdqe,
64 /// Convert word to doubleword
65 cwd,
66 /// Convert doubleword to quadword264 /// Convert doubleword to quadword
67 cdq,265 cdq,
68 /// Convert doubleword to quadword266 /// Convert doubleword to quadword
69 cqo,267 cdqe,
268 /// Conditional move
269 cmov,
70 /// Logical compare270 /// Logical compare
271 /// Compare string
71 cmp,272 cmp,
72 /// Compare and exchange273 /// Compare and exchange
73 cmpxchg,
74 /// Compare and exchange bytes274 /// Compare and exchange bytes
75 cmpxchgb,275 cmpxchg,
276 /// Convert doubleword to quadword
277 cqo,
278 /// Convert word to doubleword
279 cwd,
280 /// Convert word to doubleword
281 cwde,
76 /// Unsigned division282 /// Unsigned division
77 div,283 div,
78 /// Store integer with truncation284 /// Store integer with truncation
...@@ -85,10 +291,14 @@ pub const Inst = struct {...@@ -85,10 +291,14 @@ pub const Inst = struct {
85 imul,291 imul,
86 ///292 ///
87 int3,293 int3,
294 /// Conditional jump
295 j,
88 /// Jump296 /// Jump
89 jmp,297 jmp,
90 /// Load effective address298 /// Load effective address
91 lea,299 lea,
300 /// Load string
301 lod,
92 /// Load fence302 /// Load fence
93 lfence,303 lfence,
94 /// Count the number of leading zero bits304 /// Count the number of leading zero bits
...@@ -96,6 +306,7 @@ pub const Inst = struct {...@@ -96,6 +306,7 @@ pub const Inst = struct {
96 /// Memory fence306 /// Memory fence
97 mfence,307 mfence,
98 /// Move308 /// Move
309 /// Move data from string to string
99 mov,310 mov,
100 /// Move data after swapping bytes311 /// Move data after swapping bytes
101 movbe,312 movbe,
...@@ -105,6 +316,8 @@ pub const Inst = struct {...@@ -105,6 +316,8 @@ pub const Inst = struct {
105 movq,316 movq,
106 /// Move with sign extension317 /// Move with sign extension
107 movsx,318 movsx,
319 /// Move with sign extension
320 movsxd,
108 /// Move with zero extension321 /// Move with zero extension
109 movzx,322 movzx,
110 /// Multiply323 /// Multiply
...@@ -139,6 +352,10 @@ pub const Inst = struct {...@@ -139,6 +352,10 @@ pub const Inst = struct {
139 sar,352 sar,
140 /// Integer subtraction with borrow353 /// Integer subtraction with borrow
141 sbb,354 sbb,
355 /// Scan string
356 sca,
357 /// Set byte on condition
358 set,
142 /// Store fence359 /// Store fence
143 sfence,360 sfence,
144 /// Logical shift left361 /// Logical shift left
...@@ -151,6 +368,8 @@ pub const Inst = struct {...@@ -151,6 +368,8 @@ pub const Inst = struct {
151 shrd,368 shrd,
152 /// Subtract369 /// Subtract
153 sub,370 sub,
371 /// Store string
372 sto,
154 /// Syscall373 /// Syscall
155 syscall,374 syscall,
156 /// Test condition375 /// Test condition
...@@ -505,57 +724,10 @@ pub const Inst = struct {...@@ -505,57 +724,10 @@ pub const Inst = struct {
505 /// Fused multiply-add of scalar single-precision floating-point values724 /// Fused multiply-add of scalar single-precision floating-point values
506 vfmadd231ss,725 vfmadd231ss,
507726
508 /// Compare string operands727 /// A pseudo instruction that requires special lowering.
509 cmps,728 /// This should be the only tag in this enum that doesn't
510 /// Load string729 /// directly correspond to one or more instruction mnemonics.
511 lods,730 pseudo,
512 /// Move data from string to string
513 movs,
514 /// Scan string
515 scas,
516 /// Store string
517 stos,
518
519 /// Conditional move
520 cmovcc,
521 /// Conditional jump
522 jcc,
523 /// Set byte on condition
524 setcc,
525
526 /// Mov absolute to/from memory wrt segment register to/from rax
527 mov_moffs,
528
529 /// Jump with relocation to another local MIR instruction
530 /// Uses `inst` payload.
531 jmp_reloc,
532
533 /// Call to an extern symbol via linker relocation.
534 /// Uses `relocation` payload.
535 call_extern,
536
537 /// Load effective address of a symbol not yet allocated in VM.
538 lea_linker,
539 /// Move address of a symbol not yet allocated in VM.
540 mov_linker,
541
542 /// End of prologue
543 dbg_prologue_end,
544 /// Start of epilogue
545 dbg_epilogue_begin,
546 /// Update debug line
547 /// Uses `line_column` payload containing the line and column.
548 dbg_line,
549 /// Push registers
550 /// Uses `payload` payload containing `RegisterList.asInt` directly.
551 push_regs,
552 /// Pop registers
553 /// Uses `payload` payload containing `RegisterList.asInt` directly.
554 pop_regs,
555
556 /// Tombstone
557 /// Emitter should skip this instruction.
558 dead,
559 };731 };
560732
561 pub const Ops = enum(u8) {733 pub const Ops = enum(u8) {
...@@ -579,12 +751,6 @@ pub const Inst = struct {...@@ -579,12 +751,6 @@ pub const Inst = struct {
579 /// Register, register, immediate (unsigned) operands.751 /// Register, register, immediate (unsigned) operands.
580 /// Uses `rri` payload.752 /// Uses `rri` payload.
581 rri_u,753 rri_u,
582 /// Register with condition code (CC).
583 /// Uses `r_cc` payload.
584 r_cc,
585 /// Register, register with condition code (CC).
586 /// Uses `rr_cc` payload.
587 rr_cc,
588 /// Register, immediate (sign-extended) operands.754 /// Register, immediate (sign-extended) operands.
589 /// Uses `ri` payload.755 /// Uses `ri` payload.
590 ri_s,756 ri_s,
...@@ -609,12 +775,6 @@ pub const Inst = struct {...@@ -609,12 +775,6 @@ pub const Inst = struct {
609 /// Register, memory (RIP) operands.775 /// Register, memory (RIP) operands.
610 /// Uses `rx` payload.776 /// Uses `rx` payload.
611 rm_rip,777 rm_rip,
612 /// Register, memory (SIB) operands with condition code (CC).
613 /// Uses `rx_cc` payload.
614 rm_sib_cc,
615 /// Register, memory (RIP) operands with condition code (CC).
616 /// Uses `rx_cc` payload.
617 rm_rip_cc,
618 /// Register, memory (SIB), immediate (byte) operands.778 /// Register, memory (SIB), immediate (byte) operands.
619 /// Uses `rix` payload with extra data of type `MemorySib`.779 /// Uses `rix` payload with extra data of type `MemorySib`.
620 rmi_sib,780 rmi_sib,
...@@ -634,17 +794,11 @@ pub const Inst = struct {...@@ -634,17 +794,11 @@ pub const Inst = struct {
634 /// Uses `rix` payload with extra data of type `MemoryRip`.794 /// Uses `rix` payload with extra data of type `MemoryRip`.
635 rmi_rip,795 rmi_rip,
636 /// Single memory (SIB) operand.796 /// Single memory (SIB) operand.
637 /// Uses `payload` with extra data of type `MemorySib`.797 /// Uses `x` with extra data of type `MemorySib`.
638 m_sib,798 m_sib,
639 /// Single memory (RIP) operand.799 /// Single memory (RIP) operand.
640 /// Uses `payload` with extra data of type `MemoryRip`.800 /// Uses `x` with extra data of type `MemoryRip`.
641 m_rip,801 m_rip,
642 /// Single memory (SIB) operand with condition code (CC).
643 /// Uses `x_cc` with extra data of type `MemorySib`.
644 m_sib_cc,
645 /// Single memory (RIP) operand with condition code (CC).
646 /// Uses `x_cc` with extra data of type `MemoryRip`.
647 m_rip_cc,
648 /// Memory (SIB), immediate (unsigned) operands.802 /// Memory (SIB), immediate (unsigned) operands.
649 /// Uses `ix` payload with extra data of type `MemorySib`.803 /// Uses `ix` payload with extra data of type `MemorySib`.
650 mi_sib_u,804 mi_sib_u,
...@@ -676,49 +830,17 @@ pub const Inst = struct {...@@ -676,49 +830,17 @@ pub const Inst = struct {
676 /// Uses `rix` payload with extra data of type `MemoryRip`.830 /// Uses `rix` payload with extra data of type `MemoryRip`.
677 mri_rip,831 mri_rip,
678 /// Rax, Memory moffs.832 /// Rax, Memory moffs.
679 /// Uses `payload` with extra data of type `MemoryMoffs`.833 /// Uses `x` with extra data of type `MemoryMoffs`.
680 rax_moffs,834 rax_moffs,
681 /// Memory moffs, rax.835 /// Memory moffs, rax.
682 /// Uses `payload` with extra data of type `MemoryMoffs`.836 /// Uses `x` with extra data of type `MemoryMoffs`.
683 moffs_rax,837 moffs_rax,
684 /// Single memory (SIB) operand with lock prefix.
685 /// Uses `payload` with extra data of type `MemorySib`.
686 lock_m_sib,
687 /// Single memory (RIP) operand with lock prefix.
688 /// Uses `payload` with extra data of type `MemoryRip`.
689 lock_m_rip,
690 /// Memory (SIB), immediate (unsigned) operands with lock prefix.
691 /// Uses `xi` payload with extra data of type `MemorySib`.
692 lock_mi_sib_u,
693 /// Memory (RIP), immediate (unsigned) operands with lock prefix.
694 /// Uses `xi` payload with extra data of type `MemoryRip`.
695 lock_mi_rip_u,
696 /// Memory (SIB), immediate (sign-extend) operands with lock prefix.
697 /// Uses `xi` payload with extra data of type `MemorySib`.
698 lock_mi_sib_s,
699 /// Memory (RIP), immediate (sign-extend) operands with lock prefix.
700 /// Uses `xi` payload with extra data of type `MemoryRip`.
701 lock_mi_rip_s,
702 /// Memory (SIB), register operands with lock prefix.
703 /// Uses `rx` payload with extra data of type `MemorySib`.
704 lock_mr_sib,
705 /// Memory (RIP), register operands with lock prefix.
706 /// Uses `rx` payload with extra data of type `MemoryRip`.
707 lock_mr_rip,
708 /// Memory moffs, rax with lock prefix.
709 /// Uses `payload` with extra data of type `MemoryMoffs`.
710 lock_moffs_rax,
711 /// References another Mir instruction directly.838 /// References another Mir instruction directly.
712 /// Uses `inst` payload.839 /// Uses `inst` payload.
713 inst,840 inst,
714 /// References another Mir instruction directly with condition code (CC).841 /// Linker relocation - external function.
715 /// Uses `inst_cc` payload.
716 inst_cc,
717 /// String repeat and width
718 /// Uses `string` payload.
719 string,
720 /// Uses `reloc` payload.842 /// Uses `reloc` payload.
721 reloc,843 extern_fn_reloc,
722 /// Linker relocation - GOT indirection.844 /// Linker relocation - GOT indirection.
723 /// Uses `rx` payload with extra data of type `Reloc`.845 /// Uses `rx` payload with extra data of type `Reloc`.
724 got_reloc,846 got_reloc,
...@@ -731,74 +853,125 @@ pub const Inst = struct {...@@ -731,74 +853,125 @@ pub const Inst = struct {
731 /// Linker relocation - threadlocal variable via GOT indirection.853 /// Linker relocation - threadlocal variable via GOT indirection.
732 /// Uses `rx` payload with extra data of type `Reloc`.854 /// Uses `rx` payload with extra data of type `Reloc`.
733 tlv_reloc,855 tlv_reloc,
856
857 // Pseudo instructions:
858
859 /// Conditional move if zero flag set and parity flag not set
860 /// Clobbers the source operand!
861 /// Uses `rr` payload.
862 pseudo_cmov_z_and_np_rr,
863 /// Conditional move if zero flag not set or parity flag set
864 /// Uses `rr` payload.
865 pseudo_cmov_nz_or_p_rr,
866 /// Conditional move if zero flag not set or parity flag set
867 /// Uses `rx` payload.
868 pseudo_cmov_nz_or_p_rm_sib,
869 /// Conditional move if zero flag not set or parity flag set
870 /// Uses `rx` payload.
871 pseudo_cmov_nz_or_p_rm_rip,
872 /// Set byte if zero flag set and parity flag not set
873 /// Requires a scratch register!
874 /// Uses `r_scratch` payload.
875 pseudo_set_z_and_np_r,
876 /// Set byte if zero flag set and parity flag not set
877 /// Requires a scratch register!
878 /// Uses `x_scratch` payload.
879 pseudo_set_z_and_np_m_sib,
880 /// Set byte if zero flag set and parity flag not set
881 /// Requires a scratch register!
882 /// Uses `x_scratch` payload.
883 pseudo_set_z_and_np_m_rip,
884 /// Set byte if zero flag not set or parity flag set
885 /// Requires a scratch register!
886 /// Uses `r_scratch` payload.
887 pseudo_set_nz_or_p_r,
888 /// Set byte if zero flag not set or parity flag set
889 /// Requires a scratch register!
890 /// Uses `x_scratch` payload.
891 pseudo_set_nz_or_p_m_sib,
892 /// Set byte if zero flag not set or parity flag set
893 /// Requires a scratch register!
894 /// Uses `x_scratch` payload.
895 pseudo_set_nz_or_p_m_rip,
896 /// Jump if zero flag set and parity flag not set
897 /// Uses `inst` payload.
898 pseudo_j_z_and_np_inst,
899 /// Jump if zero flag not set or parity flag set
900 /// Uses `inst` payload.
901 pseudo_j_nz_or_p_inst,
902
903 /// Push registers
904 /// Uses `reg_list` payload.
905 pseudo_push_reg_list,
906 /// Pop registers
907 /// Uses `reg_list` payload.
908 pseudo_pop_reg_list,
909
910 /// End of prologue
911 pseudo_dbg_prologue_end_none,
912 /// Update debug line
913 /// Uses `line_column` payload.
914 pseudo_dbg_line_line_column,
915 /// Start of epilogue
916 pseudo_dbg_epilogue_begin_none,
917
918 /// Tombstone
919 /// Emitter should skip this instruction.
920 pseudo_dead_none,
734 };921 };
735922
736 pub const Data = union {923 pub const Data = union {
924 none: struct {
925 fixes: Fixes = ._,
926 },
737 /// References another Mir instruction.927 /// References another Mir instruction.
738 inst: Index,928 inst: struct {
739 /// Another instruction with condition code (CC).929 fixes: Fixes = ._,
740 /// Used by `jcc`.
741 inst_cc: struct {
742 /// Another instruction.
743 inst: Index,930 inst: Index,
744 /// A condition code for use with EFLAGS register.
745 cc: bits.Condition,
746 },931 },
747 /// A 32-bit immediate value.932 /// A 32-bit immediate value.
748 i: u32,933 i: struct {
749 r: Register,934 fixes: Fixes = ._,
935 i: u32,
936 },
937 r: struct {
938 fixes: Fixes = ._,
939 r1: Register,
940 },
750 rr: struct {941 rr: struct {
942 fixes: Fixes = ._,
751 r1: Register,943 r1: Register,
752 r2: Register,944 r2: Register,
753 },945 },
754 rrr: struct {946 rrr: struct {
947 fixes: Fixes = ._,
755 r1: Register,948 r1: Register,
756 r2: Register,949 r2: Register,
757 r3: Register,950 r3: Register,
758 },951 },
759 rrri: struct {952 rrri: struct {
953 fixes: Fixes = ._,
760 r1: Register,954 r1: Register,
761 r2: Register,955 r2: Register,
762 r3: Register,956 r3: Register,
763 i: u8,957 i: u8,
764 },958 },
765 rri: struct {959 rri: struct {
960 fixes: Fixes = ._,
766 r1: Register,961 r1: Register,
767 r2: Register,962 r2: Register,
768 i: u32,963 i: u32,
769 },964 },
770 /// Condition code (CC), followed by custom payload found in extra.
771 x_cc: struct {
772 scratch: Register,
773 cc: bits.Condition,
774 payload: u32,
775 },
776 /// Register with condition code (CC).
777 r_cc: struct {
778 r: Register,
779 scratch: Register,
780 cc: bits.Condition,
781 },
782 /// Register, register with condition code (CC).
783 rr_cc: struct {
784 r1: Register,
785 r2: Register,
786 cc: bits.Condition,
787 },
788 /// Register, immediate.965 /// Register, immediate.
789 ri: struct {966 ri: struct {
790 r: Register,967 fixes: Fixes = ._,
968 r1: Register,
791 i: u32,969 i: u32,
792 },970 },
793 /// Register, followed by custom payload found in extra.971 /// Register, followed by custom payload found in extra.
794 rx: struct {972 rx: struct {
795 r: Register,973 fixes: Fixes = ._,
796 payload: u32,974 r1: Register,
797 },
798 /// Register with condition code (CC), followed by custom payload found in extra.
799 rx_cc: struct {
800 r: Register,
801 cc: bits.Condition,
802 payload: u32,975 payload: u32,
803 },976 },
804 /// Immediate, followed by Custom payload found in extra.977 /// Immediate, followed by Custom payload found in extra.
...@@ -808,39 +981,54 @@ pub const Inst = struct {...@@ -808,39 +981,54 @@ pub const Inst = struct {
808 },981 },
809 /// Register, register, followed by Custom payload found in extra.982 /// Register, register, followed by Custom payload found in extra.
810 rrx: struct {983 rrx: struct {
984 fixes: Fixes = ._,
811 r1: Register,985 r1: Register,
812 r2: Register,986 r2: Register,
813 payload: u32,987 payload: u32,
814 },988 },
815 /// Register, byte immediate, followed by Custom payload found in extra.989 /// Register, byte immediate, followed by Custom payload found in extra.
816 rix: struct {990 rix: struct {
817 r: Register,991 fixes: Fixes = ._,
992 r1: Register,
818 i: u8,993 i: u8,
819 payload: u32,994 payload: u32,
820 },995 },
821 /// Register, register, byte immediate, followed by Custom payload found in extra.996 /// Register, register, byte immediate, followed by Custom payload found in extra.
822 rrix: struct {997 rrix: struct {
998 fixes: Fixes = ._,
823 r1: Register,999 r1: Register,
824 r2: Register,1000 r2: Register,
825 i: u8,1001 i: u8,
826 payload: u32,1002 payload: u32,
827 },1003 },
828 /// String instruction prefix and width.1004 /// Register, scratch register
829 string: struct {1005 r_scratch: struct {
830 repeat: bits.StringRepeat,1006 fixes: Fixes = ._,
831 width: bits.StringWidth,1007 r1: Register,
1008 scratch_reg: Register,
1009 },
1010 /// Scratch register, followed by Custom payload found in extra.
1011 x_scratch: struct {
1012 fixes: Fixes = ._,
1013 scratch_reg: Register,
1014 payload: u32,
1015 },
1016 /// Custom payload found in extra.
1017 x: struct {
1018 fixes: Fixes = ._,
1019 payload: u32,
832 },1020 },
833 /// Relocation for the linker where:1021 /// Relocation for the linker where:
834 /// * `atom_index` is the index of the source1022 /// * `atom_index` is the index of the source
835 /// * `sym_index` is the index of the target1023 /// * `sym_index` is the index of the target
836 relocation: Reloc,1024 reloc: Reloc,
837 /// Debug line and column position1025 /// Debug line and column position
838 line_column: struct {1026 line_column: struct {
839 line: u32,1027 line: u32,
840 column: u32,1028 column: u32,
841 },1029 },
842 /// Index into `extra`. Meaning of what can be found there is context-dependent.1030 /// Register list
843 payload: u32,1031 reg_list: RegisterList,
844 };1032 };
8451033
846 // Make sure we don't accidentally make instructions bigger than expected.1034 // Make sure we don't accidentally make instructions bigger than expected.
...@@ -852,6 +1040,7 @@ pub const Inst = struct {...@@ -852,6 +1040,7 @@ pub const Inst = struct {
852 }1040 }
853};1041};
8541042
1043/// A linker symbol not yet allocated in VM.
855pub const Reloc = struct {1044pub const Reloc = struct {
856 /// Index of the containing atom.1045 /// Index of the containing atom.
857 atom_index: u32,1046 atom_index: u32,
...@@ -887,16 +1076,6 @@ pub const RegisterList = struct {...@@ -887,16 +1076,6 @@ pub const RegisterList = struct {
887 return self.bitset.iterator(options);1076 return self.bitset.iterator(options);
888 }1077 }
8891078
890 pub fn asInt(self: Self) u32 {
891 return self.bitset.mask;
892 }
893
894 pub fn fromInt(mask: u32) Self {
895 return .{
896 .bitset = BitSet{ .mask = @intCast(BitSet.MaskInt, mask) },
897 };
898 }
899
900 pub fn count(self: Self) u32 {1079 pub fn count(self: Self) u32 {
901 return @intCast(u32, self.bitset.count());1080 return @intCast(u32, self.bitset.count());
902 }1081 }
src/arch/x86_64/bits.zig-3
...@@ -6,9 +6,6 @@ const Allocator = std.mem.Allocator;...@@ -6,9 +6,6 @@ const Allocator = std.mem.Allocator;
6const ArrayList = std.ArrayList;6const ArrayList = std.ArrayList;
7const DW = std.dwarf;7const DW = std.dwarf;
88
9pub const StringRepeat = enum(u3) { none, rep, repe, repz, repne, repnz };
10pub const StringWidth = enum(u2) { b, w, d, q };
11
12/// EFLAGS condition codes9/// EFLAGS condition codes
13pub const Condition = enum(u5) {10pub const Condition = enum(u5) {
14 /// above11 /// above