Commit 3fde14035b013646f42519189dbaa4534564d78b | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2022-05-17 09:20:02+02:00
committer	GitHub <noreply@github.com> 2022-05-17 09:20:02+02:00
log	3fde14035b013646f42519189dbaa4534564d78b
tree	002d2027d53644cb09dd1421051dda23570c135d
parent	a4369918b19e4920f51f40a2b05781dda45462f7
parent	b618dbdf6989235242ef35f9b676848794fc2229
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11658 from koachan/sparc64-codegen

stage2: sparc64: Make basic test harness run

4 files changed, 1908 insertions(+), 61 deletions(-)

src
- arch
  - sparc64
    CodeGen.zig
    Emit.zig
    Mir.zig
    bits.zig

src/arch/sparc64/CodeGen.zig+1407-36

...	...	@@ -23,11 +23,14 @@ const FnResult = @import("../../codegen.zig").FnResult;
23	23	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
24	24	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25	25	const RegisterManager = RegisterManagerFn(Self, Register, &abi.allocatable_regs);
	26	const RegisterLock = RegisterManager.RegisterLock;
26	27
27	28	const build_options = @import("build_options");
28	29
29	30	const bits = @import("bits.zig");
30	31	const abi = @import("abi.zig");
	32	const Instruction = bits.Instruction;
	33	const ShiftWidth = Instruction.ShiftWidth;
31	34	const Register = bits.Register;
32	35
33	36	const Self = @This();
...	...	@@ -90,6 +93,9 @@ register_manager: RegisterManager = .{},
90	93	/// Maps offset to what is stored there.
91	94	stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
92	95
	96	/// Tracks the current instruction allocated to the compare flags
	97	compare_flags_inst: ?Air.Inst.Index = null,
	98
93	99	/// Offset from the stack base, representing the end of the stack frame.
94	100	max_end_stack: u32 = 0,
95	101	/// Represents the current end stack offset. If there is no existing slot
...	...	@@ -125,6 +131,12 @@ const MCValue = union(enum) {
125	131	stack_offset: u32,
126	132	/// The value is a pointer to one of the stack variables (payload is stack offset).
127	133	ptr_stack_offset: u32,
	134	/// The value is in the compare flags assuming an unsigned operation,
	135	/// with this operator applied on top of it.
	136	compare_flags_unsigned: math.CompareOperator,
	137	/// The value is in the compare flags assuming a signed operation,
	138	/// with this operator applied on top of it.
	139	compare_flags_signed: math.CompareOperator,
128	140
129	141	fn isMemory(mcv: MCValue) bool {
130	142	return switch (mcv) {
...	...	@@ -367,18 +379,31 @@ fn gen(self: *Self) !void {
367	379
368	380	// exitlude jumps
369	381	if (self.exitlude_jump_relocs.items.len > 0 and
370		self.exitlude_jump_relocs.items[self.exitlude_jump_relocs.items.len - 1] == self.mir_instructions.len - 2)
	382	self.exitlude_jump_relocs.items[self.exitlude_jump_relocs.items.len - 1] == self.mir_instructions.len - 3)
371	383	{
372	384	// If the last Mir instruction (apart from the
373	385	// dbg_epilogue_begin) is the last exitlude jump
374		// relocation (which would just jump one instruction
	386	// relocation (which would just jump two instructions
375	387	// further), it can be safely removed
376		self.mir_instructions.orderedRemove(self.exitlude_jump_relocs.pop());
	388	const index = self.exitlude_jump_relocs.pop();
	389
	390	// First, remove the delay slot, then remove
	391	// the branch instruction itself.
	392	self.mir_instructions.orderedRemove(index + 1);
	393	self.mir_instructions.orderedRemove(index);
377	394	}
378	395
379	396	for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
380		_ = jmp_reloc;
381		return self.fail("TODO add branches in sparc64", .{});
	397	self.mir_instructions.set(jmp_reloc, .{
	398	.tag = .bpcc,
	399	.data = .{
	400	.branch_predict_int = .{
	401	.ccr = .xcc,
	402	.cond = .al,
	403	.inst = @intCast(u32, self.mir_instructions.len),
	404	},
	405	},
	406	});
382	407	}
383	408
384	409	// Backpatch stack offset
...	...	@@ -458,7 +483,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
458	483
459	484	switch (air_tags[inst]) {
460	485	// zig fmt: off
461		.add, .ptr_add => @panic("TODO try self.airBinOp(inst)"),
	486	.add, .ptr_add => try self.airBinOp(inst),
462	487	.addwrap => @panic("TODO try self.airAddWrap(inst)"),
463	488	.add_sat => @panic("TODO try self.airAddSat(inst)"),
464	489	.sub, .ptr_sub => @panic("TODO try self.airBinOp(inst)"),
...	...	@@ -498,12 +523,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
498	523
499	524	.div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst),
500	525
501		.cmp_lt => @panic("TODO try self.airCmp(inst, .lt)"),
502		.cmp_lte => @panic("TODO try self.airCmp(inst, .lte)"),
503		.cmp_eq => @panic("TODO try self.airCmp(inst, .eq)"),
504		.cmp_gte => @panic("TODO try self.airCmp(inst, .gte)"),
505		.cmp_gt => @panic("TODO try self.airCmp(inst, .gt)"),
506		.cmp_neq => @panic("TODO try self.airCmp(inst, .neq)"),
	526	.cmp_lt => try self.airCmp(inst, .lt),
	527	.cmp_lte => try self.airCmp(inst, .lte),
	528	.cmp_eq => try self.airCmp(inst, .eq),
	529	.cmp_gte => try self.airCmp(inst, .gte),
	530	.cmp_gt => try self.airCmp(inst, .gt),
	531	.cmp_neq => try self.airCmp(inst, .neq),
507	532	.cmp_vector => @panic("TODO try self.airCmpVector(inst)"),
508	533	.cmp_lt_errors_len => @panic("TODO try self.airCmpLtErrorsLen(inst)"),
509	534
...	...	@@ -514,18 +539,18 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
514	539	.xor => @panic("TODO try self.airXor(inst)"),
515	540	.shr, .shr_exact => @panic("TODO try self.airShr(inst)"),
516	541
517		.alloc => @panic("TODO try self.airAlloc(inst)"),
	542	.alloc => try self.airAlloc(inst),
518	543	.ret_ptr => try self.airRetPtr(inst),
519	544	.arg => try self.airArg(inst),
520	545	.assembly => try self.airAsm(inst),
521		.bitcast => @panic("TODO try self.airBitCast(inst)"),
	546	.bitcast => try self.airBitCast(inst),
522	547	.block => try self.airBlock(inst),
523		.br => @panic("TODO try self.airBr(inst)"),
	548	.br => try self.airBr(inst),
524	549	.breakpoint => try self.airBreakpoint(),
525	550	.ret_addr => @panic("TODO try self.airRetAddr(inst)"),
526	551	.frame_addr => @panic("TODO try self.airFrameAddress(inst)"),
527	552	.fence => @panic("TODO try self.airFence()"),
528		.cond_br => @panic("TODO try self.airCondBr(inst)"),
	553	.cond_br => try self.airCondBr(inst),
529	554	.dbg_stmt => try self.airDbgStmt(inst),
530	555	.fptrunc => @panic("TODO try self.airFptrunc(inst)"),
531	556	.fpext => @panic("TODO try self.airFpext(inst)"),
...	...	@@ -536,12 +561,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
536	561	.is_non_null_ptr => @panic("TODO try self.airIsNonNullPtr(inst)"),
537	562	.is_null => @panic("TODO try self.airIsNull(inst)"),
538	563	.is_null_ptr => @panic("TODO try self.airIsNullPtr(inst)"),
539		.is_non_err => @panic("TODO try self.airIsNonErr(inst)"),
	564	.is_non_err => try self.airIsNonErr(inst),
540	565	.is_non_err_ptr => @panic("TODO try self.airIsNonErrPtr(inst)"),
541		.is_err => @panic("TODO try self.airIsErr(inst)"),
	566	.is_err => try self.airIsErr(inst),
542	567	.is_err_ptr => @panic("TODO try self.airIsErrPtr(inst)"),
543		.load => @panic("TODO try self.airLoad(inst)"),
544		.loop => @panic("TODO try self.airLoop(inst)"),
	568	.load => try self.airLoad(inst),
	569	.loop => try self.airLoop(inst),
545	570	.not => @panic("TODO try self.airNot(inst)"),
546	571	.ptrtoint => @panic("TODO try self.airPtrToInt(inst)"),
547	572	.ret => try self.airRet(inst),
...	...	@@ -598,22 +623,22 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
598	623	.atomic_store_release => @panic("TODO try self.airAtomicStore(inst, .Release)"),
599	624	.atomic_store_seq_cst => @panic("TODO try self.airAtomicStore(inst, .SeqCst)"),
600	625
601		.struct_field_ptr_index_0 => @panic("TODO try self.airStructFieldPtrIndex(inst, 0)"),
602		.struct_field_ptr_index_1 => @panic("TODO try self.airStructFieldPtrIndex(inst, 1)"),
603		.struct_field_ptr_index_2 => @panic("TODO try self.airStructFieldPtrIndex(inst, 2)"),
604		.struct_field_ptr_index_3 => @panic("TODO try self.airStructFieldPtrIndex(inst, 3)"),
	626	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
	627	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
	628	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
	629	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
605	630
606	631	.field_parent_ptr => @panic("TODO try self.airFieldParentPtr(inst)"),
607	632
608	633	.switch_br => try self.airSwitch(inst),
609	634	.slice_ptr => @panic("TODO try self.airSlicePtr(inst)"),
610		.slice_len => @panic("TODO try self.airSliceLen(inst)"),
	635	.slice_len => try self.airSliceLen(inst),
611	636
612	637	.ptr_slice_len_ptr => @panic("TODO try self.airPtrSliceLenPtr(inst)"),
613	638	.ptr_slice_ptr_ptr => @panic("TODO try self.airPtrSlicePtrPtr(inst)"),
614	639
615	640	.array_elem_val => @panic("TODO try self.airArrayElemVal(inst)"),
616		.slice_elem_val => @panic("TODO try self.airSliceElemVal(inst)"),
	641	.slice_elem_val => try self.airSliceElemVal(inst),
617	642	.slice_elem_ptr => @panic("TODO try self.airSliceElemPtr(inst)"),
618	643	.ptr_elem_val => @panic("TODO try self.airPtrElemVal(inst)"),
619	644	.ptr_elem_ptr => @panic("TODO try self.airPtrElemPtr(inst)"),
...	...	@@ -625,8 +650,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
625	650	.optional_payload => @panic("TODO try self.airOptionalPayload(inst)"),
626	651	.optional_payload_ptr => @panic("TODO try self.airOptionalPayloadPtr(inst)"),
627	652	.optional_payload_ptr_set => @panic("TODO try self.airOptionalPayloadPtrSet(inst)"),
628		.unwrap_errunion_err => @panic("TODO try self.airUnwrapErrErr(inst)"),
629		.unwrap_errunion_payload => @panic("TODO try self.airUnwrapErrPayload(inst)"),
	653	.unwrap_errunion_err => try self.airUnwrapErrErr(inst),
	654	.unwrap_errunion_payload => try self.airUnwrapErrPayload(inst),
630	655	.unwrap_errunion_err_ptr => @panic("TODO try self.airUnwrapErrErrPtr(inst)"),
631	656	.unwrap_errunion_payload_ptr=> @panic("TODO try self.airUnwrapErrPayloadPtr(inst)"),
632	657	.errunion_payload_ptr_set => @panic("TODO try self.airErrUnionPayloadPtrSet(inst)"),
...	...	@@ -648,6 +673,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
648	673	}
649	674	}
650	675
	676	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
	677	const stack_offset = try self.allocMemPtr(inst);
	678	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
	679	}
	680
651	681	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
652	682	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
653	683	const extra = self.air.extraData(Air.Asm, ty_pl.payload);
...	...	@@ -719,7 +749,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
719	749	.data = .{
720	750	.trap = .{
721	751	.is_imm = true,
722		.cond = 0b1000, // TODO need to look into changing this into an enum
	752	.cond = .al,
723	753	.rs2_or_imm = .{ .imm = 0x6d },
724	754	},
725	755	},
...	...	@@ -795,6 +825,27 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
795	825	return self.finishAir(inst, mcv, .{ .none, .none, .none });
796	826	}
797	827
	828	fn airBinOp(self: *Self, inst: Air.Inst.Index) !void {
	829	const tag = self.air.instructions.items(.tag)[inst];
	830	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	831	const lhs = try self.resolveInst(bin_op.lhs);
	832	const rhs = try self.resolveInst(bin_op.rhs);
	833	const lhs_ty = self.air.typeOf(bin_op.lhs);
	834	const rhs_ty = self.air.typeOf(bin_op.rhs);
	835
	836	const result: MCValue = if (self.liveness.isUnused(inst))
	837	.dead
	838	else
	839	try self.binOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
	840	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	841	}
	842
	843	fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
	844	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	845	const result = try self.resolveInst(ty_op.operand);
	846	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	847	}
	848
798	849	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
799	850	try self.blocks.putNoClobber(self.gpa, inst, .{
800	851	// A block is a setup to be able to jump to the end.
...	...	@@ -829,6 +880,12 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
829	880	return self.finishAir(inst, result, .{ .none, .none, .none });
830	881	}
831	882
	883	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
	884	const branch = self.air.instructions.items(.data)[inst].br;
	885	try self.br(branch.block_inst, branch.operand);
	886	return self.finishAir(inst, .dead, .{ branch.operand, .none, .none });
	887	}
	888
832	889	fn airBreakpoint(self: *Self) !void {
833	890	// ta 0x01
834	891	_ = try self.addInst(.{
...	...	@@ -836,7 +893,7 @@ fn airBreakpoint(self: *Self) !void {
836	893	.data = .{
837	894	.trap = .{
838	895	.is_imm = true,
839		.cond = 0b1000, // TODO need to look into changing this into an enum
	896	.cond = .al,
840	897	.rs2_or_imm = .{ .imm = 0x01 },
841	898	},
842	899	},
...	...	@@ -872,6 +929,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
872	929	.unreach => unreachable,
873	930	.dead => unreachable,
874	931	.memory => unreachable,
	932	.compare_flags_signed => unreachable,
	933	.compare_flags_unsigned => unreachable,
875	934	.register => \|reg\| {
876	935	try self.register_manager.getReg(reg, null);
877	936	try self.genSetReg(arg_ty, reg, arg_mcv);
...	...	@@ -960,6 +1019,252 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
960	1019	@panic("TODO handle return value with BigTomb");
961	1020	}
962	1021
	1022	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
	1023	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1024	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1025	const lhs = try self.resolveInst(bin_op.lhs);
	1026	const rhs = try self.resolveInst(bin_op.rhs);
	1027	const lhs_ty = self.air.typeOf(bin_op.lhs);
	1028
	1029	var int_buffer: Type.Payload.Bits = undefined;
	1030	const int_ty = switch (lhs_ty.zigTypeTag()) {
	1031	.Vector => unreachable, // Should be handled by cmp_vector?
	1032	.Enum => lhs_ty.intTagType(&int_buffer),
	1033	.Int => lhs_ty,
	1034	.Bool => Type.initTag(.u1),
	1035	.Pointer => Type.usize,
	1036	.ErrorSet => Type.initTag(.u16),
	1037	.Optional => blk: {
	1038	var opt_buffer: Type.Payload.ElemType = undefined;
	1039	const payload_ty = lhs_ty.optionalChild(&opt_buffer);
	1040	if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
	1041	break :blk Type.initTag(.u1);
	1042	} else if (lhs_ty.isPtrLikeOptional()) {
	1043	break :blk Type.usize;
	1044	} else {
	1045	return self.fail("TODO SPARCv9 cmp non-pointer optionals", .{});
	1046	}
	1047	},
	1048	.Float => return self.fail("TODO SPARCv9 cmp floats", .{}),
	1049	else => unreachable,
	1050	};
	1051
	1052	const int_info = int_ty.intInfo(self.target.*);
	1053	if (int_info.bits <= 64) {
	1054	_ = try self.binOp(.cmp_eq, inst, lhs, rhs, int_ty, int_ty);
	1055
	1056	try self.spillCompareFlagsIfOccupied();
	1057	self.compare_flags_inst = inst;
	1058
	1059	break :result switch (int_info.signedness) {
	1060	.signed => MCValue{ .compare_flags_signed = op },
	1061	.unsigned => MCValue{ .compare_flags_unsigned = op },
	1062	};
	1063	} else {
	1064	return self.fail("TODO SPARCv9 cmp for ints > 64 bits", .{});
	1065	}
	1066	};
	1067	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	1068	}
	1069
	1070	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
	1071	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
	1072	const cond = try self.resolveInst(pl_op.operand);
	1073	const extra = self.air.extraData(Air.CondBr, pl_op.payload);
	1074	const then_body = self.air.extra[extra.end..][0..extra.data.then_body_len];
	1075	const else_body = self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len];
	1076	const liveness_condbr = self.liveness.getCondBr(inst);
	1077
	1078	// Here we either emit a BPcc for branching on CCR content,
	1079	// or emit a BPr to branch on register content.
	1080	const reloc: Mir.Inst.Index = switch (cond) {
	1081	.compare_flags_signed,
	1082	.compare_flags_unsigned,
	1083	=> try self.addInst(.{
	1084	.tag = .bpcc,
	1085	.data = .{
	1086	.branch_predict_int = .{
	1087	.ccr = .xcc,
	1088	.cond = switch (cond) {
	1089	.compare_flags_signed => \|cmp_op\| blk: {
	1090	// Here we map to the opposite condition because the jump is to the false branch.
	1091	const condition = Instruction.ICondition.fromCompareOperatorSigned(cmp_op);
	1092	break :blk condition.negate();
	1093	},
	1094	.compare_flags_unsigned => \|cmp_op\| blk: {
	1095	// Here we map to the opposite condition because the jump is to the false branch.
	1096	const condition = Instruction.ICondition.fromCompareOperatorUnsigned(cmp_op);
	1097	break :blk condition.negate();
	1098	},
	1099	else => unreachable,
	1100	},
	1101	.inst = undefined, // Will be filled by performReloc
	1102	},
	1103	},
	1104	}),
	1105	else => blk: {
	1106	const reg = switch (cond) {
	1107	.register => \|r\| r,
	1108	else => try self.copyToTmpRegister(Type.bool, cond),
	1109	};
	1110
	1111	break :blk try self.addInst(.{
	1112	.tag = .bpr,
	1113	.data = .{
	1114	.branch_predict_reg = .{
	1115	.cond = .eq_zero,
	1116	.rs1 = reg,
	1117	.inst = undefined, // populated later through performReloc
	1118	},
	1119	},
	1120	});
	1121	},
	1122	};
	1123
	1124	// Regardless of the branch type that's emitted, we need to reserve
	1125	// a space for the delay slot.
	1126	// TODO Find a way to fill this delay slot
	1127	_ = try self.addInst(.{
	1128	.tag = .nop,
	1129	.data = .{ .nop = {} },
	1130	});
	1131
	1132	// If the condition dies here in this condbr instruction, process
	1133	// that death now instead of later as this has an effect on
	1134	// whether it needs to be spilled in the branches
	1135	if (self.liveness.operandDies(inst, 0)) {
	1136	const op_int = @enumToInt(pl_op.operand);
	1137	if (op_int >= Air.Inst.Ref.typed_value_map.len) {
	1138	const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
	1139	self.processDeath(op_index);
	1140	}
	1141	}
	1142
	1143	// Capture the state of register and stack allocation state so that we can revert to it.
	1144	const parent_next_stack_offset = self.next_stack_offset;
	1145	const parent_free_registers = self.register_manager.free_registers;
	1146	var parent_stack = try self.stack.clone(self.gpa);
	1147	defer parent_stack.deinit(self.gpa);
	1148	const parent_registers = self.register_manager.registers;
	1149	const parent_compare_flags_inst = self.compare_flags_inst;
	1150
	1151	try self.branch_stack.append(.{});
	1152	errdefer {
	1153	_ = self.branch_stack.pop();
	1154	}
	1155
	1156	try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
	1157	for (liveness_condbr.then_deaths) \|operand\| {
	1158	self.processDeath(operand);
	1159	}
	1160	try self.genBody(then_body);
	1161
	1162	// Revert to the previous register and stack allocation state.
	1163
	1164	var saved_then_branch = self.branch_stack.pop();
	1165	defer saved_then_branch.deinit(self.gpa);
	1166
	1167	self.register_manager.registers = parent_registers;
	1168	self.compare_flags_inst = parent_compare_flags_inst;
	1169
	1170	self.stack.deinit(self.gpa);
	1171	self.stack = parent_stack;
	1172	parent_stack = .{};
	1173
	1174	self.next_stack_offset = parent_next_stack_offset;
	1175	self.register_manager.free_registers = parent_free_registers;
	1176
	1177	try self.performReloc(reloc);
	1178	const else_branch = self.branch_stack.addOneAssumeCapacity();
	1179	else_branch.* = .{};
	1180
	1181	try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
	1182	for (liveness_condbr.else_deaths) \|operand\| {
	1183	self.processDeath(operand);
	1184	}
	1185	try self.genBody(else_body);
	1186
	1187	// At this point, each branch will possibly have conflicting values for where
	1188	// each instruction is stored. They agree, however, on which instructions are alive/dead.
	1189	// We use the first ("then") branch as canonical, and here emit
	1190	// instructions into the second ("else") branch to make it conform.
	1191	// We continue respect the data structure semantic guarantees of the else_branch so
	1192	// that we can use all the code emitting abstractions. This is why at the bottom we
	1193	// assert that parent_branch.free_registers equals the saved_then_branch.free_registers
	1194	// rather than assigning it.
	1195	const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2];
	1196	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count());
	1197
	1198	const else_slice = else_branch.inst_table.entries.slice();
	1199	const else_keys = else_slice.items(.key);
	1200	const else_values = else_slice.items(.value);
	1201	for (else_keys) \|else_key, else_idx\| {
	1202	const else_value = else_values[else_idx];
	1203	const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) \|then_entry\| blk: {
	1204	// The instruction's MCValue is overridden in both branches.
	1205	parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value);
	1206	if (else_value == .dead) {
	1207	assert(then_entry.value == .dead);
	1208	continue;
	1209	}
	1210	break :blk then_entry.value;
	1211	} else blk: {
	1212	if (else_value == .dead)
	1213	continue;
	1214	// The instruction is only overridden in the else branch.
	1215	var i: usize = self.branch_stack.items.len - 2;
	1216	while (true) {
	1217	i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
	1218	if (self.branch_stack.items[i].inst_table.get(else_key)) \|mcv\| {
	1219	assert(mcv != .dead);
	1220	break :blk mcv;
	1221	}
	1222	}
	1223	};
	1224	log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv });
	1225	// TODO make sure the destination stack offset / register does not already have something
	1226	// going on there.
	1227	try self.setRegOrMem(self.air.typeOfIndex(else_key), canon_mcv, else_value);
	1228	// TODO track the new register / stack allocation
	1229	}
	1230	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count());
	1231	const then_slice = saved_then_branch.inst_table.entries.slice();
	1232	const then_keys = then_slice.items(.key);
	1233	const then_values = then_slice.items(.value);
	1234	for (then_keys) \|then_key, then_idx\| {
	1235	const then_value = then_values[then_idx];
	1236	// We already deleted the items from this table that matched the else_branch.
	1237	// So these are all instructions that are only overridden in the then branch.
	1238	parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
	1239	if (then_value == .dead)
	1240	continue;
	1241	const parent_mcv = blk: {
	1242	var i: usize = self.branch_stack.items.len - 2;
	1243	while (true) {
	1244	i -= 1;
	1245	if (self.branch_stack.items[i].inst_table.get(then_key)) \|mcv\| {
	1246	assert(mcv != .dead);
	1247	break :blk mcv;
	1248	}
	1249	}
	1250	};
	1251	log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value });
	1252	// TODO make sure the destination stack offset / register does not already have something
	1253	// going on there.
	1254	try self.setRegOrMem(self.air.typeOfIndex(then_key), parent_mcv, then_value);
	1255	// TODO track the new register / stack allocation
	1256	}
	1257
	1258	{
	1259	var item = self.branch_stack.pop();
	1260	item.deinit(self.gpa);
	1261	}
	1262
	1263	// We already took care of pl_op.operand earlier, so we're going
	1264	// to pass .none here
	1265	return self.finishAir(inst, .unreach, .{ .none, .none, .none });
	1266	}
	1267
963	1268	fn airDbgBlock(self: *Self, inst: Air.Inst.Index) !void {
964	1269	// TODO emit debug info lexical block
965	1270	return self.finishAir(inst, .dead, .{ .none, .none, .none });
...	...	@@ -1004,6 +1309,67 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
1004	1309	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1005	1310	}
1006	1311
	1312	fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
	1313	const un_op = self.air.instructions.items(.data)[inst].un_op;
	1314	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1315	const operand = try self.resolveInst(un_op);
	1316	const ty = self.air.typeOf(un_op);
	1317	break :result try self.isErr(ty, operand);
	1318	};
	1319	return self.finishAir(inst, result, .{ un_op, .none, .none });
	1320	}
	1321
	1322	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
	1323	const un_op = self.air.instructions.items(.data)[inst].un_op;
	1324	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1325	const operand = try self.resolveInst(un_op);
	1326	const ty = self.air.typeOf(un_op);
	1327	break :result try self.isNonErr(ty, operand);
	1328	};
	1329	return self.finishAir(inst, result, .{ un_op, .none, .none });
	1330	}
	1331
	1332	fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
	1333	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1334	const elem_ty = self.air.typeOfIndex(inst);
	1335	const elem_size = elem_ty.abiSize(self.target.*);
	1336	const result: MCValue = result: {
	1337	if (!elem_ty.hasRuntimeBits())
	1338	break :result MCValue.none;
	1339
	1340	const ptr = try self.resolveInst(ty_op.operand);
	1341	const is_volatile = self.air.typeOf(ty_op.operand).isVolatilePtr();
	1342	if (self.liveness.isUnused(inst) and !is_volatile)
	1343	break :result MCValue.dead;
	1344
	1345	const dst_mcv: MCValue = blk: {
	1346	if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
	1347	// The MCValue that holds the pointer can be re-used as the value.
	1348	break :blk switch (ptr) {
	1349	.register => \|r\| MCValue{ .register = r },
	1350	else => ptr,
	1351	};
	1352	} else {
	1353	break :blk try self.allocRegOrMem(inst, true);
	1354	}
	1355	};
	1356	try self.load(dst_mcv, ptr, self.air.typeOf(ty_op.operand));
	1357	break :result dst_mcv;
	1358	};
	1359	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1360	}
	1361
	1362	fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
	1363	// A loop is a setup to be able to jump back to the beginning.
	1364	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	1365	const loop = self.air.extraData(Air.Block, ty_pl.payload);
	1366	const body = self.air.extra[loop.end .. loop.end + loop.data.body_len];
	1367	const start = @intCast(u32, self.mir_instructions.len);
	1368	try self.genBody(body);
	1369	try self.jump(start);
	1370	return self.finishAirBookkeeping();
	1371	}
	1372
1007	1373	fn airRet(self: *Self, inst: Air.Inst.Index) !void {
1008	1374	const un_op = self.air.instructions.items(.data)[inst].un_op;
1009	1375	const operand = try self.resolveInst(un_op);
...	...	@@ -1024,11 +1390,87 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
1024	1390	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
1025	1391	}
1026	1392
	1393	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
	1394	const is_volatile = false; // TODO
	1395	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1396
	1397	if (!is_volatile and self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
	1398	const result: MCValue = result: {
	1399	const slice_mcv = try self.resolveInst(bin_op.lhs);
	1400	const index_mcv = try self.resolveInst(bin_op.rhs);
	1401
	1402	const slice_ty = self.air.typeOf(bin_op.lhs);
	1403	const elem_ty = slice_ty.childType();
	1404	const elem_size = elem_ty.abiSize(self.target.*);
	1405
	1406	var buf: Type.SlicePtrFieldTypeBuffer = undefined;
	1407	const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
	1408
	1409	const index_lock: ?RegisterLock = if (index_mcv == .register)
	1410	self.register_manager.lockRegAssumeUnused(index_mcv.register)
	1411	else
	1412	null;
	1413	defer if (index_lock) \|reg\| self.register_manager.unlockReg(reg);
	1414
	1415	const base_mcv: MCValue = switch (slice_mcv) {
	1416	.stack_offset => \|off\| .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, .{ .stack_offset = off }) },
	1417	else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}),
	1418	};
	1419	const base_lock = self.register_manager.lockRegAssumeUnused(base_mcv.register);
	1420	defer self.register_manager.unlockReg(base_lock);
	1421
	1422	switch (elem_size) {
	1423	else => {
	1424	// TODO skip the ptr_add emission entirely and use native addressing modes
	1425	// i.e sllx/mulx then R+R or scale immediate then R+I
	1426	const dest = try self.allocRegOrMem(inst, true);
	1427	const addr = try self.binOp(.ptr_add, null, base_mcv, index_mcv, slice_ptr_field_type, Type.usize);
	1428	try self.load(dest, addr, slice_ptr_field_type);
	1429
	1430	break :result dest;
	1431	},
	1432	}
	1433	};
	1434	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
	1435	}
	1436
	1437	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
	1438	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1439	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1440	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
	1441	const ptr_bytes = @divExact(ptr_bits, 8);
	1442	const mcv = try self.resolveInst(ty_op.operand);
	1443	switch (mcv) {
	1444	.dead, .unreach, .none => unreachable,
	1445	.register => unreachable, // a slice doesn't fit in one register
	1446	.stack_offset => \|off\| {
	1447	break :result MCValue{ .stack_offset = off - ptr_bytes };
	1448	},
	1449	.memory => \|addr\| {
	1450	break :result MCValue{ .memory = addr + ptr_bytes };
	1451	},
	1452	else => return self.fail("TODO implement slice_len for {}", .{mcv}),
	1453	}
	1454	};
	1455	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1456	}
	1457
1027	1458	fn airStore(self: *Self, inst: Air.Inst.Index) !void {
1028		_ = self;
1029		_ = inst;
	1459	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1460	const ptr = try self.resolveInst(bin_op.lhs);
	1461	const value = try self.resolveInst(bin_op.rhs);
	1462	const ptr_ty = self.air.typeOf(bin_op.lhs);
	1463	const value_ty = self.air.typeOf(bin_op.rhs);
	1464
	1465	try self.store(ptr, value, ptr_ty, value_ty);
1030	1466
1031		return self.fail("TODO implement store for {}", .{self.target.cpu.arch});
	1467	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
	1468	}
	1469
	1470	fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
	1471	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1472	const result = try self.structFieldPtr(inst, ty_op.operand, index);
	1473	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1032	1474	}
1033	1475
1034	1476	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -1038,6 +1480,31 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
1038	1480	return self.fail("TODO implement switch for {}", .{self.target.cpu.arch});
1039	1481	}
1040	1482
	1483	fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
	1484	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1485	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1486	const error_union_ty = self.air.typeOf(ty_op.operand);
	1487	const payload_ty = error_union_ty.errorUnionPayload();
	1488	const mcv = try self.resolveInst(ty_op.operand);
	1489	if (!payload_ty.hasRuntimeBits()) break :result mcv;
	1490
	1491	return self.fail("TODO implement unwrap error union error for non-empty payloads", .{});
	1492	};
	1493	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1494	}
	1495
	1496	fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
	1497	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1498	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	1499	const error_union_ty = self.air.typeOf(ty_op.operand);
	1500	const payload_ty = error_union_ty.errorUnionPayload();
	1501	if (!payload_ty.hasRuntimeBits()) break :result MCValue.none;
	1502
	1503	return self.fail("TODO implement unwrap error union payload for non-empty payloads", .{});
	1504	};
	1505	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1506	}
	1507
1041	1508	// Common helper functions
1042	1509
1043	1510	/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
...	...	@@ -1126,6 +1593,459 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
1126	1593	return MCValue{ .stack_offset = stack_offset };
1127	1594	}
1128	1595
	1596	/// For all your binary operation needs, this function will generate
	1597	/// the corresponding Mir instruction(s). Returns the location of the
	1598	/// result.
	1599	///
	1600	/// If the binary operation itself happens to be an Air instruction,
	1601	/// pass the corresponding index in the inst parameter. That helps
	1602	/// this function do stuff like reusing operands.
	1603	///
	1604	/// This function does not do any lowering to Mir itself, but instead
	1605	/// looks at the lhs and rhs and determines which kind of lowering
	1606	/// would be best suitable and then delegates the lowering to other
	1607	/// functions.
	1608	fn binOp(
	1609	self: *Self,
	1610	tag: Air.Inst.Tag,
	1611	maybe_inst: ?Air.Inst.Index,
	1612	lhs: MCValue,
	1613	rhs: MCValue,
	1614	lhs_ty: Type,
	1615	rhs_ty: Type,
	1616	) InnerError!MCValue {
	1617	const mod = self.bin_file.options.module.?;
	1618	switch (tag) {
	1619	.add, .cmp_eq => {
	1620	switch (lhs_ty.zigTypeTag()) {
	1621	.Float => return self.fail("TODO binary operations on floats", .{}),
	1622	.Vector => return self.fail("TODO binary operations on vectors", .{}),
	1623	.Int => {
	1624	assert(lhs_ty.eql(rhs_ty, mod));
	1625	const int_info = lhs_ty.intInfo(self.target.*);
	1626	if (int_info.bits <= 64) {
	1627	// Only say yes if the operation is
	1628	// commutative, i.e. we can swap both of the
	1629	// operands
	1630	const lhs_immediate_ok = switch (tag) {
	1631	.add => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12),
	1632	.sub, .cmp_eq => false,
	1633	else => unreachable,
	1634	};
	1635	const rhs_immediate_ok = switch (tag) {
	1636	.add,
	1637	.sub,
	1638	.cmp_eq,
	1639	=> rhs == .immediate and rhs.immediate <= std.math.maxInt(u12),
	1640	else => unreachable,
	1641	};
	1642
	1643	const mir_tag: Mir.Inst.Tag = switch (tag) {
	1644	.add => .add,
	1645	.cmp_eq => .subcc,
	1646	else => unreachable,
	1647	};
	1648
	1649	if (rhs_immediate_ok) {
	1650	return try self.binOpImmediate(mir_tag, maybe_inst, lhs, rhs, lhs_ty, false);
	1651	} else if (lhs_immediate_ok) {
	1652	// swap lhs and rhs
	1653	return try self.binOpImmediate(mir_tag, maybe_inst, rhs, lhs, rhs_ty, true);
	1654	} else {
	1655	// TODO convert large immediates to register before adding
	1656	return try self.binOpRegister(mir_tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);
	1657	}
	1658	} else {
	1659	return self.fail("TODO binary operations on int with bits > 64", .{});
	1660	}
	1661	},
	1662	else => unreachable,
	1663	}
	1664	},
	1665
	1666	.mul => {
	1667	switch (lhs_ty.zigTypeTag()) {
	1668	.Vector => return self.fail("TODO binary operations on vectors", .{}),
	1669	.Int => {
	1670	assert(lhs_ty.eql(rhs_ty, mod));
	1671	const int_info = lhs_ty.intInfo(self.target.*);
	1672	if (int_info.bits <= 64) {
	1673	// If LHS is immediate, then swap it with RHS.
	1674	const lhs_is_imm = lhs == .immediate;
	1675	const new_lhs = if (lhs_is_imm) rhs else lhs;
	1676	const new_rhs = if (lhs_is_imm) lhs else rhs;
	1677	const new_lhs_ty = if (lhs_is_imm) rhs_ty else lhs_ty;
	1678	const new_rhs_ty = if (lhs_is_imm) lhs_ty else rhs_ty;
	1679
	1680	// At this point, RHS might be an immediate
	1681	// If it's a power of two immediate then we emit an shl instead
	1682	// TODO add similar checks for LHS
	1683	if (new_rhs == .immediate and math.isPowerOfTwo(new_rhs.immediate)) {
	1684	return try self.binOp(.shl, maybe_inst, new_lhs, .{ .immediate = math.log2(new_rhs.immediate) }, new_lhs_ty, Type.usize);
	1685	}
	1686
	1687	return try self.binOpRegister(.mulx, maybe_inst, new_lhs, new_rhs, new_lhs_ty, new_rhs_ty);
	1688	} else {
	1689	return self.fail("TODO binary operations on int with bits > 64", .{});
	1690	}
	1691	},
	1692	else => unreachable,
	1693	}
	1694	},
	1695
	1696	.ptr_add => {
	1697	switch (lhs_ty.zigTypeTag()) {
	1698	.Pointer => {
	1699	const ptr_ty = lhs_ty;
	1700	const elem_ty = switch (ptr_ty.ptrSize()) {
	1701	.One => ptr_ty.childType().childType(), // ptr to array, so get array element type
	1702	else => ptr_ty.childType(),
	1703	};
	1704	const elem_size = elem_ty.abiSize(self.target.*);
	1705
	1706	if (elem_size == 1) {
	1707	const base_tag: Mir.Inst.Tag = switch (tag) {
	1708	.ptr_add => .add,
	1709	else => unreachable,
	1710	};
	1711
	1712	return try self.binOpRegister(base_tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);
	1713	} else {
	1714	// convert the offset into a byte offset by
	1715	// multiplying it with elem_size
	1716
	1717	const offset = try self.binOp(.mul, null, rhs, .{ .immediate = elem_size }, Type.usize, Type.usize);
	1718	const addr = try self.binOp(tag, null, lhs, offset, Type.initTag(.manyptr_u8), Type.usize);
	1719	return addr;
	1720	}
	1721	},
	1722	else => unreachable,
	1723	}
	1724	},
	1725
	1726	.shl => {
	1727	const base_tag: Air.Inst.Tag = switch (tag) {
	1728	.shl => .shl_exact,
	1729	else => unreachable,
	1730	};
	1731
	1732	// Generate a shl_exact/shr_exact
	1733	const result = try self.binOp(base_tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);
	1734
	1735	// Truncate if necessary
	1736	switch (tag) {
	1737	.shl => switch (lhs_ty.zigTypeTag()) {
	1738	.Vector => return self.fail("TODO binary operations on vectors", .{}),
	1739	.Int => {
	1740	const int_info = lhs_ty.intInfo(self.target.*);
	1741	if (int_info.bits <= 64) {
	1742	const result_reg = result.register;
	1743	try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits);
	1744	return result;
	1745	} else {
	1746	return self.fail("TODO binary operations on integers > u64/i64", .{});
	1747	}
	1748	},
	1749	else => unreachable,
	1750	},
	1751	else => unreachable,
	1752	}
	1753	},
	1754
	1755	.shl_exact => {
	1756	switch (lhs_ty.zigTypeTag()) {
	1757	.Vector => return self.fail("TODO binary operations on vectors", .{}),
	1758	.Int => {
	1759	const int_info = lhs_ty.intInfo(self.target.*);
	1760	if (int_info.bits <= 64) {
	1761	const rhs_immediate_ok = rhs == .immediate;
	1762
	1763	const mir_tag: Mir.Inst.Tag = switch (tag) {
	1764	.shl_exact => .sllx,
	1765	else => unreachable,
	1766	};
	1767
	1768	if (rhs_immediate_ok) {
	1769	return try self.binOpImmediate(mir_tag, maybe_inst, lhs, rhs, lhs_ty, false);
	1770	} else {
	1771	return try self.binOpRegister(mir_tag, maybe_inst, lhs, rhs, lhs_ty, rhs_ty);
	1772	}
	1773	} else {
	1774	return self.fail("TODO binary operations on int with bits > 64", .{});
	1775	}
	1776	},
	1777	else => unreachable,
	1778	}
	1779	},
	1780
	1781	else => return self.fail("TODO implement {} binOp for SPARCv9", .{tag}),
	1782	}
	1783	}
	1784
	1785	/// Don't call this function directly. Use binOp instead.
	1786	///
	1787	/// Calling this function signals an intention to generate a Mir
	1788	/// instruction of the form
	1789	///
	1790	/// op dest, lhs, #rhs_imm
	1791	///
	1792	/// Set lhs_and_rhs_swapped to true iff inst.bin_op.lhs corresponds to
	1793	/// rhs and vice versa. This parameter is only used when maybe_inst !=
	1794	/// null.
	1795	///
	1796	/// Asserts that generating an instruction of that form is possible.
	1797	fn binOpImmediate(
	1798	self: *Self,
	1799	mir_tag: Mir.Inst.Tag,
	1800	maybe_inst: ?Air.Inst.Index,
	1801	lhs: MCValue,
	1802	rhs: MCValue,
	1803	lhs_ty: Type,
	1804	lhs_and_rhs_swapped: bool,
	1805	) !MCValue {
	1806	const lhs_is_register = lhs == .register;
	1807
	1808	const lhs_lock: ?RegisterLock = if (lhs_is_register)
	1809	self.register_manager.lockReg(lhs.register)
	1810	else
	1811	null;
	1812	defer if (lhs_lock) \|reg\| self.register_manager.unlockReg(reg);
	1813
	1814	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
	1815
	1816	const lhs_reg = if (lhs_is_register) lhs.register else blk: {
	1817	const track_inst: ?Air.Inst.Index = if (maybe_inst) \|inst\| inst: {
	1818	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1819	break :inst Air.refToIndex(
	1820	if (lhs_and_rhs_swapped) bin_op.rhs else bin_op.lhs,
	1821	).?;
	1822	} else null;
	1823
	1824	const reg = try self.register_manager.allocReg(track_inst);
	1825
	1826	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
	1827
	1828	break :blk reg;
	1829	};
	1830	const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
	1831	defer if (new_lhs_lock) \|reg\| self.register_manager.unlockReg(reg);
	1832
	1833	const dest_reg = switch (mir_tag) {
	1834	else => if (maybe_inst) \|inst\| blk: {
	1835	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1836
	1837	if (lhs_is_register and self.reuseOperand(
	1838	inst,
	1839	if (lhs_and_rhs_swapped) bin_op.rhs else bin_op.lhs,
	1840	if (lhs_and_rhs_swapped) 1 else 0,
	1841	lhs,
	1842	)) {
	1843	break :blk lhs_reg;
	1844	} else {
	1845	break :blk try self.register_manager.allocReg(inst);
	1846	}
	1847	} else blk: {
	1848	break :blk try self.register_manager.allocReg(null);
	1849	},
	1850	};
	1851
	1852	if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
	1853
	1854	const mir_data: Mir.Inst.Data = switch (mir_tag) {
	1855	.add,
	1856	.mulx,
	1857	.subcc,
	1858	=> .{
	1859	.arithmetic_3op = .{
	1860	.is_imm = true,
	1861	.rd = dest_reg,
	1862	.rs1 = lhs_reg,
	1863	.rs2_or_imm = .{ .imm = @intCast(i13, rhs.immediate) },
	1864	},
	1865	},
	1866	.sllx => .{
	1867	.shift = .{
	1868	.is_imm = true,
	1869	.width = ShiftWidth.shift64,
	1870	.rd = dest_reg,
	1871	.rs1 = lhs_reg,
	1872	.rs2_or_imm = .{ .imm = @intCast(u6, rhs.immediate) },
	1873	},
	1874	},
	1875	else => unreachable,
	1876	};
	1877
	1878	_ = try self.addInst(.{
	1879	.tag = mir_tag,
	1880	.data = mir_data,
	1881	});
	1882
	1883	return MCValue{ .register = dest_reg };
	1884	}
	1885
	1886	/// Don't call this function directly. Use binOp instead.
	1887	///
	1888	/// Calling this function signals an intention to generate a Mir
	1889	/// instruction of the form
	1890	///
	1891	/// op dest, lhs, rhs
	1892	///
	1893	/// Asserts that generating an instruction of that form is possible.
	1894	fn binOpRegister(
	1895	self: *Self,
	1896	mir_tag: Mir.Inst.Tag,
	1897	maybe_inst: ?Air.Inst.Index,
	1898	lhs: MCValue,
	1899	rhs: MCValue,
	1900	lhs_ty: Type,
	1901	rhs_ty: Type,
	1902	) !MCValue {
	1903	const lhs_is_register = lhs == .register;
	1904	const rhs_is_register = rhs == .register;
	1905
	1906	const lhs_lock: ?RegisterLock = if (lhs_is_register)
	1907	self.register_manager.lockReg(lhs.register)
	1908	else
	1909	null;
	1910	defer if (lhs_lock) \|reg\| self.register_manager.unlockReg(reg);
	1911
	1912	const rhs_lock: ?RegisterLock = if (rhs_is_register)
	1913	self.register_manager.lockReg(rhs.register)
	1914	else
	1915	null;
	1916	defer if (rhs_lock) \|reg\| self.register_manager.unlockReg(reg);
	1917
	1918	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
	1919
	1920	const lhs_reg = if (lhs_is_register) lhs.register else blk: {
	1921	const track_inst: ?Air.Inst.Index = if (maybe_inst) \|inst\| inst: {
	1922	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1923	break :inst Air.refToIndex(bin_op.lhs).?;
	1924	} else null;
	1925
	1926	const reg = try self.register_manager.allocReg(track_inst);
	1927	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
	1928
	1929	break :blk reg;
	1930	};
	1931	const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
	1932	defer if (new_lhs_lock) \|reg\| self.register_manager.unlockReg(reg);
	1933
	1934	const rhs_reg = if (rhs_is_register) rhs.register else blk: {
	1935	const track_inst: ?Air.Inst.Index = if (maybe_inst) \|inst\| inst: {
	1936	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1937	break :inst Air.refToIndex(bin_op.rhs).?;
	1938	} else null;
	1939
	1940	const reg = try self.register_manager.allocReg(track_inst);
	1941	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
	1942
	1943	break :blk reg;
	1944	};
	1945	const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
	1946	defer if (new_rhs_lock) \|reg\| self.register_manager.unlockReg(reg);
	1947
	1948	const dest_reg = switch (mir_tag) {
	1949	else => if (maybe_inst) \|inst\| blk: {
	1950	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1951
	1952	if (lhs_is_register and self.reuseOperand(inst, bin_op.lhs, 0, lhs)) {
	1953	break :blk lhs_reg;
	1954	} else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {
	1955	break :blk rhs_reg;
	1956	} else {
	1957	break :blk try self.register_manager.allocReg(inst);
	1958	}
	1959	} else blk: {
	1960	break :blk try self.register_manager.allocReg(null);
	1961	},
	1962	};
	1963
	1964	if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
	1965	if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
	1966
	1967	const mir_data: Mir.Inst.Data = switch (mir_tag) {
	1968	.add,
	1969	.mulx,
	1970	.subcc,
	1971	=> .{
	1972	.arithmetic_3op = .{
	1973	.is_imm = false,
	1974	.rd = dest_reg,
	1975	.rs1 = lhs_reg,
	1976	.rs2_or_imm = .{ .rs2 = rhs_reg },
	1977	},
	1978	},
	1979	.sllx => .{
	1980	.shift = .{
	1981	.is_imm = false,
	1982	.width = ShiftWidth.shift64,
	1983	.rd = dest_reg,
	1984	.rs1 = lhs_reg,
	1985	.rs2_or_imm = .{ .rs2 = rhs_reg },
	1986	},
	1987	},
	1988	else => unreachable,
	1989	};
	1990
	1991	_ = try self.addInst(.{
	1992	.tag = mir_tag,
	1993	.data = mir_data,
	1994	});
	1995
	1996	return MCValue{ .register = dest_reg };
	1997	}
	1998
	1999	fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
	2000	const block_data = self.blocks.getPtr(block).?;
	2001
	2002	if (self.air.typeOf(operand).hasRuntimeBits()) {
	2003	const operand_mcv = try self.resolveInst(operand);
	2004	const block_mcv = block_data.mcv;
	2005	if (block_mcv == .none) {
	2006	block_data.mcv = switch (operand_mcv) {
	2007	.none, .dead, .unreach => unreachable,
	2008	.register, .stack_offset, .memory => operand_mcv,
	2009	.immediate => blk: {
	2010	const new_mcv = try self.allocRegOrMem(block, true);
	2011	try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv);
	2012	break :blk new_mcv;
	2013	},
	2014	else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}),
	2015	};
	2016	} else {
	2017	try self.setRegOrMem(self.air.typeOfIndex(block), block_mcv, operand_mcv);
	2018	}
	2019	}
	2020	return self.brVoid(block);
	2021	}
	2022
	2023	fn brVoid(self: *Self, block: Air.Inst.Index) !void {
	2024	const block_data = self.blocks.getPtr(block).?;
	2025
	2026	// Emit a jump with a relocation. It will be patched up after the block ends.
	2027	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
	2028
	2029	const br_index = try self.addInst(.{
	2030	.tag = .bpcc,
	2031	.data = .{
	2032	.branch_predict_int = .{
	2033	.ccr = .xcc,
	2034	.cond = .al,
	2035	.inst = undefined, // Will be filled by performReloc
	2036	},
	2037	},
	2038	});
	2039
	2040	// TODO Find a way to fill this delay slot
	2041	_ = try self.addInst(.{
	2042	.tag = .nop,
	2043	.data = .{ .nop = {} },
	2044	});
	2045
	2046	block_data.relocs.appendAssumeCapacity(br_index);
	2047	}
	2048
1129	2049	/// Copies a value to a register without tracking the register. The register is not considered
1130	2050	/// allocated. A second call to `copyToTmpRegister` may return the same register.
1131	2051	/// This can have a side effect of spilling instructions to the stack to free up a register.
...	...	@@ -1222,6 +2142,76 @@ fn genArgDbgInfo(self: *Self, inst: Air.Inst.Index, mcv: MCValue, arg_index: u32
1222	2142	}
1223	2143	}
1224	2144
	2145	// TODO replace this to call to extern memcpy
	2146	fn genInlineMemcpy(
	2147	self: *Self,
	2148	src: Register,
	2149	dst: Register,
	2150	len: Register,
	2151	tmp: Register,
	2152	) !void {
	2153	// Here we assume that len > 0.
	2154	// Also we do the copy from end -> start address to save a register.
	2155
	2156	// sub len, 1, len
	2157	_ = try self.addInst(.{
	2158	.tag = .sub,
	2159	.data = .{ .arithmetic_3op = .{
	2160	.is_imm = true,
	2161	.rs1 = len,
	2162	.rs2_or_imm = .{ .imm = 1 },
	2163	.rd = len,
	2164	} },
	2165	});
	2166
	2167	// loop:
	2168	// ldub [src + len], tmp
	2169	_ = try self.addInst(.{
	2170	.tag = .ldub,
	2171	.data = .{ .arithmetic_3op = .{
	2172	.is_imm = false,
	2173	.rs1 = src,
	2174	.rs2_or_imm = .{ .rs2 = len },
	2175	.rd = tmp,
	2176	} },
	2177	});
	2178
	2179	// stb tmp, [dst + len]
	2180	_ = try self.addInst(.{
	2181	.tag = .stb,
	2182	.data = .{ .arithmetic_3op = .{
	2183	.is_imm = false,
	2184	.rs1 = dst,
	2185	.rs2_or_imm = .{ .rs2 = len },
	2186	.rd = tmp,
	2187	} },
	2188	});
	2189
	2190	// brnz len, loop
	2191	_ = try self.addInst(.{
	2192	.tag = .bpr,
	2193	.data = .{ .branch_predict_reg = .{
	2194	.cond = .ne_zero,
	2195	.rs1 = len,
	2196	.inst = @intCast(u32, self.mir_instructions.len - 2),
	2197	} },
	2198	});
	2199
	2200	// Delay slot:
	2201	// sub len, 1, len
	2202	_ = try self.addInst(.{
	2203	.tag = .sub,
	2204	.data = .{ .arithmetic_3op = .{
	2205	.is_imm = true,
	2206	.rs1 = len,
	2207	.rs2_or_imm = .{ .imm = 1 },
	2208	.rd = len,
	2209	} },
	2210	});
	2211
	2212	// end:
	2213	}
	2214
1225	2215	fn genLoad(self: *Self, value_reg: Register, addr_reg: Register, comptime off_type: type, off: off_type, abi_size: u64) !void {
1226	2216	assert(off_type == Register or off_type == i13);
1227	2217
...	...	@@ -1259,6 +2249,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
1259	2249	switch (mcv) {
1260	2250	.dead => unreachable,
1261	2251	.unreach, .none => return, // Nothing to do.
	2252	.compare_flags_signed => return self.fail("TODO: genSetReg for compare_flags_signed", .{}),
	2253	.compare_flags_unsigned => return self.fail("TODO: genSetReg for compare_flags_unsigned", .{}),
1262	2254	.undef => {
1263	2255	if (!self.wantSafety())
1264	2256	return; // The already existing value will do just fine.
...	...	@@ -1426,6 +2418,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
1426	2418	else => return self.fail("TODO implement memset", .{}),
1427	2419	}
1428	2420	},
	2421	.compare_flags_unsigned,
	2422	.compare_flags_signed,
1429	2423	.immediate,
1430	2424	.ptr_stack_offset,
1431	2425	=> {
...	...	@@ -1438,7 +2432,47 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
1438	2432	return self.fail("TODO larger stack offsets", .{});
1439	2433	return self.genStore(reg, .sp, i13, simm13, abi_size);
1440	2434	},
1441		.memory, .stack_offset => return self.fail("TODO implement memcpy", .{}),
	2435	.memory, .stack_offset => {
	2436	switch (mcv) {
	2437	.stack_offset => \|off\| {
	2438	if (stack_offset == off)
	2439	return; // Copy stack variable to itself; nothing to do.
	2440	},
	2441	else => {},
	2442	}
	2443
	2444	if (abi_size <= 8) {
	2445	const reg = try self.copyToTmpRegister(ty, mcv);
	2446	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
	2447	} else {
	2448	var ptr_ty_payload: Type.Payload.ElemType = .{
	2449	.base = .{ .tag = .single_mut_pointer },
	2450	.data = ty,
	2451	};
	2452	const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
	2453
	2454	const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
	2455	const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
	2456	defer for (regs_locks) \|reg\| {
	2457	self.register_manager.unlockReg(reg);
	2458	};
	2459
	2460	const src_reg = regs[0];
	2461	const dst_reg = regs[1];
	2462	const len_reg = regs[2];
	2463	const tmp_reg = regs[3];
	2464
	2465	switch (mcv) {
	2466	.stack_offset => \|off\| try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off }),
	2467	.memory => \|addr\| try self.genSetReg(Type.usize, src_reg, .{ .immediate = addr }),
	2468	else => unreachable,
	2469	}
	2470
	2471	try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = stack_offset });
	2472	try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
	2473	try self.genInlineMemcpy(src_reg, dst_reg, len_reg, tmp_reg);
	2474	}
	2475	},
1442	2476	}
1443	2477	}
1444	2478
...	...	@@ -1504,6 +2538,34 @@ fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue {
1504	2538	return self.fail("TODO implement int genTypedValue of > 64 bits", .{});
1505	2539	}
1506	2540	},
	2541	.ErrorSet => {
	2542	const err_name = typed_value.val.castTag(.@"error").?.data.name;
	2543	const module = self.bin_file.options.module.?;
	2544	const global_error_set = module.global_error_set;
	2545	const error_index = global_error_set.get(err_name).?;
	2546	return MCValue{ .immediate = error_index };
	2547	},
	2548	.ErrorUnion => {
	2549	const error_type = typed_value.ty.errorUnionSet();
	2550	const payload_type = typed_value.ty.errorUnionPayload();
	2551
	2552	if (typed_value.val.castTag(.eu_payload)) \|pl\| {
	2553	if (!payload_type.hasRuntimeBits()) {
	2554	// We use the error type directly as the type.
	2555	return MCValue{ .immediate = 0 };
	2556	}
	2557
	2558	_ = pl;
	2559	return self.fail("TODO implement error union const of type '{}' (non-error)", .{typed_value.ty.fmtDebug()});
	2560	} else {
	2561	if (!payload_type.hasRuntimeBits()) {
	2562	// We use the error type directly as the type.
	2563	return self.genTypedValue(.{ .ty = error_type, .val = typed_value.val });
	2564	}
	2565
	2566	return self.fail("TODO implement error union const of type '{}' (error)", .{typed_value.ty.fmtDebug()});
	2567	}
	2568	},
1507	2569	.ComptimeInt => unreachable, // semantic analysis prevents this
1508	2570	.ComptimeFloat => unreachable, // semantic analysis prevents this
1509	2571	else => return self.fail("TODO implement const of type '{}'", .{typed_value.ty.fmtDebug()}),
...	...	@@ -1522,6 +2584,54 @@ fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue {
1522	2584	}
1523	2585	}
1524	2586
	2587	fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
	2588	const error_type = ty.errorUnionSet();
	2589	const payload_type = ty.errorUnionPayload();
	2590
	2591	if (!error_type.hasRuntimeBits()) {
	2592	return MCValue{ .immediate = 0 }; // always false
	2593	} else if (!payload_type.hasRuntimeBits()) {
	2594	if (error_type.abiSize(self.target.*) <= 8) {
	2595	const reg_mcv: MCValue = switch (operand) {
	2596	.register => operand,
	2597	else => .{ .register = try self.copyToTmpRegister(error_type, operand) },
	2598	};
	2599
	2600	_ = try self.addInst(.{
	2601	.tag = .subcc,
	2602	.data = .{ .arithmetic_3op = .{
	2603	.is_imm = true,
	2604	.rs1 = reg_mcv.register,
	2605	.rs2_or_imm = .{ .imm = 0 },
	2606	.rd = .g0,
	2607	} },
	2608	});
	2609
	2610	return MCValue{ .compare_flags_unsigned = .gt };
	2611	} else {
	2612	return self.fail("TODO isErr for errors with size > 8", .{});
	2613	}
	2614	} else {
	2615	return self.fail("TODO isErr for non-empty payloads", .{});
	2616	}
	2617	}
	2618
	2619	fn isNonErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
	2620	// Call isErr, then negate the result.
	2621	const is_err_result = try self.isErr(ty, operand);
	2622	switch (is_err_result) {
	2623	.compare_flags_unsigned => \|op\| {
	2624	assert(op == .gt);
	2625	return MCValue{ .compare_flags_unsigned = .lte };
	2626	},
	2627	.immediate => \|imm\| {
	2628	assert(imm == 0);
	2629	return MCValue{ .immediate = 1 };
	2630	},
	2631	else => unreachable,
	2632	}
	2633	}
	2634
1525	2635	fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb {
1526	2636	try self.ensureProcessDeathCapacity(operand_count + 1);
1527	2637	return BigTomb{
...	...	@@ -1533,6 +2643,88 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT
1533	2643	};
1534	2644	}
1535	2645
	2646	/// Send control flow to `inst`.
	2647	fn jump(self: *Self, inst: Mir.Inst.Index) !void {
	2648	_ = try self.addInst(.{
	2649	.tag = .bpcc,
	2650	.data = .{
	2651	.branch_predict_int = .{
	2652	.cond = .al,
	2653	.ccr = .xcc,
	2654	.inst = inst,
	2655	},
	2656	},
	2657	});
	2658
	2659	// TODO find out a way to fill this delay slot
	2660	_ = try self.addInst(.{
	2661	.tag = .nop,
	2662	.data = .{ .nop = {} },
	2663	});
	2664	}
	2665
	2666	fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
	2667	const elem_ty = ptr_ty.elemType();
	2668	const elem_size = elem_ty.abiSize(self.target.*);
	2669
	2670	switch (ptr) {
	2671	.none => unreachable,
	2672	.undef => unreachable,
	2673	.unreach => unreachable,
	2674	.dead => unreachable,
	2675	.compare_flags_unsigned,
	2676	.compare_flags_signed,
	2677	=> unreachable, // cannot hold an address
	2678	.immediate => \|imm\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
	2679	.ptr_stack_offset => \|off\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
	2680	.register => \|addr_reg\| {
	2681	const addr_reg_lock = self.register_manager.lockReg(addr_reg);
	2682	defer if (addr_reg_lock) \|reg\| self.register_manager.unlockReg(reg);
	2683
	2684	switch (dst_mcv) {
	2685	.dead => unreachable,
	2686	.undef => unreachable,
	2687	.compare_flags_signed, .compare_flags_unsigned => unreachable,
	2688	.register => \|dst_reg\| {
	2689	try self.genLoad(dst_reg, addr_reg, i13, 0, elem_size);
	2690	},
	2691	.stack_offset => \|off\| {
	2692	if (elem_size <= 8) {
	2693	const tmp_reg = try self.register_manager.allocReg(null);
	2694	const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	2695	defer self.register_manager.unlockReg(tmp_reg_lock);
	2696
	2697	try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
	2698	try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
	2699	} else {
	2700	const regs = try self.register_manager.allocRegs(3, .{ null, null, null });
	2701	const regs_locks = self.register_manager.lockRegsAssumeUnused(3, regs);
	2702	defer for (regs_locks) \|reg\| {
	2703	self.register_manager.unlockReg(reg);
	2704	};
	2705
	2706	const src_reg = addr_reg;
	2707	const dst_reg = regs[0];
	2708	const len_reg = regs[1];
	2709	const tmp_reg = regs[2];
	2710
	2711	try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = off });
	2712	try self.genSetReg(Type.usize, len_reg, .{ .immediate = elem_size });
	2713	try self.genInlineMemcpy(src_reg, dst_reg, len_reg, tmp_reg);
	2714	}
	2715	},
	2716	else => return self.fail("TODO load from register into {}", .{dst_mcv}),
	2717	}
	2718	},
	2719	.memory,
	2720	.stack_offset,
	2721	=> {
	2722	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
	2723	try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty);
	2724	},
	2725	}
	2726	}
	2727
1536	2728	fn lowerDeclRef(self: *Self, tv: TypedValue, decl_index: Module.Decl.Index) InnerError!MCValue {
1537	2729	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
1538	2730	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
...	...	@@ -1568,7 +2760,7 @@ fn parseRegName(name: []const u8) ?Register {
1568	2760	fn performReloc(self: *Self, inst: Mir.Inst.Index) !void {
1569	2761	const tag = self.mir_instructions.items(.tag)[inst];
1570	2762	switch (tag) {
1571		.bpcc => self.mir_instructions.items(.data)[inst].branch_predict.inst = @intCast(Mir.Inst.Index, self.mir_instructions.len),
	2763	.bpcc => self.mir_instructions.items(.data)[inst].branch_predict_int.inst = @intCast(Mir.Inst.Index, self.mir_instructions.len),
1572	2764	else => unreachable,
1573	2765	}
1574	2766	}
...	...	@@ -1585,6 +2777,9 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void {
1585	2777	.register => \|reg\| {
1586	2778	self.register_manager.freeReg(reg);
1587	2779	},
	2780	.compare_flags_signed, .compare_flags_unsigned => {
	2781	self.compare_flags_inst = null;
	2782	},
1588	2783	else => {}, // TODO process stack allocation death
1589	2784	}
1590	2785	}
...	...	@@ -1718,11 +2913,18 @@ fn ret(self: *Self, mcv: MCValue) !void {
1718	2913	const ret_ty = self.fn_type.fnReturnType();
1719	2914	try self.setRegOrMem(ret_ty, self.ret_mcv, mcv);
1720	2915
1721		// Just add space for an instruction, patch this later
	2916	// Just add space for a branch instruction, patch this later
1722	2917	const index = try self.addInst(.{
1723	2918	.tag = .nop,
1724	2919	.data = .{ .nop = {} },
1725	2920	});
	2921
	2922	// Reserve space for the delay slot too
	2923	// TODO find out a way to fill this
	2924	_ = try self.addInst(.{
	2925	.tag = .nop,
	2926	.data = .{ .nop = {} },
	2927	});
1726	2928	try self.exitlude_jump_relocs.append(self.gpa, index);
1727	2929	}
1728	2930
...	...	@@ -1770,6 +2972,29 @@ fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
1770	2972	}
1771	2973	}
1772	2974
	2975	/// Save the current instruction stored in the compare flags if
	2976	/// occupied
	2977	fn spillCompareFlagsIfOccupied(self: *Self) !void {
	2978	if (self.compare_flags_inst) \|inst_to_save\| {
	2979	const mcv = self.getResolvedInstValue(inst_to_save);
	2980	switch (mcv) {
	2981	.compare_flags_signed,
	2982	.compare_flags_unsigned,
	2983	=> {},
	2984	else => unreachable, // mcv doesn't occupy the compare flags
	2985	}
	2986
	2987	const new_mcv = try self.allocRegOrMem(inst_to_save, true);
	2988	try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
	2989	log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
	2990
	2991	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
	2992	try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
	2993
	2994	self.compare_flags_inst = null;
	2995	}
	2996	}
	2997
1773	2998	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
1774	2999	const stack_mcv = try self.allocRegOrMem(inst, false);
1775	3000	log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
...	...	@@ -1780,6 +3005,152 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
1780	3005	try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
1781	3006	}
1782	3007
	3008	fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {
	3009	const abi_size = value_ty.abiSize(self.target.*);
	3010
	3011	switch (ptr) {
	3012	.none => unreachable,
	3013	.undef => unreachable,
	3014	.unreach => unreachable,
	3015	.dead => unreachable,
	3016	.compare_flags_unsigned,
	3017	.compare_flags_signed,
	3018	=> unreachable, // cannot hold an address
	3019	.immediate => \|imm\| {
	3020	try self.setRegOrMem(value_ty, .{ .memory = imm }, value);
	3021	},
	3022	.ptr_stack_offset => \|off\| {
	3023	try self.genSetStack(value_ty, off, value);
	3024	},
	3025	.register => \|addr_reg\| {
	3026	const addr_reg_lock = self.register_manager.lockReg(addr_reg);
	3027	defer if (addr_reg_lock) \|reg\| self.register_manager.unlockReg(reg);
	3028
	3029	switch (value) {
	3030	.register => \|value_reg\| {
	3031	try self.genStore(value_reg, addr_reg, i13, 0, abi_size);
	3032	},
	3033	else => {
	3034	return self.fail("TODO implement copying of memory", .{});
	3035	},
	3036	}
	3037	},
	3038	.memory,
	3039	.stack_offset,
	3040	=> {
	3041	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
	3042	try self.store(.{ .register = addr_reg }, value, ptr_ty, value_ty);
	3043	},
	3044	}
	3045	}
	3046
	3047	fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
	3048	return if (self.liveness.isUnused(inst)) .dead else result: {
	3049	const mcv = try self.resolveInst(operand);
	3050	const ptr_ty = self.air.typeOf(operand);
	3051	const struct_ty = ptr_ty.childType();
	3052	const struct_field_offset = @intCast(u32, struct_ty.structFieldOffset(index, self.target.*));
	3053	switch (mcv) {
	3054	.ptr_stack_offset => \|off\| {
	3055	break :result MCValue{ .ptr_stack_offset = off - struct_field_offset };
	3056	},
	3057	else => {
	3058	const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
	3059	.immediate = struct_field_offset,
	3060	});
	3061	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
	3062	defer self.register_manager.unlockReg(offset_reg_lock);
	3063
	3064	const addr_reg = try self.copyToTmpRegister(ptr_ty, mcv);
	3065	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	3066	defer self.register_manager.unlockReg(addr_reg_lock);
	3067
	3068	const dest = try self.binOp(
	3069	.add,
	3070	null,
	3071	.{ .register = addr_reg },
	3072	.{ .register = offset_reg },
	3073	Type.usize,
	3074	Type.usize,
	3075	);
	3076
	3077	break :result dest;
	3078	},
	3079	}
	3080	};
	3081	}
	3082
	3083	fn truncRegister(
	3084	self: *Self,
	3085	operand_reg: Register,
	3086	dest_reg: Register,
	3087	int_signedness: std.builtin.Signedness,
	3088	int_bits: u16,
	3089	) !void {
	3090	switch (int_bits) {
	3091	1...31, 33...63 => {
	3092	_ = try self.addInst(.{
	3093	.tag = .sllx,
	3094	.data = .{
	3095	.shift = .{
	3096	.is_imm = true,
	3097	.width = ShiftWidth.shift64,
	3098	.rd = dest_reg,
	3099	.rs1 = operand_reg,
	3100	.rs2_or_imm = .{ .imm = @intCast(u6, 64 - int_bits) },
	3101	},
	3102	},
	3103	});
	3104	_ = try self.addInst(.{
	3105	.tag = switch (int_signedness) {
	3106	.signed => .srax,
	3107	.unsigned => .srlx,
	3108	},
	3109	.data = .{
	3110	.shift = .{
	3111	.is_imm = true,
	3112	.width = ShiftWidth.shift32,
	3113	.rd = dest_reg,
	3114	.rs1 = dest_reg,
	3115	.rs2_or_imm = .{ .imm = @intCast(u6, int_bits) },
	3116	},
	3117	},
	3118	});
	3119	},
	3120	32 => {
	3121	_ = try self.addInst(.{
	3122	.tag = switch (int_signedness) {
	3123	.signed => .sra,
	3124	.unsigned => .srl,
	3125	},
	3126	.data = .{
	3127	.shift = .{
	3128	.is_imm = true,
	3129	.width = ShiftWidth.shift32,
	3130	.rd = dest_reg,
	3131	.rs1 = operand_reg,
	3132	.rs2_or_imm = .{ .imm = 0 },
	3133	},
	3134	},
	3135	});
	3136	},
	3137	64 => {
	3138	_ = try self.addInst(.{
	3139	.tag = .@"or",
	3140	.data = .{
	3141	.arithmetic_3op = .{
	3142	.is_imm = true,
	3143	.rd = dest_reg,
	3144	.rs1 = .g0,
	3145	.rs2_or_imm = .{ .rs2 = operand_reg },
	3146	},
	3147	},
	3148	});
	3149	},
	3150	else => unreachable,
	3151	}
	3152	}
	3153
1783	3154	/// TODO support scope overrides. Also note this logic is duplicated with `Module.wantSafety`.
1784	3155	fn wantSafety(self: *Self) bool {
1785	3156	return switch (self.bin_file.options.optimize_mode) {

src/arch/sparc64/Emit.zig+263-1

...	...	@@ -8,6 +8,7 @@ const link = @import("../../link.zig");
8	8	const Module = @import("../../Module.zig");
9	9	const ErrorMsg = Module.ErrorMsg;
10	10	const Liveness = @import("../../Liveness.zig");
	11	const log = std.log.scoped(.sparcv9_emit);
11	12	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
12	13	const DW = std.dwarf;
13	14	const leb128 = std.leb;
...	...	@@ -31,16 +32,44 @@ prev_di_column: u32,
31	32	/// Relative to the beginning of `code`.
32	33	prev_di_pc: usize,
33	34
	35	/// The branch type of every branch
	36	branch_types: std.AutoHashMapUnmanaged(Mir.Inst.Index, BranchType) = .{},
	37	/// For every forward branch, maps the target instruction to a list of
	38	/// branches which branch to this target instruction
	39	branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUnmanaged(Mir.Inst.Index)) = .{},
	40	/// For backward branches: stores the code offset of the target
	41	/// instruction
	42	///
	43	/// For forward branches: stores the code offset of the branch
	44	/// instruction
	45	code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
	46
34	47	const InnerError = error{
35	48	OutOfMemory,
36	49	EmitFail,
37	50	};
38	51
	52	const BranchType = enum {
	53	bpcc,
	54	bpr,
	55	fn default(tag: Mir.Inst.Tag) BranchType {
	56	return switch (tag) {
	57	.bpcc => .bpcc,
	58	.bpr => .bpr,
	59	else => unreachable,
	60	};
	61	}
	62	};
	63
39	64	pub fn emitMir(
40	65	emit: *Emit,
41	66	) InnerError!void {
42	67	const mir_tags = emit.mir.instructions.items(.tag);
43	68
	69	// Convert absolute addresses into offsets and
	70	// find smallest lowerings for branch instructions
	71	try emit.lowerBranches();
	72
44	73	// Emit machine code
45	74	for (mir_tags) \|tag, index\| {
46	75	const inst = @intCast(u32, index);
...	...	@@ -51,7 +80,8 @@ pub fn emitMir(
51	80
52	81	.add => try emit.mirArithmetic3Op(inst),
53	82
54		.bpcc => @panic("TODO implement sparc64 bpcc"),
	83	.bpr => try emit.mirConditionalBranch(inst),
	84	.bpcc => try emit.mirConditionalBranch(inst),
55	85
56	86	.call => @panic("TODO implement sparc64 call"),
57	87
...	...	@@ -64,6 +94,8 @@ pub fn emitMir(
64	94
65	95	.@"or" => try emit.mirArithmetic3Op(inst),
66	96
	97	.mulx => try emit.mirArithmetic3Op(inst),
	98
67	99	.nop => try emit.mirNop(),
68	100
69	101	.@"return" => try emit.mirArithmetic2Op(inst),
...	...	@@ -73,7 +105,12 @@ pub fn emitMir(
73	105
74	106	.sethi => try emit.mirSethi(inst),
75	107
	108	.sll => @panic("TODO implement sparc64 sll"),
	109	.srl => @panic("TODO implement sparc64 srl"),
	110	.sra => @panic("TODO implement sparc64 sra"),
76	111	.sllx => @panic("TODO implement sparc64 sllx"),
	112	.srlx => @panic("TODO implement sparc64 srlx"),
	113	.srax => @panic("TODO implement sparc64 srax"),
77	114
78	115	.stb => try emit.mirArithmetic3Op(inst),
79	116	.sth => try emit.mirArithmetic3Op(inst),
...	...	@@ -81,6 +118,7 @@ pub fn emitMir(
81	118	.stx => try emit.mirArithmetic3Op(inst),
82	119
83	120	.sub => try emit.mirArithmetic3Op(inst),
	121	.subcc => try emit.mirArithmetic3Op(inst),
84	122
85	123	.tcc => try emit.mirTrap(inst),
86	124	}
...	...	@@ -88,6 +126,14 @@ pub fn emitMir(
88	126	}
89	127
90	128	pub fn deinit(emit: *Emit) void {
	129	var iter = emit.branch_forward_origins.valueIterator();
	130	while (iter.next()) \|origin_list\| {
	131	origin_list.deinit(emit.bin_file.allocator);
	132	}
	133
	134	emit.branch_types.deinit(emit.bin_file.allocator);
	135	emit.branch_forward_origins.deinit(emit.bin_file.allocator);
	136	emit.code_offset_mapping.deinit(emit.bin_file.allocator);
91	137	emit.* = undefined;
92	138	}
93	139
...	...	@@ -161,6 +207,7 @@ fn mirArithmetic3Op(emit: *Emit, inst: Mir.Inst.Index) !void {
161	207	.lduw => try emit.writeInstruction(Instruction.lduw(i13, rs1, imm, rd)),
162	208	.ldx => try emit.writeInstruction(Instruction.ldx(i13, rs1, imm, rd)),
163	209	.@"or" => try emit.writeInstruction(Instruction.@"or"(i13, rs1, imm, rd)),
	210	.mulx => try emit.writeInstruction(Instruction.mulx(i13, rs1, imm, rd)),
164	211	.save => try emit.writeInstruction(Instruction.save(i13, rs1, imm, rd)),
165	212	.restore => try emit.writeInstruction(Instruction.restore(i13, rs1, imm, rd)),
166	213	.stb => try emit.writeInstruction(Instruction.stb(i13, rs1, imm, rd)),
...	...	@@ -168,6 +215,7 @@ fn mirArithmetic3Op(emit: *Emit, inst: Mir.Inst.Index) !void {
168	215	.stw => try emit.writeInstruction(Instruction.stw(i13, rs1, imm, rd)),
169	216	.stx => try emit.writeInstruction(Instruction.stx(i13, rs1, imm, rd)),
170	217	.sub => try emit.writeInstruction(Instruction.sub(i13, rs1, imm, rd)),
	218	.subcc => try emit.writeInstruction(Instruction.subcc(i13, rs1, imm, rd)),
171	219	else => unreachable,
172	220	}
173	221	} else {
...	...	@@ -180,6 +228,7 @@ fn mirArithmetic3Op(emit: *Emit, inst: Mir.Inst.Index) !void {
180	228	.lduw => try emit.writeInstruction(Instruction.lduw(Register, rs1, rs2, rd)),
181	229	.ldx => try emit.writeInstruction(Instruction.ldx(Register, rs1, rs2, rd)),
182	230	.@"or" => try emit.writeInstruction(Instruction.@"or"(Register, rs1, rs2, rd)),
	231	.mulx => try emit.writeInstruction(Instruction.mulx(Register, rs1, rs2, rd)),
183	232	.save => try emit.writeInstruction(Instruction.save(Register, rs1, rs2, rd)),
184	233	.restore => try emit.writeInstruction(Instruction.restore(Register, rs1, rs2, rd)),
185	234	.stb => try emit.writeInstruction(Instruction.stb(Register, rs1, rs2, rd)),
...	...	@@ -187,11 +236,56 @@ fn mirArithmetic3Op(emit: *Emit, inst: Mir.Inst.Index) !void {
187	236	.stw => try emit.writeInstruction(Instruction.stw(Register, rs1, rs2, rd)),
188	237	.stx => try emit.writeInstruction(Instruction.stx(Register, rs1, rs2, rd)),
189	238	.sub => try emit.writeInstruction(Instruction.sub(Register, rs1, rs2, rd)),
	239	.subcc => try emit.writeInstruction(Instruction.subcc(Register, rs1, rs2, rd)),
190	240	else => unreachable,
191	241	}
192	242	}
193	243	}
194	244
	245	fn mirConditionalBranch(emit: *Emit, inst: Mir.Inst.Index) !void {
	246	const tag = emit.mir.instructions.items(.tag)[inst];
	247	const branch_type = emit.branch_types.get(inst).?;
	248
	249	switch (branch_type) {
	250	.bpcc => switch (tag) {
	251	.bpcc => {
	252	const branch_predict_int = emit.mir.instructions.items(.data)[inst].branch_predict_int;
	253	const offset = @intCast(i64, emit.code_offset_mapping.get(branch_predict_int.inst).?) - @intCast(i64, emit.code.items.len);
	254	log.debug("mirConditionalBranch: {} offset={}", .{ inst, offset });
	255
	256	try emit.writeInstruction(
	257	Instruction.bpcc(
	258	branch_predict_int.cond,
	259	branch_predict_int.annul,
	260	branch_predict_int.pt,
	261	branch_predict_int.ccr,
	262	@intCast(i21, offset),
	263	),
	264	);
	265	},
	266	else => unreachable,
	267	},
	268	.bpr => switch (tag) {
	269	.bpr => {
	270	const branch_predict_reg = emit.mir.instructions.items(.data)[inst].branch_predict_reg;
	271	const offset = @intCast(i64, emit.code_offset_mapping.get(branch_predict_reg.inst).?) - @intCast(i64, emit.code.items.len);
	272	log.debug("mirConditionalBranch: {} offset={}", .{ inst, offset });
	273
	274	try emit.writeInstruction(
	275	Instruction.bpr(
	276	branch_predict_reg.cond,
	277	branch_predict_reg.annul,
	278	branch_predict_reg.pt,
	279	branch_predict_reg.rs1,
	280	@intCast(i18, offset),
	281	),
	282	);
	283	},
	284	else => unreachable,
	285	},
	286	}
	287	}
	288
195	289	fn mirNop(emit: *Emit) !void {
196	290	try emit.writeInstruction(Instruction.nop());
197	291	}
...	...	@@ -232,6 +326,16 @@ fn mirTrap(emit: *Emit, inst: Mir.Inst.Index) !void {
232	326
233	327	// Common helper functions
234	328
	329	fn branchTarget(emit: *Emit, inst: Mir.Inst.Index) Mir.Inst.Index {
	330	const tag = emit.mir.instructions.items(.tag)[inst];
	331
	332	switch (tag) {
	333	.bpcc => return emit.mir.instructions.items(.data)[inst].branch_predict_int.inst,
	334	.bpr => return emit.mir.instructions.items(.data)[inst].branch_predict_reg.inst,
	335	else => unreachable,
	336	}
	337	}
	338
235	339	fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) !void {
236	340	const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
237	341	const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
...	...	@@ -264,6 +368,164 @@ fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {
264	368	return error.EmitFail;
265	369	}
266	370
	371	fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize {
	372	const tag = emit.mir.instructions.items(.tag)[inst];
	373
	374	switch (tag) {
	375	.dbg_line,
	376	.dbg_epilogue_begin,
	377	.dbg_prologue_end,
	378	=> return 0,
	379	// Currently Mir instructions always map to single machine instruction.
	380	else => return 4,
	381	}
	382	}
	383
	384	fn isBranch(tag: Mir.Inst.Tag) bool {
	385	return switch (tag) {
	386	.bpcc => true,
	387	.bpr => true,
	388	else => false,
	389	};
	390	}
	391
	392	fn lowerBranches(emit: *Emit) !void {
	393	const mir_tags = emit.mir.instructions.items(.tag);
	394	const allocator = emit.bin_file.allocator;
	395
	396	// First pass: Note down all branches and their target
	397	// instructions, i.e. populate branch_types,
	398	// branch_forward_origins, and code_offset_mapping
	399	//
	400	// TODO optimization opportunity: do this in codegen while
	401	// generating MIR
	402	for (mir_tags) \|tag, index\| {
	403	const inst = @intCast(u32, index);
	404	if (isBranch(tag)) {
	405	const target_inst = emit.branchTarget(inst);
	406
	407	// Remember this branch instruction
	408	try emit.branch_types.put(allocator, inst, BranchType.default(tag));
	409
	410	// Forward branches require some extra stuff: We only
	411	// know their offset once we arrive at the target
	412	// instruction. Therefore, we need to be able to
	413	// access the branch instruction when we visit the
	414	// target instruction in order to manipulate its type
	415	// etc.
	416	if (target_inst > inst) {
	417	// Remember the branch instruction index
	418	try emit.code_offset_mapping.put(allocator, inst, 0);
	419
	420	if (emit.branch_forward_origins.getPtr(target_inst)) \|origin_list\| {
	421	try origin_list.append(allocator, inst);
	422	} else {
	423	var origin_list: std.ArrayListUnmanaged(Mir.Inst.Index) = .{};
	424	try origin_list.append(allocator, inst);
	425	try emit.branch_forward_origins.put(allocator, target_inst, origin_list);
	426	}
	427	}
	428
	429	// Remember the target instruction index so that we
	430	// update the real code offset in all future passes
	431	//
	432	// putNoClobber may not be used as the put operation
	433	// may clobber the entry when multiple branches branch
	434	// to the same target instruction
	435	try emit.code_offset_mapping.put(allocator, target_inst, 0);
	436	}
	437	}
	438
	439	// Further passes: Until all branches are lowered, interate
	440	// through all instructions and calculate new offsets and
	441	// potentially new branch types
	442	var all_branches_lowered = false;
	443	while (!all_branches_lowered) {
	444	all_branches_lowered = true;
	445	var current_code_offset: usize = 0;
	446
	447	for (mir_tags) \|tag, index\| {
	448	const inst = @intCast(u32, index);
	449
	450	// If this instruction contained in the code offset
	451	// mapping (when it is a target of a branch or if it is a
	452	// forward branch), update the code offset
	453	if (emit.code_offset_mapping.getPtr(inst)) \|offset\| {
	454	offset.* = current_code_offset;
	455	}
	456
	457	// If this instruction is a backward branch, calculate the
	458	// offset, which may potentially update the branch type
	459	if (isBranch(tag)) {
	460	const target_inst = emit.branchTarget(inst);
	461	if (target_inst < inst) {
	462	const target_offset = emit.code_offset_mapping.get(target_inst).?;
	463	const offset = @intCast(i64, target_offset) - @intCast(i64, current_code_offset);
	464	const branch_type = emit.branch_types.getPtr(inst).?;
	465	const optimal_branch_type = try emit.optimalBranchType(tag, offset);
	466	if (branch_type.* != optimal_branch_type) {
	467	branch_type.* = optimal_branch_type;
	468	all_branches_lowered = false;
	469	}
	470
	471	log.debug("lowerBranches: branch {} has offset {}", .{ inst, offset });
	472	}
	473	}
	474
	475	// If this instruction is the target of one or more
	476	// forward branches, calculate the offset, which may
	477	// potentially update the branch type
	478	if (emit.branch_forward_origins.get(inst)) \|origin_list\| {
	479	for (origin_list.items) \|forward_branch_inst\| {
	480	const branch_tag = emit.mir.instructions.items(.tag)[forward_branch_inst];
	481	const forward_branch_inst_offset = emit.code_offset_mapping.get(forward_branch_inst).?;
	482	const offset = @intCast(i64, current_code_offset) - @intCast(i64, forward_branch_inst_offset);
	483	const branch_type = emit.branch_types.getPtr(forward_branch_inst).?;
	484	const optimal_branch_type = try emit.optimalBranchType(branch_tag, offset);
	485	if (branch_type.* != optimal_branch_type) {
	486	branch_type.* = optimal_branch_type;
	487	all_branches_lowered = false;
	488	}
	489
	490	log.debug("lowerBranches: branch {} has offset {}", .{ forward_branch_inst, offset });
	491	}
	492	}
	493
	494	// Increment code offset
	495	current_code_offset += emit.instructionSize(inst);
	496	}
	497	}
	498	}
	499
	500	fn optimalBranchType(emit: *Emit, tag: Mir.Inst.Tag, offset: i64) !BranchType {
	501	assert(offset & 0b11 == 0);
	502
	503	switch (tag) {
	504	// TODO use the following strategy to implement long branches:
	505	// - Negate the conditional and target of the original instruction;
	506	// - In the space immediately after the branch, load
	507	// the address of the original target, preferrably in
	508	// a PC-relative way, into %o7; and
	509	// - jmpl %o7 + %g0, %g0
	510
	511	.bpcc => {
	512	if (std.math.cast(i21, offset)) \|_\| {
	513	return BranchType.bpcc;
	514	} else \|_\| {
	515	return emit.fail("TODO support BPcc branches larger than +-1 MiB", .{});
	516	}
	517	},
	518	.bpr => {
	519	if (std.math.cast(i18, offset)) \|_\| {
	520	return BranchType.bpr;
	521	} else \|_\| {
	522	return emit.fail("TODO support BPr branches larger than +-128 KiB", .{});
	523	}
	524	},
	525	else => unreachable,
	526	}
	527	}
	528
267	529	fn writeInstruction(emit: *Emit, instruction: Instruction) !void {
268	530	// SPARCv9 instructions are always arranged in BE regardless of the
269	531	// endianness mode the CPU is running in (Section 3.1 of the ISA specification).

src/arch/sparc64/Mir.zig+35-7

...	...	@@ -43,8 +43,12 @@ pub const Inst = struct {
43	43	// TODO add other operations.
44	44	add,
45	45
	46	/// A.3 Branch on Integer Register with Prediction (BPr)
	47	/// This uses the branch_predict_reg field.
	48	bpr,
	49
46	50	/// A.7 Branch on Integer Condition Codes with Prediction (BPcc)
47		/// This uses the branch_predict field.
	51	/// This uses the branch_predict_int field.
48	52	bpcc,
49	53
50	54	/// A.8 Call and Link
...	...	@@ -70,6 +74,11 @@ pub const Inst = struct {
70	74	// TODO add other operations.
71	75	@"or",
72	76
	77	/// A.37 Multiply and Divide (64-bit)
	78	/// This uses the arithmetic_3op field.
	79	// TODO add other operations.
	80	mulx,
	81
73	82	/// A.40 No Operation
74	83	/// This uses the nop field.
75	84	nop,
...	...	@@ -89,8 +98,12 @@ pub const Inst = struct {
89	98
90	99	/// A.49 Shift
91	100	/// This uses the shift field.
92		// TODO add other operations.
	101	sll,
	102	srl,
	103	sra,
93	104	sllx,
	105	srlx,
	106	srax,
94	107
95	108	/// A.54 Store Integer
96	109	/// This uses the arithmetic_3op field.
...	...	@@ -106,10 +119,15 @@ pub const Inst = struct {
106	119	/// This uses the arithmetic_3op field.
107	120	// TODO add other operations.
108	121	sub,
	122	subcc,
109	123
110	124	/// A.61 Trap on Integer Condition Codes (Tcc)
111	125	/// This uses the trap field.
112	126	tcc,
	127
	128	// TODO add synthetic instructions
	129	// TODO add cmp synthetic instruction to avoid wasting a register when
	130	// comparing with subcc
113	131	};
114	132
115	133	/// The position of an MIR instruction within the `Mir` instructions array.
...	...	@@ -164,13 +182,23 @@ pub const Inst = struct {
164	182	link: Register = .o7,
165	183	},
166	184
167		/// Branch with prediction.
	185	/// Branch with prediction, checking the integer status code
168	186	/// Used by e.g. bpcc
169		branch_predict: struct {
	187	branch_predict_int: struct {
170	188	annul: bool = false,
171	189	pt: bool = true,
172	190	ccr: Instruction.CCR,
173		cond: Instruction.Condition,
	191	cond: Instruction.ICondition,
	192	inst: Index,
	193	},
	194
	195	/// Branch with prediction, comparing a register's content with zero
	196	/// Used by e.g. bpr
	197	branch_predict_reg: struct {
	198	annul: bool = false,
	199	pt: bool = true,
	200	cond: Instruction.RCondition,
	201	rs1: Register,
174	202	inst: Index,
175	203	},
176	204
...	...	@@ -191,7 +219,7 @@ pub const Inst = struct {
191	219	/// if is_imm true then it uses the imm field of rs2_or_imm,
192	220	/// otherwise it uses rs2 field.
193	221	///
194		/// Used by e.g. add, sub
	222	/// Used by e.g. sllx
195	223	shift: struct {
196	224	is_imm: bool,
197	225	width: Instruction.ShiftWidth,
...	...	@@ -210,7 +238,7 @@ pub const Inst = struct {
210	238	/// Used by e.g. tcc
211	239	trap: struct {
212	240	is_imm: bool = true,
213		cond: Instruction.Condition,
	241	cond: Instruction.ICondition,
214	242	ccr: Instruction.CCR = .icc,
215	243	rs1: Register = .g0,
216	244	rs2_or_imm: union {

src/arch/sparc64/bits.zig+203-17

...	...	@@ -512,10 +512,172 @@ pub const Instruction = union(enum) {
512	512	lookaside: bool = false,
513	513	};
514	514
515		// TODO: Need to define an enum for `cond` values
516		// This is kinda challenging since the cond values have different meanings
517		// depending on whether it's operating on integer or FP CCR.
518		pub const Condition = u4;
	515	// In SPARCv9, FP and integer comparison operations
	516	// are encoded differently.
	517
	518	pub const FCondition = enum(u4) {
	519	/// Branch Never
	520	nv,
	521	/// Branch on Not Equal
	522	ne,
	523	/// Branch on Less or Greater
	524	lg,
	525	/// Branch on Unordered or Less
	526	ul,
	527	/// Branch on Less
	528	lt,
	529	/// Branch on Unordered or Greater
	530	ug,
	531	/// Branch on Greater
	532	gt,
	533	/// Branch on Unordered
	534	un,
	535	/// Branch Always
	536	al,
	537	/// Branch on Equal
	538	eq,
	539	/// Branch on Unordered or Equal
	540	ue,
	541	/// Branch on Greater or Equal
	542	ge,
	543	/// Branch on Unordered or Greater or Equal
	544	uge,
	545	/// Branch on Less or Equal
	546	le,
	547	/// Branch on Unordered or Less or Equal
	548	ule,
	549	/// Branch on Ordered
	550	ord,
	551
	552	/// Converts a std.math.CompareOperator into a condition flag,
	553	/// i.e. returns the condition that is true iff the result of the
	554	/// comparison is true.
	555	pub fn fromCompareOperator(op: std.math.CompareOperator) FCondition {
	556	return switch (op) {
	557	.gte => .ge,
	558	.gt => .gt,
	559	.neq => .ne,
	560	.lt => .lt,
	561	.lte => .le,
	562	.eq => .eq,
	563	};
	564	}
	565
	566	/// Returns the condition which is true iff the given condition is
	567	/// false (if such a condition exists).
	568	pub fn negate(cond: FCondition) FCondition {
	569	return switch (cond) {
	570	.eq => .ne,
	571	.ne => .eq,
	572	.ge => .ul,
	573	.ul => .ge,
	574	.le => .ug,
	575	.ug => .le,
	576	.lt => .uge,
	577	.uge => .lt,
	578	.gt => .ule,
	579	.ule => .gt,
	580	.ue => .lg,
	581	.lg => .ue,
	582	.ord => .un,
	583	.un => .ord,
	584	.al => unreachable,
	585	.nv => unreachable,
	586	};
	587	}
	588	};
	589
	590	pub const ICondition = enum(u4) {
	591	/// Branch Never
	592	nv,
	593	/// Branch on Equal
	594	eq,
	595	/// Branch on Less or Equal
	596	le,
	597	/// Branch on Less
	598	lt,
	599	/// Branch on Less or Equal Unsigned
	600	leu,
	601	/// Branch on Carry Set (Less than, Unsigned)
	602	cs,
	603	/// Branch on Negative
	604	neg,
	605	/// Branch on Overflow Set
	606	vs,
	607	/// Branch Always
	608	al,
	609	/// Branch on Not Equal
	610	ne,
	611	/// Branch on Greater
	612	gt,
	613	/// Branch on Greater or Equal
	614	ge,
	615	/// Branch on Greater Unsigned
	616	gu,
	617	/// Branch on Carry Clear (Greater Than or Equal, Unsigned)
	618	cc,
	619	/// Branch on Positive
	620	pos,
	621	/// Branch on Overflow Clear
	622	vc,
	623
	624	/// Converts a std.math.CompareOperator into a condition flag,
	625	/// i.e. returns the condition that is true iff the result of the
	626	/// comparison is true. Assumes signed comparison.
	627	pub fn fromCompareOperatorSigned(op: std.math.CompareOperator) ICondition {
	628	return switch (op) {
	629	.gte => .ge,
	630	.gt => .gt,
	631	.neq => .ne,
	632	.lt => .lt,
	633	.lte => .le,
	634	.eq => .eq,
	635	};
	636	}
	637
	638	/// Converts a std.math.CompareOperator into a condition flag,
	639	/// i.e. returns the condition that is true iff the result of the
	640	/// comparison is true. Assumes unsigned comparison.
	641	pub fn fromCompareOperatorUnsigned(op: std.math.CompareOperator) ICondition {
	642	return switch (op) {
	643	.gte => .cc,
	644	.gt => .gu,
	645	.neq => .ne,
	646	.lt => .cs,
	647	.lte => .le,
	648	.eq => .eq,
	649	};
	650	}
	651
	652	/// Returns the condition which is true iff the given condition is
	653	/// false (if such a condition exists).
	654	pub fn negate(cond: ICondition) ICondition {
	655	return switch (cond) {
	656	.eq => .ne,
	657	.ne => .eq,
	658	.cs => .cc,
	659	.cc => .cs,
	660	.neg => .pos,
	661	.pos => .neg,
	662	.vs => .vc,
	663	.vc => .vs,
	664	.gu => .leu,
	665	.leu => .gu,
	666	.ge => .lt,
	667	.lt => .ge,
	668	.gt => .le,
	669	.le => .gt,
	670	.al => unreachable,
	671	.nv => unreachable,
	672	};
	673	}
	674	};
	675
	676	pub const Condition = packed union {
	677	fcond: FCondition,
	678	icond: ICondition,
	679	encoded: u4,
	680	};
519	681
520	682	pub fn toU32(self: Instruction) u32 {
521	683	// TODO: Remove this once packed structs work.
...	...	@@ -593,7 +755,7 @@ pub const Instruction = union(enum) {
593	755	return Instruction{
594	756	.format_2b = .{
595	757	.a = @boolToInt(annul),
596		.cond = cond,
	758	.cond = cond.encoded,
597	759	.op2 = op2,
598	760	.disp22 = udisp_truncated,
599	761	},
...	...	@@ -614,7 +776,7 @@ pub const Instruction = union(enum) {
614	776	return Instruction{
615	777	.format_2c = .{
616	778	.a = @boolToInt(annul),
617		.cond = cond,
	779	.cond = cond.encoded,
618	780	.op2 = op2,
619	781	.cc1 = ccr_cc1,
620	782	.cc0 = ccr_cc0,
...	...	@@ -895,7 +1057,7 @@ pub const Instruction = union(enum) {
895	1057	.rd = rd.enc(),
896	1058	.op3 = op3,
897	1059	.cc2 = ccr_cc2,
898		.cond = cond,
	1060	.cond = cond.encoded,
899	1061	.cc1 = ccr_cc1,
900	1062	.cc0 = ccr_cc0,
901	1063	.rs2 = rs2.enc(),
...	...	@@ -912,7 +1074,7 @@ pub const Instruction = union(enum) {
912	1074	.rd = rd.enc(),
913	1075	.op3 = op3,
914	1076	.cc2 = ccr_cc2,
915		.cond = cond,
	1077	.cond = cond.encoded,
916	1078	.cc1 = ccr_cc1,
917	1079	.cc0 = ccr_cc0,
918	1080	.simm11 = @bitCast(u11, imm),
...	...	@@ -960,7 +1122,7 @@ pub const Instruction = union(enum) {
960	1122	.format_4g = .{
961	1123	.rd = rd.enc(),
962	1124	.op3 = op3,
963		.cond = cond,
	1125	.cond = cond.encoded,
964	1126	.opf_cc = opf_cc,
965	1127	.opf_low = opf_low,
966	1128	.rs2 = rs2.enc(),
...	...	@@ -979,6 +1141,14 @@ pub const Instruction = union(enum) {
979	1141	};
980	1142	}
981	1143
	1144	pub fn bpcc(cond: ICondition, annul: bool, pt: bool, ccr: CCR, disp: i21) Instruction {
	1145	return format2c(0b001, .{ .icond = cond }, annul, pt, ccr, disp);
	1146	}
	1147
	1148	pub fn bpr(cond: RCondition, annul: bool, pt: bool, rs1: Register, disp: i18) Instruction {
	1149	return format2d(0b011, cond, annul, pt, rs1, disp);
	1150	}
	1151
982	1152	pub fn jmpl(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction {
983	1153	return switch (s2) {
984	1154	Register => format3a(0b10, 0b11_1000, rs1, rs2, rd),
...	...	@@ -1027,6 +1197,14 @@ pub const Instruction = union(enum) {
1027	1197	};
1028	1198	}
1029	1199
	1200	pub fn mulx(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction {
	1201	return switch (s2) {
	1202	Register => format3a(0b10, 0b00_1001, rs1, rs2, rd),
	1203	i13 => format3b(0b10, 0b00_1001, rs1, rs2, rd),
	1204	else => unreachable,
	1205	};
	1206	}
	1207
1030	1208	pub fn nop() Instruction {
1031	1209	return sethi(0, .g0);
1032	1210	}
...	...	@@ -1099,11 +1277,19 @@ pub const Instruction = union(enum) {
1099	1277	};
1100	1278	}
1101	1279
1102		pub fn trap(comptime s2: type, cond: Condition, ccr: CCR, rs1: Register, rs2: s2) Instruction {
	1280	pub fn subcc(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction {
	1281	return switch (s2) {
	1282	Register => format3a(0b10, 0b01_0100, rs1, rs2, rd),
	1283	i13 => format3b(0b10, 0b01_0100, rs1, rs2, rd),
	1284	else => unreachable,
	1285	};
	1286	}
	1287
	1288	pub fn trap(comptime s2: type, cond: ICondition, ccr: CCR, rs1: Register, rs2: s2) Instruction {
1103	1289	// Tcc instructions abuse the rd field to store the conditionals.
1104	1290	return switch (s2) {
1105		Register => format4a(0b11_1010, ccr, rs1, rs2, @intToEnum(Register, cond)),
1106		u7 => format4e(0b11_1010, ccr, rs1, @intToEnum(Register, cond), rs2),
	1291	Register => format4a(0b11_1010, ccr, rs1, rs2, @intToEnum(Register, @enumToInt(cond))),
	1292	u7 => format4e(0b11_1010, ccr, rs1, @intToEnum(Register, @enumToInt(cond)), rs2),
1107	1293	else => unreachable,
1108	1294	};
1109	1295	}
...	...	@@ -1128,11 +1314,11 @@ test "Serialize formats" {
1128	1314	.expected = 0b00_00000_100_0000000000000000000000,
1129	1315	},
1130	1316	.{
1131		.inst = Instruction.format2b(6, 3, true, -4),
	1317	.inst = Instruction.format2b(6, .{ .icond = .lt }, true, -4),
1132	1318	.expected = 0b00_1_0011_110_1111111111111111111111,
1133	1319	},
1134	1320	.{
1135		.inst = Instruction.format2c(3, 0, false, true, .xcc, 8),
	1321	.inst = Instruction.format2c(3, .{ .icond = .nv }, false, true, .xcc, 8),
1136	1322	.expected = 0b00_0_0000_011_1_0_1_0000000000000000010,
1137	1323	},
1138	1324	.{
...	...	@@ -1224,11 +1410,11 @@ test "Serialize formats" {
1224	1410	.expected = 0b10_10010_001000_00000_1_1_0_11111111111,
1225	1411	},
1226	1412	.{
1227		.inst = Instruction.format4c(8, 0, .xcc, .g0, .o1),
	1413	.inst = Instruction.format4c(8, .{ .icond = .nv }, .xcc, .g0, .o1),
1228	1414	.expected = 0b10_01001_001000_1_0000_0_1_0_000000_00000,
1229	1415	},
1230	1416	.{
1231		.inst = Instruction.format4d(8, 0, .xcc, 0, .l2),
	1417	.inst = Instruction.format4d(8, .{ .icond = .nv }, .xcc, 0, .l2),
1232	1418	.expected = 0b10_10010_001000_1_0000_1_1_0_00000000000,
1233	1419	},
1234	1420	.{
...	...	@@ -1240,7 +1426,7 @@ test "Serialize formats" {
1240	1426	.expected = 0b10_10010_001000_00000_0_001_00100_01001,
1241	1427	},
1242	1428	.{
1243		.inst = Instruction.format4g(8, 4, 2, 0, .o1, .l2),
	1429	.inst = Instruction.format4g(8, 4, 2, .{ .icond = .nv }, .o1, .l2),
1244	1430	.expected = 0b10_10010_001000_0_0000_010_000100_01001,
1245	1431	},
1246	1432	};