Commit aaa641feba8866ac38f2d06a8db2a24fa134e2f5 | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2022-01-18 11:58:22+01:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2022-01-18 11:58:22+01:00
log	aaa641feba8866ac38f2d06a8db2a24fa134e2f5
tree	3b528f866b44275ea4762d224bdfc6263c3fd273
parent	4938fb8f5ca9551b69036fdb644103ffc9ef49cf

stage2: add inline memset for x86_64 backend

* introduce new Mir tag `mov_mem_index_imm` which selects instruction
  of the form `OP ptr [reg + rax*1 + imm32], imm32` where the encoded
  flags select the appropriate ptr width for memory store operation
  (note that scale is fixed and set at 1)

4 files changed, 181 insertions(+), 9 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Mir.zig
    PrintMir.zig

src/arch/x86_64/CodeGen.zig+93-2

...	...	@@ -3148,7 +3148,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3148	3148	2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
3149	3149	4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
3150	3150	8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
3151		else => return self.fail("TODO implement memset", .{}),
	3151	else => return self.genInlineMemset(ty, stack_offset, .{ .immediate = 0xaa }),
3152	3152	}
3153	3153	},
3154	3154	.compare_flags_unsigned => \|op\| {
...	...	@@ -3398,6 +3398,97 @@ fn genInlineMemcpy(
3398	3398	try self.performReloc(loop_reloc);
3399	3399	}
3400	3400
	3401	fn genInlineMemset(self: *Self, ty: Type, stack_offset: u32, value: MCValue) InnerError!void {
	3402	try self.register_manager.getReg(.rax, null);
	3403	const abi_size = ty.abiSize(self.target.*);
	3404	const adj_off = stack_offset + abi_size;
	3405	if (adj_off > 128) {
	3406	return self.fail("TODO inline memset with large stack offset", .{});
	3407	}
	3408	const negative_offset = @bitCast(u32, -@intCast(i32, adj_off));
	3409
	3410	// We are actually counting `abi_size` bytes; however, we reuse the index register
	3411	// as both the counter and offset scaler, hence we need to subtract one from `abi_size`
	3412	// and count until -1.
	3413	if (abi_size > math.maxInt(i32)) {
	3414	// movabs rax, abi_size - 1
	3415	const payload = try self.addExtra(Mir.Imm64.encode(abi_size - 1));
	3416	_ = try self.addInst(.{
	3417	.tag = .movabs,
	3418	.ops = (Mir.Ops{
	3419	.reg1 = .rax,
	3420	}).encode(),
	3421	.data = .{ .payload = payload },
	3422	});
	3423	} else {
	3424	// mov rax, abi_size - 1
	3425	_ = try self.addInst(.{
	3426	.tag = .mov,
	3427	.ops = (Mir.Ops{
	3428	.reg1 = .rax,
	3429	}).encode(),
	3430	.data = .{ .imm = @truncate(u32, abi_size - 1) },
	3431	});
	3432	}
	3433
	3434	// loop:
	3435	// cmp rax, -1
	3436	const loop_start = try self.addInst(.{
	3437	.tag = .cmp,
	3438	.ops = (Mir.Ops{
	3439	.reg1 = .rax,
	3440	}).encode(),
	3441	.data = .{ .imm = @bitCast(u32, @as(i32, -1)) },
	3442	});
	3443
	3444	// je end
	3445	const loop_reloc = try self.addInst(.{
	3446	.tag = .cond_jmp_eq_ne,
	3447	.ops = (Mir.Ops{ .flags = 0b01 }).encode(),
	3448	.data = .{ .inst = undefined },
	3449	});
	3450
	3451	switch (value) {
	3452	.immediate => \|x\| {
	3453	if (x > math.maxInt(i32)) {
	3454	return self.fail("TODO inline memset for value immediate larger than 32bits", .{});
	3455	}
	3456	// mov byte ptr [rbp + rax + stack_offset], imm
	3457	const payload = try self.addExtra(Mir.ImmPair{
	3458	.dest_off = negative_offset,
	3459	.operand = @truncate(u32, x),
	3460	});
	3461	_ = try self.addInst(.{
	3462	.tag = .mov_mem_index_imm,
	3463	.ops = (Mir.Ops{
	3464	.reg1 = .rbp,
	3465	}).encode(),
	3466	.data = .{ .payload = payload },
	3467	});
	3468	},
	3469	else => return self.fail("TODO inline memset for value of type {}", .{value}),
	3470	}
	3471
	3472	// sub rax, 1
	3473	_ = try self.addInst(.{
	3474	.tag = .sub,
	3475	.ops = (Mir.Ops{
	3476	.reg1 = .rax,
	3477	}).encode(),
	3478	.data = .{ .imm = 1 },
	3479	});
	3480
	3481	// jmp loop
	3482	_ = try self.addInst(.{
	3483	.tag = .jmp,
	3484	.ops = (Mir.Ops{ .flags = 0b00 }).encode(),
	3485	.data = .{ .inst = loop_start },
	3486	});
	3487
	3488	// end:
	3489	try self.performReloc(loop_reloc);
	3490	}
	3491
3401	3492	fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
3402	3493	switch (mcv) {
3403	3494	.dead => unreachable,
...	...	@@ -3639,7 +3730,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
3639	3730	const stack_offset = try self.allocMem(inst, 16, 16);
3640	3731	const array_ty = ptr_ty.childType();
3641	3732	const array_len = array_ty.arrayLenIncludingSentinel();
3642		try self.genSetStack(Type.initTag(.usize), stack_offset + 8, ptr);
	3733	try self.genSetStack(ptr_ty, stack_offset + 8, ptr);
3643	3734	try self.genSetStack(Type.initTag(.u64), stack_offset + 16, .{ .immediate = array_len });
3644	3735	break :blk .{ .stack_offset = stack_offset };
3645	3736	};

src/arch/x86_64/Emit.zig+33

...	...	@@ -115,6 +115,16 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
115	115	.cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst),
116	116	.mov_scale_imm => try emit.mirArithScaleImm(.mov, inst),
117	117
	118	.adc_mem_index_imm => try emit.mirArithMemIndexImm(.adc, inst),
	119	.add_mem_index_imm => try emit.mirArithMemIndexImm(.add, inst),
	120	.sub_mem_index_imm => try emit.mirArithMemIndexImm(.sub, inst),
	121	.xor_mem_index_imm => try emit.mirArithMemIndexImm(.xor, inst),
	122	.and_mem_index_imm => try emit.mirArithMemIndexImm(.@"and", inst),
	123	.or_mem_index_imm => try emit.mirArithMemIndexImm(.@"or", inst),
	124	.sbb_mem_index_imm => try emit.mirArithMemIndexImm(.sbb, inst),
	125	.cmp_mem_index_imm => try emit.mirArithMemIndexImm(.cmp, inst),
	126	.mov_mem_index_imm => try emit.mirArithMemIndexImm(.mov, inst),
	127
118	128	.movabs => try emit.mirMovabs(inst),
119	129
120	130	.lea => try emit.mirLea(inst),
...	...	@@ -549,6 +559,29 @@ fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void
549	559	}), imm_pair.operand, emit.code) catch \|err\| emit.failWithLoweringError(err);
550	560	}
551	561
	562	fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	563	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	564	assert(ops.reg2 == .none);
	565	const payload = emit.mir.instructions.items(.data)[inst].payload;
	566	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
	567	const ptr_size: Memory.PtrSize = switch (ops.flags) {
	568	0b00 => .byte_ptr,
	569	0b01 => .word_ptr,
	570	0b10 => .dword_ptr,
	571	0b11 => .qword_ptr,
	572	};
	573	const scale_index = ScaleIndex{
	574	.scale = 0,
	575	.index = .rax,
	576	};
	577	// OP ptr [reg1 + rax*1 + imm32], imm32
	578	return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
	579	.disp = imm_pair.dest_off,
	580	.base = ops.reg1,
	581	.scale_index = scale_index,
	582	}), imm_pair.operand, emit.code) catch \|err\| emit.failWithLoweringError(err);
	583	}
	584
552	585	fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
553	586	const tag = emit.mir.instructions.items(.tag)[inst];
554	587	assert(tag == .movabs);

src/arch/x86_64/Mir.zig+23

...	...	@@ -79,6 +79,13 @@ pub const Inst = struct {
79	79	/// * Data field `payload` points at `ImmPair`.
80	80	adc_scale_imm,
81	81
	82	/// ops flags: form:
	83	/// 0b00 byte ptr [reg1 + rax + imm32], imm8
	84	/// 0b01 word ptr [reg1 + rax + imm32], imm16
	85	/// 0b10 dword ptr [reg1 + rax + imm32], imm32
	86	/// 0b11 qword ptr [reg1 + rax + imm32], imm32 (sign-extended to imm64)
	87	adc_mem_index_imm,
	88
82	89	// The following instructions all have the same encoding as `adc`.
83	90
84	91	add,
...	...	@@ -86,81 +93,97 @@ pub const Inst = struct {
86	93	add_scale_src,
87	94	add_scale_dst,
88	95	add_scale_imm,
	96	add_mem_index_imm,
89	97	sub,
90	98	sub_mem_imm,
91	99	sub_scale_src,
92	100	sub_scale_dst,
93	101	sub_scale_imm,
	102	sub_mem_index_imm,
94	103	xor,
95	104	xor_mem_imm,
96	105	xor_scale_src,
97	106	xor_scale_dst,
98	107	xor_scale_imm,
	108	xor_mem_index_imm,
99	109	@"and",
100	110	and_mem_imm,
101	111	and_scale_src,
102	112	and_scale_dst,
103	113	and_scale_imm,
	114	and_mem_index_imm,
104	115	@"or",
105	116	or_mem_imm,
106	117	or_scale_src,
107	118	or_scale_dst,
108	119	or_scale_imm,
	120	or_mem_index_imm,
109	121	rol,
110	122	rol_mem_imm,
111	123	rol_scale_src,
112	124	rol_scale_dst,
113	125	rol_scale_imm,
	126	rol_mem_index_imm,
114	127	ror,
115	128	ror_mem_imm,
116	129	ror_scale_src,
117	130	ror_scale_dst,
118	131	ror_scale_imm,
	132	ror_mem_index_imm,
119	133	rcl,
120	134	rcl_mem_imm,
121	135	rcl_scale_src,
122	136	rcl_scale_dst,
123	137	rcl_scale_imm,
	138	rcl_mem_index_imm,
124	139	rcr,
125	140	rcr_mem_imm,
126	141	rcr_scale_src,
127	142	rcr_scale_dst,
128	143	rcr_scale_imm,
	144	rcr_mem_index_imm,
129	145	shl,
130	146	shl_mem_imm,
131	147	shl_scale_src,
132	148	shl_scale_dst,
133	149	shl_scale_imm,
	150	shl_mem_index_imm,
134	151	sal,
135	152	sal_mem_imm,
136	153	sal_scale_src,
137	154	sal_scale_dst,
138	155	sal_scale_imm,
	156	sal_mem_index_imm,
139	157	shr,
140	158	shr_mem_imm,
141	159	shr_scale_src,
142	160	shr_scale_dst,
143	161	shr_scale_imm,
	162	shr_mem_index_imm,
144	163	sar,
145	164	sar_mem_imm,
146	165	sar_scale_src,
147	166	sar_scale_dst,
148	167	sar_scale_imm,
	168	sar_mem_index_imm,
149	169	sbb,
150	170	sbb_mem_imm,
151	171	sbb_scale_src,
152	172	sbb_scale_dst,
153	173	sbb_scale_imm,
	174	sbb_mem_index_imm,
154	175	cmp,
155	176	cmp_mem_imm,
156	177	cmp_scale_src,
157	178	cmp_scale_dst,
158	179	cmp_scale_imm,
	180	cmp_mem_index_imm,
159	181	mov,
160	182	mov_mem_imm,
161	183	mov_scale_src,
162	184	mov_scale_dst,
163	185	mov_scale_imm,
	186	mov_mem_index_imm,
164	187
165	188	/// ops flags: form:
166	189	/// 0b00 reg1, [reg2 + imm32]

src/arch/x86_64/PrintMir.zig+32-7

...	...	@@ -64,6 +64,7 @@ pub fn printMir(print: *const Print, w: anytype, mir_to_air_map: std.AutoHashMap
64	64	.@"or" => try print.mirArith(.@"or", inst, w),
65	65	.sbb => try print.mirArith(.sbb, inst, w),
66	66	.cmp => try print.mirArith(.cmp, inst, w),
	67	.mov => try print.mirArith(.mov, inst, w),
67	68
68	69	.adc_mem_imm => try print.mirArithMemImm(.adc, inst, w),
69	70	.add_mem_imm => try print.mirArithMemImm(.add, inst, w),
...	...	@@ -73,6 +74,7 @@ pub fn printMir(print: *const Print, w: anytype, mir_to_air_map: std.AutoHashMap
73	74	.or_mem_imm => try print.mirArithMemImm(.@"or", inst, w),
74	75	.sbb_mem_imm => try print.mirArithMemImm(.sbb, inst, w),
75	76	.cmp_mem_imm => try print.mirArithMemImm(.cmp, inst, w),
	77	.mov_mem_imm => try print.mirArithMemImm(.mov, inst, w),
76	78
77	79	.adc_scale_src => try print.mirArithScaleSrc(.adc, inst, w),
78	80	.add_scale_src => try print.mirArithScaleSrc(.add, inst, w),
...	...	@@ -82,6 +84,7 @@ pub fn printMir(print: *const Print, w: anytype, mir_to_air_map: std.AutoHashMap
82	84	.or_scale_src => try print.mirArithScaleSrc(.@"or", inst, w),
83	85	.sbb_scale_src => try print.mirArithScaleSrc(.sbb, inst, w),
84	86	.cmp_scale_src => try print.mirArithScaleSrc(.cmp, inst, w),
	87	.mov_scale_src => try print.mirArithScaleSrc(.mov, inst, w),
85	88
86	89	.adc_scale_dst => try print.mirArithScaleDst(.adc, inst, w),
87	90	.add_scale_dst => try print.mirArithScaleDst(.add, inst, w),
...	...	@@ -91,6 +94,7 @@ pub fn printMir(print: *const Print, w: anytype, mir_to_air_map: std.AutoHashMap
91	94	.or_scale_dst => try print.mirArithScaleDst(.@"or", inst, w),
92	95	.sbb_scale_dst => try print.mirArithScaleDst(.sbb, inst, w),
93	96	.cmp_scale_dst => try print.mirArithScaleDst(.cmp, inst, w),
	97	.mov_scale_dst => try print.mirArithScaleDst(.mov, inst, w),
94	98
95	99	.adc_scale_imm => try print.mirArithScaleImm(.adc, inst, w),
96	100	.add_scale_imm => try print.mirArithScaleImm(.add, inst, w),
...	...	@@ -100,11 +104,18 @@ pub fn printMir(print: *const Print, w: anytype, mir_to_air_map: std.AutoHashMap
100	104	.or_scale_imm => try print.mirArithScaleImm(.@"or", inst, w),
101	105	.sbb_scale_imm => try print.mirArithScaleImm(.sbb, inst, w),
102	106	.cmp_scale_imm => try print.mirArithScaleImm(.cmp, inst, w),
103
104		.mov => try print.mirArith(.mov, inst, w),
105		.mov_scale_src => try print.mirArithScaleSrc(.mov, inst, w),
106		.mov_scale_dst => try print.mirArithScaleDst(.mov, inst, w),
107	107	.mov_scale_imm => try print.mirArithScaleImm(.mov, inst, w),
	108
	109	.adc_mem_index_imm => try print.mirArithMemIndexImm(.adc, inst, w),
	110	.add_mem_index_imm => try print.mirArithMemIndexImm(.add, inst, w),
	111	.sub_mem_index_imm => try print.mirArithMemIndexImm(.sub, inst, w),
	112	.xor_mem_index_imm => try print.mirArithMemIndexImm(.xor, inst, w),
	113	.and_mem_index_imm => try print.mirArithMemIndexImm(.@"and", inst, w),
	114	.or_mem_index_imm => try print.mirArithMemIndexImm(.@"or", inst, w),
	115	.sbb_mem_index_imm => try print.mirArithMemIndexImm(.sbb, inst, w),
	116	.cmp_mem_index_imm => try print.mirArithMemIndexImm(.cmp, inst, w),
	117	.mov_mem_index_imm => try print.mirArithMemIndexImm(.mov, inst, w),
	118
108	119	.movabs => try print.mirMovabs(inst, w),
109	120
110	121	.lea => try print.mirLea(inst, w),
...	...	@@ -316,11 +327,11 @@ fn mirArithScaleDst(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index
316	327
317	328	if (ops.reg2 == .none) {
318	329	// OP [reg1 + scale*rax + 0], imm32
319		try w.print("{s} [{s} + {d}*rcx + 0], {d}\n", .{ @tagName(tag), @tagName(ops.reg1), scale, imm });
	330	try w.print("{s} [{s} + {d}*rax + 0], {d}\n", .{ @tagName(tag), @tagName(ops.reg1), scale, imm });
320	331	}
321	332
322	333	// OP [reg1 + scale*rax + imm32], reg2
323		try w.print("{s} [{s} + {d}*rcx + {d}], {s}\n", .{ @tagName(tag), @tagName(ops.reg1), scale, imm, @tagName(ops.reg2) });
	334	try w.print("{s} [{s} + {d}*rax + {d}], {s}\n", .{ @tagName(tag), @tagName(ops.reg1), scale, imm, @tagName(ops.reg2) });
324	335	}
325	336
326	337	fn mirArithScaleImm(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: anytype) !void {
...	...	@@ -328,7 +339,21 @@ fn mirArithScaleImm(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index
328	339	const scale = ops.flags;
329	340	const payload = print.mir.instructions.items(.data)[inst].payload;
330	341	const imm_pair = print.mir.extraData(Mir.ImmPair, payload).data;
331		try w.print("{s} [{s} + {d}*rcx + {d}], {d}\n", .{ @tagName(tag), @tagName(ops.reg1), scale, imm_pair.dest_off, imm_pair.operand });
	342	try w.print("{s} [{s} + {d}*rax + {d}], {d}\n", .{ @tagName(tag), @tagName(ops.reg1), scale, imm_pair.dest_off, imm_pair.operand });
	343	}
	344
	345	fn mirArithMemIndexImm(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: anytype) !void {
	346	const ops = Mir.Ops.decode(print.mir.instructions.items(.ops)[inst]);
	347	const payload = print.mir.instructions.items(.data)[inst].payload;
	348	const imm_pair = print.mir.extraData(Mir.ImmPair, payload).data;
	349	try w.print("{s} ", .{@tagName(tag)});
	350	switch (ops.flags) {
	351	0b00 => try w.print("byte ptr ", .{}),
	352	0b01 => try w.print("word ptr ", .{}),
	353	0b10 => try w.print("dword ptr ", .{}),
	354	0b11 => try w.print("qword ptr ", .{}),
	355	}
	356	try w.print("[{s} + 1*rax + {d}], {d}\n", .{ @tagName(ops.reg1), imm_pair.dest_off, imm_pair.operand });
332	357	}
333	358
334	359	fn mirMovabs(print: *const Print, inst: Mir.Inst.Index, w: anytype) !void {