authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-05-31 00:19:23-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-05-31 14:17:54-04:00
logd410693dadfe791e616e78239fa0cec707b95cfa
treecc797ccc0b13a5b65d676a7aac78c88f3c42fa38
parent26aea8cfa1ea98329e18458cb23be2d60ef52507

LLVM: elide some loads when lowering

Generally, the load instruction may need to make a copy of an isByRef=true value, such as in the case of the following code: ```zig pub fn swap(comptime T: type, a: *T, b: *T) void { const tmp = a.*; a.* = b.*; b.* = tmp; } ``` However, it only needs to do so if there are any instructions which can possibly write to memory. When calling functions with isByRef=true parameters, the AIR code that is generated looks like loads followed directly by call. This allows for a peephole optimization when lowering loads: if the load instruction operates on an isByRef=true type and dies before any side effects occur, then we can safely lower the load as a no-op that returns its operand. This is one out of three changes I intend to make to address #11498. However I will put these changes in separate branches and merge them separately so that we can have three independent points on the perf charts.

2 files changed, 423 insertions(+), 5 deletions(-)

src/Liveness.zig+396
......@@ -112,6 +112,402 @@ pub fn clearOperandDeath(l: Liveness, inst: Air.Inst.Index, operand: OperandInt)
112112 l.tomb_bits[usize_index] &= ~mask;
113113}
114114
115const OperandCategory = enum {
116 /// The operand lives on, but this instruction cannot possibly mutate memory.
117 none,
118 /// The operand lives on and this instruction can mutate memory.
119 write,
120 /// The operand dies at this instruction.
121 tomb,
122 /// The operand lives on, and this instruction is noreturn.
123 noret,
124 /// This instruction is too complicated for analysis, no information is available.
125 complex,
126};
127
128/// Given an instruction that we are examining, and an operand that we are looking for,
129/// returns a classification.
130pub fn categorizeOperand(
131 l: Liveness,
132 air: Air,
133 inst: Air.Inst.Index,
134 operand: Air.Inst.Index,
135) OperandCategory {
136 const air_tags = air.instructions.items(.tag);
137 const air_datas = air.instructions.items(.data);
138 const operand_ref = Air.indexToRef(operand);
139 switch (air_tags[inst]) {
140 .add,
141 .addwrap,
142 .add_sat,
143 .sub,
144 .subwrap,
145 .sub_sat,
146 .mul,
147 .mulwrap,
148 .mul_sat,
149 .div_float,
150 .div_trunc,
151 .div_floor,
152 .div_exact,
153 .rem,
154 .mod,
155 .bit_and,
156 .bit_or,
157 .xor,
158 .cmp_lt,
159 .cmp_lte,
160 .cmp_eq,
161 .cmp_gte,
162 .cmp_gt,
163 .cmp_neq,
164 .bool_and,
165 .bool_or,
166 .array_elem_val,
167 .slice_elem_val,
168 .ptr_elem_val,
169 .shl,
170 .shl_exact,
171 .shl_sat,
172 .shr,
173 .shr_exact,
174 .min,
175 .max,
176 => {
177 const o = air_datas[inst].bin_op;
178 if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
179 if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
180 return .none;
181 },
182
183 .store,
184 .atomic_store_unordered,
185 .atomic_store_monotonic,
186 .atomic_store_release,
187 .atomic_store_seq_cst,
188 .set_union_tag,
189 => {
190 const o = air_datas[inst].bin_op;
191 if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
192 if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
193 return .write;
194 },
195
196 .arg,
197 .alloc,
198 .ret_ptr,
199 .constant,
200 .const_ty,
201 .breakpoint,
202 .dbg_stmt,
203 .dbg_inline_begin,
204 .dbg_inline_end,
205 .dbg_block_begin,
206 .dbg_block_end,
207 .unreach,
208 .ret_addr,
209 .frame_addr,
210 .wasm_memory_size,
211 .err_return_trace,
212 => return .none,
213
214 .fence => return .write,
215
216 .not,
217 .bitcast,
218 .load,
219 .fpext,
220 .fptrunc,
221 .intcast,
222 .trunc,
223 .optional_payload,
224 .optional_payload_ptr,
225 .wrap_optional,
226 .unwrap_errunion_payload,
227 .unwrap_errunion_err,
228 .unwrap_errunion_payload_ptr,
229 .unwrap_errunion_err_ptr,
230 .wrap_errunion_payload,
231 .wrap_errunion_err,
232 .slice_ptr,
233 .slice_len,
234 .ptr_slice_len_ptr,
235 .ptr_slice_ptr_ptr,
236 .struct_field_ptr_index_0,
237 .struct_field_ptr_index_1,
238 .struct_field_ptr_index_2,
239 .struct_field_ptr_index_3,
240 .array_to_slice,
241 .float_to_int,
242 .int_to_float,
243 .get_union_tag,
244 .clz,
245 .ctz,
246 .popcount,
247 .byte_swap,
248 .bit_reverse,
249 .splat,
250 => {
251 const o = air_datas[inst].ty_op;
252 if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
253 return .none;
254 },
255
256 .optional_payload_ptr_set,
257 .errunion_payload_ptr_set,
258 => {
259 const o = air_datas[inst].ty_op;
260 if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
261 return .write;
262 },
263
264 .is_null,
265 .is_non_null,
266 .is_null_ptr,
267 .is_non_null_ptr,
268 .is_err,
269 .is_non_err,
270 .is_err_ptr,
271 .is_non_err_ptr,
272 .ptrtoint,
273 .bool_to_int,
274 .tag_name,
275 .error_name,
276 .sqrt,
277 .sin,
278 .cos,
279 .tan,
280 .exp,
281 .exp2,
282 .log,
283 .log2,
284 .log10,
285 .fabs,
286 .floor,
287 .ceil,
288 .round,
289 .trunc_float,
290 .cmp_lt_errors_len,
291 => {
292 const o = air_datas[inst].un_op;
293 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
294 return .none;
295 },
296
297 .ret,
298 .ret_load,
299 => {
300 const o = air_datas[inst].un_op;
301 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .noret);
302 return .noret;
303 },
304
305 .set_err_return_trace => {
306 const o = air_datas[inst].un_op;
307 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
308 return .write;
309 },
310
311 .add_with_overflow,
312 .sub_with_overflow,
313 .mul_with_overflow,
314 .shl_with_overflow,
315 .ptr_add,
316 .ptr_sub,
317 .ptr_elem_ptr,
318 .slice_elem_ptr,
319 .slice,
320 => {
321 const ty_pl = air_datas[inst].ty_pl;
322 const extra = air.extraData(Air.Bin, ty_pl.payload).data;
323 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
324 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
325 return .none;
326 },
327
328 .dbg_var_ptr,
329 .dbg_var_val,
330 => {
331 const o = air_datas[inst].pl_op.operand;
332 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
333 return .none;
334 },
335
336 .prefetch => {
337 const prefetch = air_datas[inst].prefetch;
338 if (prefetch.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
339 return .none;
340 },
341
342 .call, .call_always_tail, .call_never_tail, .call_never_inline => {
343 const inst_data = air_datas[inst].pl_op;
344 const callee = inst_data.operand;
345 const extra = air.extraData(Air.Call, inst_data.payload);
346 const args = @ptrCast([]const Air.Inst.Ref, air.extra[extra.end..][0..extra.data.args_len]);
347 if (args.len + 1 <= bpi - 1) {
348 if (callee == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
349 for (args) |arg, i| {
350 if (arg == operand_ref) return matchOperandSmallIndex(l, inst, @intCast(OperandInt, i + 1), .write);
351 }
352 return .write;
353 }
354 var bt = l.iterateBigTomb(inst);
355 if (bt.feed()) {
356 if (callee == operand_ref) return .tomb;
357 } else {
358 if (callee == operand_ref) return .write;
359 }
360 for (args) |arg| {
361 if (bt.feed()) {
362 if (arg == operand_ref) return .tomb;
363 } else {
364 if (arg == operand_ref) return .write;
365 }
366 }
367 return .write;
368 },
369 .select => {
370 const pl_op = air_datas[inst].pl_op;
371 const extra = air.extraData(Air.Bin, pl_op.payload).data;
372 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
373 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
374 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
375 return .none;
376 },
377 .shuffle => {
378 const extra = air.extraData(Air.Shuffle, air_datas[inst].ty_pl.payload).data;
379 if (extra.a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
380 if (extra.b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
381 return .none;
382 },
383 .reduce => {
384 const reduce = air_datas[inst].reduce;
385 if (reduce.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
386 return .none;
387 },
388 .cmp_vector => {
389 const extra = air.extraData(Air.VectorCmp, air_datas[inst].ty_pl.payload).data;
390 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
391 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
392 return .none;
393 },
394 .aggregate_init => {
395 const ty_pl = air_datas[inst].ty_pl;
396 const aggregate_ty = air.getRefType(ty_pl.ty);
397 const len = @intCast(usize, aggregate_ty.arrayLen());
398 const elements = @ptrCast([]const Air.Inst.Ref, air.extra[ty_pl.payload..][0..len]);
399
400 if (elements.len <= bpi - 1) {
401 for (elements) |elem, i| {
402 if (elem == operand_ref) return matchOperandSmallIndex(l, inst, @intCast(OperandInt, i), .none);
403 }
404 return .none;
405 }
406
407 var bt = l.iterateBigTomb(inst);
408 for (elements) |elem| {
409 if (bt.feed()) {
410 if (elem == operand_ref) return .tomb;
411 } else {
412 if (elem == operand_ref) return .write;
413 }
414 }
415 return .write;
416 },
417 .union_init => {
418 const extra = air.extraData(Air.UnionInit, air_datas[inst].ty_pl.payload).data;
419 if (extra.init == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
420 return .none;
421 },
422 .struct_field_ptr, .struct_field_val => {
423 const extra = air.extraData(Air.StructField, air_datas[inst].ty_pl.payload).data;
424 if (extra.struct_operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
425 return .none;
426 },
427 .field_parent_ptr => {
428 const extra = air.extraData(Air.FieldParentPtr, air_datas[inst].ty_pl.payload).data;
429 if (extra.field_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
430 return .none;
431 },
432 .cmpxchg_strong, .cmpxchg_weak => {
433 const extra = air.extraData(Air.Cmpxchg, air_datas[inst].ty_pl.payload).data;
434 if (extra.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
435 if (extra.expected_value == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
436 if (extra.new_value == operand_ref) return matchOperandSmallIndex(l, inst, 2, .write);
437 return .write;
438 },
439 .mul_add => {
440 const pl_op = air_datas[inst].pl_op;
441 const extra = air.extraData(Air.Bin, pl_op.payload).data;
442 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
443 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
444 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
445 return .none;
446 },
447 .atomic_load => {
448 const ptr = air_datas[inst].atomic_load.ptr;
449 if (ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
450 return .none;
451 },
452 .atomic_rmw => {
453 const pl_op = air_datas[inst].pl_op;
454 const extra = air.extraData(Air.AtomicRmw, pl_op.payload).data;
455 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
456 if (extra.operand == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
457 return .write;
458 },
459 .memset,
460 .memcpy,
461 => {
462 const pl_op = air_datas[inst].pl_op;
463 const extra = air.extraData(Air.Bin, pl_op.payload).data;
464 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
465 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
466 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .write);
467 return .write;
468 },
469
470 .br => {
471 const br = air_datas[inst].br;
472 if (br.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .noret);
473 return .noret;
474 },
475 .assembly => {
476 return .complex;
477 },
478 .block => {
479 return .complex;
480 },
481 .loop => {
482 return .complex;
483 },
484 .cond_br => {
485 return .complex;
486 },
487 .switch_br => {
488 return .complex;
489 },
490 .wasm_memory_grow => {
491 const pl_op = air_datas[inst].pl_op;
492 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
493 return .none;
494 },
495 }
496}
497
498fn matchOperandSmallIndex(
499 l: Liveness,
500 inst: Air.Inst.Index,
501 operand: OperandInt,
502 default: OperandCategory,
503) OperandCategory {
504 if (operandDies(l, inst, operand)) {
505 return .tomb;
506 } else {
507 return default;
508 }
509}
510
115511/// Higher level API.
116512pub const CondBrSlices = struct {
117513 then_deaths: []const Air.Inst.Index,
src/codegen/llvm.zig+27-5
......@@ -3885,7 +3885,7 @@ pub const FuncGen = struct {
38853885
38863886 fn genBody(self: *FuncGen, body: []const Air.Inst.Index) Error!void {
38873887 const air_tags = self.air.instructions.items(.tag);
3888 for (body) |inst| {
3888 for (body) |inst, i| {
38893889 const opt_value: ?*const llvm.Value = switch (air_tags[inst]) {
38903890 // zig fmt: off
38913891 .add => try self.airAdd(inst),
......@@ -3976,7 +3976,7 @@ pub const FuncGen = struct {
39763976 .fptrunc => try self.airFptrunc(inst),
39773977 .fpext => try self.airFpext(inst),
39783978 .ptrtoint => try self.airPtrToInt(inst),
3979 .load => try self.airLoad(inst),
3979 .load => try self.airLoad(inst, body, i + 1),
39803980 .loop => try self.airLoop(inst),
39813981 .not => try self.airNot(inst),
39823982 .ret => try self.airRet(inst),
......@@ -6982,11 +6982,33 @@ pub const FuncGen = struct {
69826982 return null;
69836983 }
69846984
6985 fn airLoad(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
6985 fn airLoad(
6986 self: *FuncGen,
6987 inst: Air.Inst.Index,
6988 body: []const Air.Inst.Index,
6989 body_i: usize,
6990 ) !?*const llvm.Value {
69866991 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
69876992 const ptr_ty = self.air.typeOf(ty_op.operand);
6988 if (!ptr_ty.isVolatilePtr() and self.liveness.isUnused(inst))
6989 return null;
6993 elide: {
6994 const ptr_info = ptr_ty.ptrInfo().data;
6995 if (ptr_info.@"volatile") break :elide;
6996 if (self.liveness.isUnused(inst)) return null;
6997 if (!isByRef(ptr_info.pointee_type)) break :elide;
6998
6999 // It would be valid to fall back to the code below here that simply calls
7000 // load(). However, as an optimization, we want to avoid unnecessary copies
7001 // of isByRef=true types. Here, we scan forward in the current block,
7002 // looking to see if this load dies before any side effects occur.
7003 // In such case, we can safely return the operand without making a copy.
7004 for (body[body_i..]) |body_inst| {
7005 switch (self.liveness.categorizeOperand(self.air, body_inst, inst)) {
7006 .none => continue,
7007 .write, .noret, .complex => break :elide,
7008 .tomb => return try self.resolveInst(ty_op.operand),
7009 }
7010 } else unreachable;
7011 }
69907012 const ptr = try self.resolveInst(ty_op.operand);
69917013 return self.load(ptr, ptr_ty);
69927014 }