authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2022-12-04 07:58:59-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-12-04 15:57:40-07:00
log7d3cc3bc8d772519d390b7f13346eeab73bc0c21
tree8b9e7c78245891efee3cc8b4f0e3ad770704bb7e
parent518392d6fe79b780164d40782f15e626444d34f8

CBE: defer invariant local reuse in loops

When a local defined outside a loop dies inside the loop, it can still be needed on subsequent loop iterations, so reuse of the local must be deferred until after the loop ends. This causes behavior tests to pass.

1 files changed, 90 insertions(+), 39 deletions(-)

src/codegen/c.zig+90-39
......@@ -69,14 +69,18 @@ pub const TypedefMap = std.ArrayHashMap(
6969 true,
7070);
7171
72const LoopDepth = u16;
7273const Local = struct {
7374 ty: Type,
7475 alignment: u32,
76 /// How many loops the last definition was nested in.
77 loop_depth: LoopDepth,
7578};
7679
7780const LocalIndex = u16;
7881const LocalsList = std.ArrayListUnmanaged(LocalIndex);
7982const LocalsMap = std.ArrayHashMapUnmanaged(Type, LocalsList, Type.HashContext32, true);
83const LocalsStack = std.ArrayListUnmanaged(LocalsMap);
8084
8185const FormatTypeAsCIdentContext = struct {
8286 ty: Type,
......@@ -265,15 +269,18 @@ pub const Function = struct {
265269 /// All the locals, to be emitted at the top of the function.
266270 locals: std.ArrayListUnmanaged(Local) = .{},
267271 /// Which locals are available for reuse, based on Type.
268 free_locals: LocalsMap = .{},
272 /// Only locals in the last stack entry are available for reuse,
273 /// other entries will become available on loop exit.
274 free_locals_stack: LocalsStack = .{},
275 free_locals_clone_depth: LoopDepth = 0,
269276 /// Locals which will not be freed by Liveness. This is used after a
270 /// Function body is lowered in order to make `free_locals` have 100% of
271 /// the locals within so that it can be used to render the block of
272 /// variable declarations at the top of a function, sorted descending by
273 /// type alignment.
277 /// Function body is lowered in order to make `free_locals_stack` have
278 /// 100% of the locals within so that it can be used to render the block
279 /// of variable declarations at the top of a function, sorted descending
280 /// by type alignment.
274281 /// The value is whether the alloc is static or not.
275282 allocs: std.AutoArrayHashMapUnmanaged(LocalIndex, bool) = .{},
276 /// Needed for memory used by Type objects used as keys in free_locals.
283 /// Needed for memory used by the keys of free_locals_stack entries.
277284 arena: std.heap.ArenaAllocator,
278285
279286 fn tyHashCtx(f: Function) Type.HashContext32 {
......@@ -312,12 +319,17 @@ pub const Function = struct {
312319 };
313320 }
314321
322 fn getFreeLocals(f: *Function) *LocalsMap {
323 return &f.free_locals_stack.items[f.free_locals_stack.items.len - 1];
324 }
325
315326 /// Skips the reuse logic.
316327 fn allocLocalValue(f: *Function, ty: Type, alignment: u32) !CValue {
317328 const gpa = f.object.dg.gpa;
318329 try f.locals.append(gpa, .{
319330 .ty = ty,
320331 .alignment = alignment,
332 .loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1),
321333 });
322334 return CValue{ .local = @intCast(LocalIndex, f.locals.items.len - 1) };
323335 }
......@@ -332,10 +344,11 @@ pub const Function = struct {
332344 fn allocAlignedLocal(f: *Function, ty: Type, mutability: Mutability, alignment: u32) !CValue {
333345 _ = mutability;
334346
335 if (f.free_locals.getPtrContext(ty, f.tyHashCtx())) |locals_list| {
347 if (f.getFreeLocals().getPtrContext(ty, f.tyHashCtx())) |locals_list| {
336348 for (locals_list.items) |local_index, i| {
337 const local = f.locals.items[local_index];
349 const local = &f.locals.items[local_index];
338350 if (local.alignment >= alignment) {
351 local.loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1);
339352 _ = locals_list.swapRemove(i);
340353 return CValue{ .local = local_index };
341354 }
......@@ -416,7 +429,10 @@ pub const Function = struct {
416429 pub fn deinit(f: *Function, gpa: mem.Allocator) void {
417430 f.allocs.deinit(gpa);
418431 f.locals.deinit(gpa);
419 deinitFreeLocalsMap(gpa, &f.free_locals);
432 for (f.free_locals_stack.items) |*free_locals| {
433 deinitFreeLocalsMap(gpa, free_locals);
434 }
435 f.free_locals_stack.deinit(gpa);
420436 f.blocks.deinit(gpa);
421437 f.value_map.deinit();
422438 f.object.code.deinit();
......@@ -2480,6 +2496,9 @@ pub fn genFunc(f: *Function) !void {
24802496 o.code_header.appendSliceAssumeCapacity("{\n ");
24812497 const empty_header_len = o.code_header.items.len;
24822498
2499 f.free_locals_stack.clearRetainingCapacity();
2500 try f.free_locals_stack.append(gpa, .{});
2501
24832502 const main_body = f.air.getMainBody();
24842503 try genBody(f, main_body);
24852504
......@@ -2490,12 +2509,13 @@ pub fn genFunc(f: *Function) !void {
24902509 // Liveness analysis, however, locals from alloc instructions will be
24912510 // missing. These are added now to complete the map. Then we can sort by
24922511 // alignment, descending.
2512 const free_locals = f.getFreeLocals();
24932513 const values = f.allocs.values();
24942514 for (f.allocs.keys()) |local_index, i| {
24952515 if (values[i]) continue; // static
24962516 const local = f.locals.items[local_index];
24972517 log.debug("inserting local {d} into free_locals", .{local_index});
2498 const gop = try f.free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx());
2518 const gop = try free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx());
24992519 if (!gop.found_existing) gop.value_ptr.* = .{};
25002520 try gop.value_ptr.append(gpa, local_index);
25012521 }
......@@ -2511,10 +2531,10 @@ pub fn genFunc(f: *Function) !void {
25112531 }
25122532 };
25132533 const target = o.dg.module.getTarget();
2514 f.free_locals.sort(SortContext{ .target = target, .keys = f.free_locals.keys() });
2534 free_locals.sort(SortContext{ .target = target, .keys = free_locals.keys() });
25152535
25162536 const w = o.code_header.writer();
2517 for (f.free_locals.values()) |list| {
2537 for (free_locals.values()) |list| {
25182538 for (list.items) |local_index| {
25192539 const local = f.locals.items[local_index];
25202540 try o.dg.renderTypeAndName(
......@@ -4282,9 +4302,30 @@ fn airLoop(f: *Function, inst: Air.Inst.Index) !CValue {
42824302 const loop = f.air.extraData(Air.Block, ty_pl.payload);
42834303 const body = f.air.extra[loop.end..][0..loop.data.body_len];
42844304 const writer = f.object.writer();
4305
4306 const gpa = f.object.dg.gpa;
4307 try f.free_locals_stack.insert(gpa, f.free_locals_stack.items.len - 1, .{});
4308
42854309 try writer.writeAll("for (;;) ");
42864310 try genBody(f, body);
42874311 try writer.writeByte('\n');
4312
4313 var old_free_locals = f.free_locals_stack.pop();
4314 defer deinitFreeLocalsMap(gpa, &old_free_locals);
4315 const new_free_locals = f.getFreeLocals();
4316 var it = new_free_locals.iterator();
4317 while (it.next()) |entry| {
4318 const gop = try old_free_locals.getOrPutContext(gpa, entry.key_ptr.*, f.tyHashCtx());
4319 if (gop.found_existing) {
4320 try gop.value_ptr.appendSlice(gpa, entry.value_ptr.items);
4321 } else {
4322 gop.value_ptr.* = entry.value_ptr.*;
4323 entry.value_ptr.* = .{};
4324 }
4325 }
4326 deinitFreeLocalsMap(gpa, new_free_locals);
4327 new_free_locals.* = old_free_locals.move();
4328
42884329 return CValue.none;
42894330}
42904331
......@@ -4303,17 +4344,19 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {
43034344 const gpa = f.object.dg.gpa;
43044345 var cloned_map = try f.value_map.clone();
43054346 defer cloned_map.deinit();
4306 var cloned_frees = try cloneFreeLocalsMap(gpa, &f.free_locals);
4347 var cloned_frees = try cloneFreeLocalsMap(gpa, f.getFreeLocals());
43074348 defer deinitFreeLocalsMap(gpa, &cloned_frees);
43084349
4309 for (liveness_condbr.then_deaths) |operand| {
4310 try die(f, inst, Air.indexToRef(operand));
4311 }
4312
43134350 // Remember how many locals there were before entering the then branch so
43144351 // that we can notice and use them in the else branch. Any new locals must
43154352 // necessarily be free already after the then branch is complete.
43164353 const pre_locals_len = @intCast(LocalIndex, f.locals.items.len);
4354 const pre_clone_depth = f.free_locals_clone_depth;
4355 f.free_locals_clone_depth = @intCast(LoopDepth, f.free_locals_stack.items.len);
4356
4357 for (liveness_condbr.then_deaths) |operand| {
4358 try die(f, inst, Air.indexToRef(operand));
4359 }
43174360
43184361 try writer.writeAll("if (");
43194362 try f.writeCValue(writer, cond, .Other);
......@@ -4322,13 +4365,15 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {
43224365 try writer.writeAll(" else ");
43234366 f.value_map.deinit();
43244367 f.value_map = cloned_map.move();
4325 deinitFreeLocalsMap(gpa, &f.free_locals);
4326 f.free_locals = cloned_frees.move();
4368 const free_locals = f.getFreeLocals();
4369 deinitFreeLocalsMap(gpa, free_locals);
4370 free_locals.* = cloned_frees.move();
4371 f.free_locals_clone_depth = pre_clone_depth;
43274372 for (liveness_condbr.else_deaths) |operand| {
43284373 try die(f, inst, Air.indexToRef(operand));
43294374 }
43304375
4331 try noticeBranchFrees(f, pre_locals_len);
4376 try noticeBranchFrees(f, pre_locals_len, inst);
43324377
43334378 try genBody(f, else_body);
43344379 try f.object.indent_writer.insertNewline();
......@@ -4390,20 +4435,25 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
43904435 if (case_i != last_case_i) {
43914436 const old_value_map = f.value_map;
43924437 f.value_map = try old_value_map.clone();
4393 const old_free_locals = f.free_locals;
4394 f.free_locals = try cloneFreeLocalsMap(gpa, &f.free_locals);
4438 var free_locals = f.getFreeLocals();
4439 const old_free_locals = free_locals.*;
4440 free_locals.* = try cloneFreeLocalsMap(gpa, free_locals);
43954441
43964442 // Remember how many locals there were before entering each branch so that
43974443 // we can notice and use them in subsequent branches. Any new locals must
43984444 // necessarily be free already after the previous branch is complete.
43994445 const pre_locals_len = @intCast(LocalIndex, f.locals.items.len);
4446 const pre_clone_depth = f.free_locals_clone_depth;
4447 f.free_locals_clone_depth = @intCast(LoopDepth, f.free_locals_stack.items.len);
44004448
44014449 {
44024450 defer {
4451 f.free_locals_clone_depth = pre_clone_depth;
44034452 f.value_map.deinit();
4404 deinitFreeLocalsMap(gpa, &f.free_locals);
4453 free_locals = f.getFreeLocals();
4454 deinitFreeLocalsMap(gpa, free_locals);
44054455 f.value_map = old_value_map;
4406 f.free_locals = old_free_locals;
4456 free_locals.* = old_free_locals;
44074457 }
44084458
44094459 for (liveness.deaths[case_i]) |operand| {
......@@ -4413,7 +4463,7 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
44134463 try genBody(f, case_body);
44144464 }
44154465
4416 try noticeBranchFrees(f, pre_locals_len);
4466 try noticeBranchFrees(f, pre_locals_len, inst);
44174467 } else {
44184468 for (liveness.deaths[case_i]) |operand| {
44194469 try die(f, inst, Air.indexToRef(operand));
......@@ -6193,7 +6243,7 @@ fn airReduce(f: *Function, inst: Air.Inst.Index) !CValue {
61936243 try writer.writeAll(init_val);
61946244 try writer.writeAll(";");
61956245 try f.object.indent_writer.insertNewline();
6196 try writer.writeAll("for(;");
6246 try writer.writeAll("for (;");
61976247 try f.writeCValue(writer, it, .Other);
61986248 try writer.print("<{d};++", .{vector_len});
61996249 try f.writeCValue(writer, it, .Other);
......@@ -6998,13 +7048,15 @@ fn die(f: *Function, inst: Air.Inst.Index, ref: Air.Inst.Ref) !void {
69987048
69997049fn freeLocal(f: *Function, inst: Air.Inst.Index, local_index: LocalIndex, ref_inst: Air.Inst.Index) !void {
70007050 const gpa = f.object.dg.gpa;
7001 const gop = try f.free_locals.getOrPutContext(
7051 const local = &f.locals.items[local_index];
7052 log.debug("%{d}: freeing t{d} (operand %{d})", .{ inst, local_index, ref_inst });
7053 if (local.loop_depth < f.free_locals_clone_depth) return;
7054 const gop = try f.free_locals_stack.items[local.loop_depth].getOrPutContext(
70027055 gpa,
7003 f.locals.items[local_index].ty,
7056 local.ty,
70047057 f.tyHashCtx(),
70057058 );
70067059 if (!gop.found_existing) gop.value_ptr.* = .{};
7007 log.debug("%{d}: freeing t{d} (operand %{d})", .{ inst, local_index, ref_inst });
70087060 if (std.debug.runtime_safety) {
70097061 // If this trips, it means a local is being inserted into the
70107062 // free_locals map while it already exists in the map, which is not
......@@ -7062,15 +7114,14 @@ fn deinitFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) void {
70627114 map.deinit(gpa);
70637115}
70647116
7065fn noticeBranchFrees(f: *Function, pre_locals_len: LocalIndex) !void {
7066 const gpa = f.object.dg.gpa;
7067 var i = pre_locals_len;
7068 while (i < f.locals.items.len) : (i += 1) {
7069 const local = f.locals.items[i];
7070 const unfreeable = f.allocs.contains(i);
7071 if (unfreeable) continue;
7072 const gop = try f.free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx());
7073 if (!gop.found_existing) gop.value_ptr.* = .{};
7074 try gop.value_ptr.append(gpa, i);
7117fn noticeBranchFrees(f: *Function, pre_locals_len: LocalIndex, inst: Air.Inst.Index) !void {
7118 for (f.locals.items[pre_locals_len..]) |*local, local_offset| {
7119 const local_index = pre_locals_len + @intCast(LocalIndex, local_offset);
7120 if (f.allocs.contains(local_index)) continue; // allocs are not freeable
7121
7122 // free more deeply nested locals from other branches at current depth
7123 assert(local.loop_depth >= f.free_locals_stack.items.len - 1);
7124 local.loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1);
7125 try freeLocal(f, inst, local_index, 0);
70757126 }
70767127}