authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2022-12-04 07:58:59-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-12-04 15:57:40-07:00
log7d3cc3bc8d772519d390b7f13346eeab73bc0c21
tree8b9e7c78245891efee3cc8b4f0e3ad770704bb7e
parent518392d6fe79b780164d40782f15e626444d34f8

CBE: defer invariant local reuse in loops

When a local defined outside a loop dies inside the loop, it can still be needed on subsequent loop iterations, so reuse of the local must be deferred until after the loop ends. This causes behavior tests to pass.

1 files changed, 90 insertions(+), 39 deletions(-)

src/codegen/c.zig+90-39
...@@ -69,14 +69,18 @@ pub const TypedefMap = std.ArrayHashMap(...@@ -69,14 +69,18 @@ pub const TypedefMap = std.ArrayHashMap(
69 true,69 true,
70);70);
7171
72const LoopDepth = u16;
72const Local = struct {73const Local = struct {
73 ty: Type,74 ty: Type,
74 alignment: u32,75 alignment: u32,
76 /// How many loops the last definition was nested in.
77 loop_depth: LoopDepth,
75};78};
7679
77const LocalIndex = u16;80const LocalIndex = u16;
78const LocalsList = std.ArrayListUnmanaged(LocalIndex);81const LocalsList = std.ArrayListUnmanaged(LocalIndex);
79const LocalsMap = std.ArrayHashMapUnmanaged(Type, LocalsList, Type.HashContext32, true);82const LocalsMap = std.ArrayHashMapUnmanaged(Type, LocalsList, Type.HashContext32, true);
83const LocalsStack = std.ArrayListUnmanaged(LocalsMap);
8084
81const FormatTypeAsCIdentContext = struct {85const FormatTypeAsCIdentContext = struct {
82 ty: Type,86 ty: Type,
...@@ -265,15 +269,18 @@ pub const Function = struct {...@@ -265,15 +269,18 @@ pub const Function = struct {
265 /// All the locals, to be emitted at the top of the function.269 /// All the locals, to be emitted at the top of the function.
266 locals: std.ArrayListUnmanaged(Local) = .{},270 locals: std.ArrayListUnmanaged(Local) = .{},
267 /// Which locals are available for reuse, based on Type.271 /// Which locals are available for reuse, based on Type.
268 free_locals: LocalsMap = .{},272 /// Only locals in the last stack entry are available for reuse,
273 /// other entries will become available on loop exit.
274 free_locals_stack: LocalsStack = .{},
275 free_locals_clone_depth: LoopDepth = 0,
269 /// Locals which will not be freed by Liveness. This is used after a276 /// Locals which will not be freed by Liveness. This is used after a
270 /// Function body is lowered in order to make `free_locals` have 100% of277 /// Function body is lowered in order to make `free_locals_stack` have
271 /// the locals within so that it can be used to render the block of278 /// 100% of the locals within so that it can be used to render the block
272 /// variable declarations at the top of a function, sorted descending by279 /// of variable declarations at the top of a function, sorted descending
273 /// type alignment.280 /// by type alignment.
274 /// The value is whether the alloc is static or not.281 /// The value is whether the alloc is static or not.
275 allocs: std.AutoArrayHashMapUnmanaged(LocalIndex, bool) = .{},282 allocs: std.AutoArrayHashMapUnmanaged(LocalIndex, bool) = .{},
276 /// Needed for memory used by Type objects used as keys in free_locals.283 /// Needed for memory used by the keys of free_locals_stack entries.
277 arena: std.heap.ArenaAllocator,284 arena: std.heap.ArenaAllocator,
278285
279 fn tyHashCtx(f: Function) Type.HashContext32 {286 fn tyHashCtx(f: Function) Type.HashContext32 {
...@@ -312,12 +319,17 @@ pub const Function = struct {...@@ -312,12 +319,17 @@ pub const Function = struct {
312 };319 };
313 }320 }
314321
322 fn getFreeLocals(f: *Function) *LocalsMap {
323 return &f.free_locals_stack.items[f.free_locals_stack.items.len - 1];
324 }
325
315 /// Skips the reuse logic.326 /// Skips the reuse logic.
316 fn allocLocalValue(f: *Function, ty: Type, alignment: u32) !CValue {327 fn allocLocalValue(f: *Function, ty: Type, alignment: u32) !CValue {
317 const gpa = f.object.dg.gpa;328 const gpa = f.object.dg.gpa;
318 try f.locals.append(gpa, .{329 try f.locals.append(gpa, .{
319 .ty = ty,330 .ty = ty,
320 .alignment = alignment,331 .alignment = alignment,
332 .loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1),
321 });333 });
322 return CValue{ .local = @intCast(LocalIndex, f.locals.items.len - 1) };334 return CValue{ .local = @intCast(LocalIndex, f.locals.items.len - 1) };
323 }335 }
...@@ -332,10 +344,11 @@ pub const Function = struct {...@@ -332,10 +344,11 @@ pub const Function = struct {
332 fn allocAlignedLocal(f: *Function, ty: Type, mutability: Mutability, alignment: u32) !CValue {344 fn allocAlignedLocal(f: *Function, ty: Type, mutability: Mutability, alignment: u32) !CValue {
333 _ = mutability;345 _ = mutability;
334346
335 if (f.free_locals.getPtrContext(ty, f.tyHashCtx())) |locals_list| {347 if (f.getFreeLocals().getPtrContext(ty, f.tyHashCtx())) |locals_list| {
336 for (locals_list.items) |local_index, i| {348 for (locals_list.items) |local_index, i| {
337 const local = f.locals.items[local_index];349 const local = &f.locals.items[local_index];
338 if (local.alignment >= alignment) {350 if (local.alignment >= alignment) {
351 local.loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1);
339 _ = locals_list.swapRemove(i);352 _ = locals_list.swapRemove(i);
340 return CValue{ .local = local_index };353 return CValue{ .local = local_index };
341 }354 }
...@@ -416,7 +429,10 @@ pub const Function = struct {...@@ -416,7 +429,10 @@ pub const Function = struct {
416 pub fn deinit(f: *Function, gpa: mem.Allocator) void {429 pub fn deinit(f: *Function, gpa: mem.Allocator) void {
417 f.allocs.deinit(gpa);430 f.allocs.deinit(gpa);
418 f.locals.deinit(gpa);431 f.locals.deinit(gpa);
419 deinitFreeLocalsMap(gpa, &f.free_locals);432 for (f.free_locals_stack.items) |*free_locals| {
433 deinitFreeLocalsMap(gpa, free_locals);
434 }
435 f.free_locals_stack.deinit(gpa);
420 f.blocks.deinit(gpa);436 f.blocks.deinit(gpa);
421 f.value_map.deinit();437 f.value_map.deinit();
422 f.object.code.deinit();438 f.object.code.deinit();
...@@ -2480,6 +2496,9 @@ pub fn genFunc(f: *Function) !void {...@@ -2480,6 +2496,9 @@ pub fn genFunc(f: *Function) !void {
2480 o.code_header.appendSliceAssumeCapacity("{\n ");2496 o.code_header.appendSliceAssumeCapacity("{\n ");
2481 const empty_header_len = o.code_header.items.len;2497 const empty_header_len = o.code_header.items.len;
24822498
2499 f.free_locals_stack.clearRetainingCapacity();
2500 try f.free_locals_stack.append(gpa, .{});
2501
2483 const main_body = f.air.getMainBody();2502 const main_body = f.air.getMainBody();
2484 try genBody(f, main_body);2503 try genBody(f, main_body);
24852504
...@@ -2490,12 +2509,13 @@ pub fn genFunc(f: *Function) !void {...@@ -2490,12 +2509,13 @@ pub fn genFunc(f: *Function) !void {
2490 // Liveness analysis, however, locals from alloc instructions will be2509 // Liveness analysis, however, locals from alloc instructions will be
2491 // missing. These are added now to complete the map. Then we can sort by2510 // missing. These are added now to complete the map. Then we can sort by
2492 // alignment, descending.2511 // alignment, descending.
2512 const free_locals = f.getFreeLocals();
2493 const values = f.allocs.values();2513 const values = f.allocs.values();
2494 for (f.allocs.keys()) |local_index, i| {2514 for (f.allocs.keys()) |local_index, i| {
2495 if (values[i]) continue; // static2515 if (values[i]) continue; // static
2496 const local = f.locals.items[local_index];2516 const local = f.locals.items[local_index];
2497 log.debug("inserting local {d} into free_locals", .{local_index});2517 log.debug("inserting local {d} into free_locals", .{local_index});
2498 const gop = try f.free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx());2518 const gop = try free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx());
2499 if (!gop.found_existing) gop.value_ptr.* = .{};2519 if (!gop.found_existing) gop.value_ptr.* = .{};
2500 try gop.value_ptr.append(gpa, local_index);2520 try gop.value_ptr.append(gpa, local_index);
2501 }2521 }
...@@ -2511,10 +2531,10 @@ pub fn genFunc(f: *Function) !void {...@@ -2511,10 +2531,10 @@ pub fn genFunc(f: *Function) !void {
2511 }2531 }
2512 };2532 };
2513 const target = o.dg.module.getTarget();2533 const target = o.dg.module.getTarget();
2514 f.free_locals.sort(SortContext{ .target = target, .keys = f.free_locals.keys() });2534 free_locals.sort(SortContext{ .target = target, .keys = free_locals.keys() });
25152535
2516 const w = o.code_header.writer();2536 const w = o.code_header.writer();
2517 for (f.free_locals.values()) |list| {2537 for (free_locals.values()) |list| {
2518 for (list.items) |local_index| {2538 for (list.items) |local_index| {
2519 const local = f.locals.items[local_index];2539 const local = f.locals.items[local_index];
2520 try o.dg.renderTypeAndName(2540 try o.dg.renderTypeAndName(
...@@ -4282,9 +4302,30 @@ fn airLoop(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4282,9 +4302,30 @@ fn airLoop(f: *Function, inst: Air.Inst.Index) !CValue {
4282 const loop = f.air.extraData(Air.Block, ty_pl.payload);4302 const loop = f.air.extraData(Air.Block, ty_pl.payload);
4283 const body = f.air.extra[loop.end..][0..loop.data.body_len];4303 const body = f.air.extra[loop.end..][0..loop.data.body_len];
4284 const writer = f.object.writer();4304 const writer = f.object.writer();
4305
4306 const gpa = f.object.dg.gpa;
4307 try f.free_locals_stack.insert(gpa, f.free_locals_stack.items.len - 1, .{});
4308
4285 try writer.writeAll("for (;;) ");4309 try writer.writeAll("for (;;) ");
4286 try genBody(f, body);4310 try genBody(f, body);
4287 try writer.writeByte('\n');4311 try writer.writeByte('\n');
4312
4313 var old_free_locals = f.free_locals_stack.pop();
4314 defer deinitFreeLocalsMap(gpa, &old_free_locals);
4315 const new_free_locals = f.getFreeLocals();
4316 var it = new_free_locals.iterator();
4317 while (it.next()) |entry| {
4318 const gop = try old_free_locals.getOrPutContext(gpa, entry.key_ptr.*, f.tyHashCtx());
4319 if (gop.found_existing) {
4320 try gop.value_ptr.appendSlice(gpa, entry.value_ptr.items);
4321 } else {
4322 gop.value_ptr.* = entry.value_ptr.*;
4323 entry.value_ptr.* = .{};
4324 }
4325 }
4326 deinitFreeLocalsMap(gpa, new_free_locals);
4327 new_free_locals.* = old_free_locals.move();
4328
4288 return CValue.none;4329 return CValue.none;
4289}4330}
42904331
...@@ -4303,17 +4344,19 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4303,17 +4344,19 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {
4303 const gpa = f.object.dg.gpa;4344 const gpa = f.object.dg.gpa;
4304 var cloned_map = try f.value_map.clone();4345 var cloned_map = try f.value_map.clone();
4305 defer cloned_map.deinit();4346 defer cloned_map.deinit();
4306 var cloned_frees = try cloneFreeLocalsMap(gpa, &f.free_locals);4347 var cloned_frees = try cloneFreeLocalsMap(gpa, f.getFreeLocals());
4307 defer deinitFreeLocalsMap(gpa, &cloned_frees);4348 defer deinitFreeLocalsMap(gpa, &cloned_frees);
43084349
4309 for (liveness_condbr.then_deaths) |operand| {
4310 try die(f, inst, Air.indexToRef(operand));
4311 }
4312
4313 // Remember how many locals there were before entering the then branch so4350 // Remember how many locals there were before entering the then branch so
4314 // that we can notice and use them in the else branch. Any new locals must4351 // that we can notice and use them in the else branch. Any new locals must
4315 // necessarily be free already after the then branch is complete.4352 // necessarily be free already after the then branch is complete.
4316 const pre_locals_len = @intCast(LocalIndex, f.locals.items.len);4353 const pre_locals_len = @intCast(LocalIndex, f.locals.items.len);
4354 const pre_clone_depth = f.free_locals_clone_depth;
4355 f.free_locals_clone_depth = @intCast(LoopDepth, f.free_locals_stack.items.len);
4356
4357 for (liveness_condbr.then_deaths) |operand| {
4358 try die(f, inst, Air.indexToRef(operand));
4359 }
43174360
4318 try writer.writeAll("if (");4361 try writer.writeAll("if (");
4319 try f.writeCValue(writer, cond, .Other);4362 try f.writeCValue(writer, cond, .Other);
...@@ -4322,13 +4365,15 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4322,13 +4365,15 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {
4322 try writer.writeAll(" else ");4365 try writer.writeAll(" else ");
4323 f.value_map.deinit();4366 f.value_map.deinit();
4324 f.value_map = cloned_map.move();4367 f.value_map = cloned_map.move();
4325 deinitFreeLocalsMap(gpa, &f.free_locals);4368 const free_locals = f.getFreeLocals();
4326 f.free_locals = cloned_frees.move();4369 deinitFreeLocalsMap(gpa, free_locals);
4370 free_locals.* = cloned_frees.move();
4371 f.free_locals_clone_depth = pre_clone_depth;
4327 for (liveness_condbr.else_deaths) |operand| {4372 for (liveness_condbr.else_deaths) |operand| {
4328 try die(f, inst, Air.indexToRef(operand));4373 try die(f, inst, Air.indexToRef(operand));
4329 }4374 }
43304375
4331 try noticeBranchFrees(f, pre_locals_len);4376 try noticeBranchFrees(f, pre_locals_len, inst);
43324377
4333 try genBody(f, else_body);4378 try genBody(f, else_body);
4334 try f.object.indent_writer.insertNewline();4379 try f.object.indent_writer.insertNewline();
...@@ -4390,20 +4435,25 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4390,20 +4435,25 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
4390 if (case_i != last_case_i) {4435 if (case_i != last_case_i) {
4391 const old_value_map = f.value_map;4436 const old_value_map = f.value_map;
4392 f.value_map = try old_value_map.clone();4437 f.value_map = try old_value_map.clone();
4393 const old_free_locals = f.free_locals;4438 var free_locals = f.getFreeLocals();
4394 f.free_locals = try cloneFreeLocalsMap(gpa, &f.free_locals);4439 const old_free_locals = free_locals.*;
4440 free_locals.* = try cloneFreeLocalsMap(gpa, free_locals);
43954441
4396 // Remember how many locals there were before entering each branch so that4442 // Remember how many locals there were before entering each branch so that
4397 // we can notice and use them in subsequent branches. Any new locals must4443 // we can notice and use them in subsequent branches. Any new locals must
4398 // necessarily be free already after the previous branch is complete.4444 // necessarily be free already after the previous branch is complete.
4399 const pre_locals_len = @intCast(LocalIndex, f.locals.items.len);4445 const pre_locals_len = @intCast(LocalIndex, f.locals.items.len);
4446 const pre_clone_depth = f.free_locals_clone_depth;
4447 f.free_locals_clone_depth = @intCast(LoopDepth, f.free_locals_stack.items.len);
44004448
4401 {4449 {
4402 defer {4450 defer {
4451 f.free_locals_clone_depth = pre_clone_depth;
4403 f.value_map.deinit();4452 f.value_map.deinit();
4404 deinitFreeLocalsMap(gpa, &f.free_locals);4453 free_locals = f.getFreeLocals();
4454 deinitFreeLocalsMap(gpa, free_locals);
4405 f.value_map = old_value_map;4455 f.value_map = old_value_map;
4406 f.free_locals = old_free_locals;4456 free_locals.* = old_free_locals;
4407 }4457 }
44084458
4409 for (liveness.deaths[case_i]) |operand| {4459 for (liveness.deaths[case_i]) |operand| {
...@@ -4413,7 +4463,7 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4413,7 +4463,7 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
4413 try genBody(f, case_body);4463 try genBody(f, case_body);
4414 }4464 }
44154465
4416 try noticeBranchFrees(f, pre_locals_len);4466 try noticeBranchFrees(f, pre_locals_len, inst);
4417 } else {4467 } else {
4418 for (liveness.deaths[case_i]) |operand| {4468 for (liveness.deaths[case_i]) |operand| {
4419 try die(f, inst, Air.indexToRef(operand));4469 try die(f, inst, Air.indexToRef(operand));
...@@ -6193,7 +6243,7 @@ fn airReduce(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -6193,7 +6243,7 @@ fn airReduce(f: *Function, inst: Air.Inst.Index) !CValue {
6193 try writer.writeAll(init_val);6243 try writer.writeAll(init_val);
6194 try writer.writeAll(";");6244 try writer.writeAll(";");
6195 try f.object.indent_writer.insertNewline();6245 try f.object.indent_writer.insertNewline();
6196 try writer.writeAll("for(;");6246 try writer.writeAll("for (;");
6197 try f.writeCValue(writer, it, .Other);6247 try f.writeCValue(writer, it, .Other);
6198 try writer.print("<{d};++", .{vector_len});6248 try writer.print("<{d};++", .{vector_len});
6199 try f.writeCValue(writer, it, .Other);6249 try f.writeCValue(writer, it, .Other);
...@@ -6998,13 +7048,15 @@ fn die(f: *Function, inst: Air.Inst.Index, ref: Air.Inst.Ref) !void {...@@ -6998,13 +7048,15 @@ fn die(f: *Function, inst: Air.Inst.Index, ref: Air.Inst.Ref) !void {
69987048
6999fn freeLocal(f: *Function, inst: Air.Inst.Index, local_index: LocalIndex, ref_inst: Air.Inst.Index) !void {7049fn freeLocal(f: *Function, inst: Air.Inst.Index, local_index: LocalIndex, ref_inst: Air.Inst.Index) !void {
7000 const gpa = f.object.dg.gpa;7050 const gpa = f.object.dg.gpa;
7001 const gop = try f.free_locals.getOrPutContext(7051 const local = &f.locals.items[local_index];
7052 log.debug("%{d}: freeing t{d} (operand %{d})", .{ inst, local_index, ref_inst });
7053 if (local.loop_depth < f.free_locals_clone_depth) return;
7054 const gop = try f.free_locals_stack.items[local.loop_depth].getOrPutContext(
7002 gpa,7055 gpa,
7003 f.locals.items[local_index].ty,7056 local.ty,
7004 f.tyHashCtx(),7057 f.tyHashCtx(),
7005 );7058 );
7006 if (!gop.found_existing) gop.value_ptr.* = .{};7059 if (!gop.found_existing) gop.value_ptr.* = .{};
7007 log.debug("%{d}: freeing t{d} (operand %{d})", .{ inst, local_index, ref_inst });
7008 if (std.debug.runtime_safety) {7060 if (std.debug.runtime_safety) {
7009 // If this trips, it means a local is being inserted into the7061 // If this trips, it means a local is being inserted into the
7010 // free_locals map while it already exists in the map, which is not7062 // free_locals map while it already exists in the map, which is not
...@@ -7062,15 +7114,14 @@ fn deinitFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) void {...@@ -7062,15 +7114,14 @@ fn deinitFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) void {
7062 map.deinit(gpa);7114 map.deinit(gpa);
7063}7115}
70647116
7065fn noticeBranchFrees(f: *Function, pre_locals_len: LocalIndex) !void {7117fn noticeBranchFrees(f: *Function, pre_locals_len: LocalIndex, inst: Air.Inst.Index) !void {
7066 const gpa = f.object.dg.gpa;7118 for (f.locals.items[pre_locals_len..]) |*local, local_offset| {
7067 var i = pre_locals_len;7119 const local_index = pre_locals_len + @intCast(LocalIndex, local_offset);
7068 while (i < f.locals.items.len) : (i += 1) {7120 if (f.allocs.contains(local_index)) continue; // allocs are not freeable
7069 const local = f.locals.items[i];7121
7070 const unfreeable = f.allocs.contains(i);7122 // free more deeply nested locals from other branches at current depth
7071 if (unfreeable) continue;7123 assert(local.loop_depth >= f.free_locals_stack.items.len - 1);
7072 const gop = try f.free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx());7124 local.loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1);
7073 if (!gop.found_existing) gop.value_ptr.* = .{};7125 try freeLocal(f, inst, local_index, 0);
7074 try gop.value_ptr.append(gpa, i);
7075 }7126 }
7076}7127}