| ... | ... | @@ -82,15 +82,20 @@ pub const LazyFnMap = std.AutoArrayHashMapUnmanaged(LazyFnKey, LazyFnValue); |
| 82 | 82 | |
| 83 | 83 | const LoopDepth = u16; |
| 84 | 84 | const Local = struct { |
| 85 | | ty: Type, |
| 86 | | alignment: u32, |
| 85 | cty_idx: CType.Index, |
| 87 | 86 | /// How many loops the last definition was nested in. |
| 88 | 87 | loop_depth: LoopDepth, |
| 88 | alignas: CType.AlignAs, |
| 89 | |
| 90 | pub fn getType(local: Local) LocalType { |
| 91 | return .{ .cty_idx = local.cty_idx, .alignas = local.alignas }; |
| 92 | } |
| 89 | 93 | }; |
| 90 | 94 | |
| 91 | 95 | const LocalIndex = u16; |
| 92 | | const LocalsList = std.ArrayListUnmanaged(LocalIndex); |
| 93 | | const LocalsMap = std.ArrayHashMapUnmanaged(Type, LocalsList, Type.HashContext32, true); |
| 96 | const LocalType = struct { cty_idx: CType.Index, alignas: CType.AlignAs }; |
| 97 | const LocalsList = std.AutoArrayHashMapUnmanaged(LocalIndex, void); |
| 98 | const LocalsMap = std.AutoArrayHashMapUnmanaged(LocalType, LocalsList); |
| 94 | 99 | const LocalsStack = std.ArrayListUnmanaged(LocalsMap); |
| 95 | 100 | |
| 96 | 101 | const ValueRenderLocation = enum { |
| ... | ... | @@ -296,10 +301,6 @@ pub const Function = struct { |
| 296 | 301 | /// Needed for memory used by the keys of free_locals_stack entries. |
| 297 | 302 | arena: std.heap.ArenaAllocator, |
| 298 | 303 | |
| 299 | | fn tyHashCtx(f: Function) Type.HashContext32 { |
| 300 | | return .{ .mod = f.object.dg.module }; |
| 301 | | } |
| 302 | | |
| 303 | 304 | fn resolveInst(f: *Function, inst: Air.Inst.Ref) !CValue { |
| 304 | 305 | const gop = try f.value_map.getOrPut(inst); |
| 305 | 306 | if (gop.found_existing) return gop.value_ptr.*; |
| ... | ... | @@ -339,10 +340,11 @@ pub const Function = struct { |
| 339 | 340 | /// Skips the reuse logic. |
| 340 | 341 | fn allocLocalValue(f: *Function, ty: Type, alignment: u32) !CValue { |
| 341 | 342 | const gpa = f.object.dg.gpa; |
| 343 | const target = f.object.dg.module.getTarget(); |
| 342 | 344 | try f.locals.append(gpa, .{ |
| 343 | | .ty = ty, |
| 344 | | .alignment = alignment, |
| 345 | .cty_idx = try f.typeToIndex(ty, .complete), |
| 345 | 346 | .loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1), |
| 347 | .alignas = CType.AlignAs.init(alignment, ty.abiAlignment(target)), |
| 346 | 348 | }); |
| 347 | 349 | return .{ .new_local = @intCast(LocalIndex, f.locals.items.len - 1) }; |
| 348 | 350 | } |
| ... | ... | @@ -355,14 +357,15 @@ pub const Function = struct { |
| 355 | 357 | |
| 356 | 358 | /// Only allocates the local; does not print anything. |
| 357 | 359 | fn allocAlignedLocal(f: *Function, ty: Type, _: CQualifiers, alignment: u32) !CValue { |
| 358 | | if (f.getFreeLocals().getPtrContext(ty, f.tyHashCtx())) |locals_list| { |
| 359 | | for (locals_list.items, 0..) |local_index, i| { |
| 360 | | const local = &f.locals.items[local_index]; |
| 361 | | if (local.alignment >= alignment) { |
| 362 | | local.loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1); |
| 363 | | _ = locals_list.swapRemove(i); |
| 364 | | return .{ .new_local = local_index }; |
| 365 | | } |
| 360 | const target = f.object.dg.module.getTarget(); |
| 361 | if (f.getFreeLocals().getPtr(.{ |
| 362 | .cty_idx = try f.typeToIndex(ty, .complete), |
| 363 | .alignas = CType.AlignAs.init(alignment, ty.abiAlignment(target)), |
| 364 | })) |locals_list| { |
| 365 | if (locals_list.popOrNull()) |local_entry| { |
| 366 | const local = &f.locals.items[local_entry.key]; |
| 367 | local.loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1); |
| 368 | return .{ .new_local = local_entry.key }; |
| 366 | 369 | } |
| 367 | 370 | } |
| 368 | 371 | |
| ... | ... | @@ -1695,22 +1698,34 @@ pub const DeclGen = struct { |
| 1695 | 1698 | qualifiers: CQualifiers, |
| 1696 | 1699 | alignment: u32, |
| 1697 | 1700 | kind: CType.Kind, |
| 1701 | ) error{ OutOfMemory, AnalysisFail }!void { |
| 1702 | const target = dg.module.getTarget(); |
| 1703 | const alignas = CType.AlignAs.init(alignment, ty.abiAlignment(target)); |
| 1704 | try dg.renderCTypeAndName(w, try dg.typeToIndex(ty, kind), name, qualifiers, alignas); |
| 1705 | } |
| 1706 | |
| 1707 | fn renderCTypeAndName( |
| 1708 | dg: *DeclGen, |
| 1709 | w: anytype, |
| 1710 | cty_idx: CType.Index, |
| 1711 | name: CValue, |
| 1712 | qualifiers: CQualifiers, |
| 1713 | alignas: CType.AlignAs, |
| 1698 | 1714 | ) error{ OutOfMemory, AnalysisFail }!void { |
| 1699 | 1715 | const store = &dg.ctypes.set; |
| 1700 | 1716 | const module = dg.module; |
| 1701 | 1717 | |
| 1702 | | if (alignment != 0) switch (std.math.order(alignment, ty.abiAlignment(dg.module.getTarget()))) { |
| 1703 | | .lt => try w.print("zig_under_align({}) ", .{alignment}), |
| 1718 | switch (std.math.order(alignas.@"align", alignas.abi)) { |
| 1719 | .lt => try w.print("zig_under_align({}) ", .{alignas.getAlign()}), |
| 1704 | 1720 | .eq => {}, |
| 1705 | | .gt => try w.print("zig_align({}) ", .{alignment}), |
| 1706 | | }; |
| 1721 | .gt => try w.print("zig_align({}) ", .{alignas.getAlign()}), |
| 1722 | } |
| 1707 | 1723 | |
| 1708 | | const idx = try dg.typeToIndex(ty, kind); |
| 1709 | 1724 | const trailing = |
| 1710 | | try renderTypePrefix(dg.decl_index, store.*, module, w, idx, .suffix, qualifiers); |
| 1725 | try renderTypePrefix(dg.decl_index, store.*, module, w, cty_idx, .suffix, qualifiers); |
| 1711 | 1726 | try w.print("{}", .{trailing}); |
| 1712 | 1727 | try dg.writeCValue(w, name); |
| 1713 | | try renderTypeSuffix(dg.decl_index, store.*, module, w, idx, .suffix, .{}); |
| 1728 | try renderTypeSuffix(dg.decl_index, store.*, module, w, cty_idx, .suffix, .{}); |
| 1714 | 1729 | } |
| 1715 | 1730 | |
| 1716 | 1731 | fn declIsGlobal(dg: *DeclGen, tv: TypedValue) bool { |
| ... | ... | @@ -2589,36 +2604,27 @@ pub fn genFunc(f: *Function) !void { |
| 2589 | 2604 | if (value) continue; // static |
| 2590 | 2605 | const local = f.locals.items[local_index]; |
| 2591 | 2606 | log.debug("inserting local {d} into free_locals", .{local_index}); |
| 2592 | | const gop = try free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx()); |
| 2607 | const gop = try free_locals.getOrPut(gpa, local.getType()); |
| 2593 | 2608 | if (!gop.found_existing) gop.value_ptr.* = .{}; |
| 2594 | | try gop.value_ptr.append(gpa, local_index); |
| 2609 | try gop.value_ptr.putNoClobber(gpa, local_index, {}); |
| 2595 | 2610 | } |
| 2596 | 2611 | |
| 2597 | 2612 | const SortContext = struct { |
| 2598 | | target: std.Target, |
| 2599 | | keys: []const Type, |
| 2613 | keys: []const LocalType, |
| 2600 | 2614 | |
| 2601 | | pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { |
| 2602 | | const a_ty = ctx.keys[a_index]; |
| 2603 | | const b_ty = ctx.keys[b_index]; |
| 2604 | | return b_ty.abiAlignment(ctx.target) < a_ty.abiAlignment(ctx.target); |
| 2615 | pub fn lessThan(ctx: @This(), lhs_index: usize, rhs_index: usize) bool { |
| 2616 | const lhs_ty = ctx.keys[lhs_index]; |
| 2617 | const rhs_ty = ctx.keys[rhs_index]; |
| 2618 | return lhs_ty.alignas.getAlign() > rhs_ty.alignas.getAlign(); |
| 2605 | 2619 | } |
| 2606 | 2620 | }; |
| 2607 | | const target = o.dg.module.getTarget(); |
| 2608 | | free_locals.sort(SortContext{ .target = target, .keys = free_locals.keys() }); |
| 2621 | free_locals.sort(SortContext{ .keys = free_locals.keys() }); |
| 2609 | 2622 | |
| 2610 | 2623 | const w = o.code_header.writer(); |
| 2611 | 2624 | for (free_locals.values()) |list| { |
| 2612 | | for (list.items) |local_index| { |
| 2625 | for (list.keys()) |local_index| { |
| 2613 | 2626 | const local = f.locals.items[local_index]; |
| 2614 | | try o.dg.renderTypeAndName( |
| 2615 | | w, |
| 2616 | | local.ty, |
| 2617 | | .{ .local = local_index }, |
| 2618 | | .{}, |
| 2619 | | local.alignment, |
| 2620 | | .complete, |
| 2621 | | ); |
| 2627 | try o.dg.renderCTypeAndName(w, local.cty_idx, .{ .local = local_index }, .{}, local.alignas); |
| 2622 | 2628 | try w.writeAll(";\n "); |
| 2623 | 2629 | } |
| 2624 | 2630 | } |
| ... | ... | @@ -4486,13 +4492,13 @@ fn airLoop(f: *Function, inst: Air.Inst.Index) !CValue { |
| 4486 | 4492 | const new_free_locals = f.getFreeLocals(); |
| 4487 | 4493 | var it = new_free_locals.iterator(); |
| 4488 | 4494 | while (it.next()) |entry| { |
| 4489 | | const gop = try old_free_locals.getOrPutContext(gpa, entry.key_ptr.*, f.tyHashCtx()); |
| 4495 | const gop = try old_free_locals.getOrPut(gpa, entry.key_ptr.*); |
| 4490 | 4496 | if (gop.found_existing) { |
| 4491 | | try gop.value_ptr.appendSlice(gpa, entry.value_ptr.items); |
| 4492 | | } else { |
| 4493 | | gop.value_ptr.* = entry.value_ptr.*; |
| 4494 | | entry.value_ptr.* = .{}; |
| 4495 | | } |
| 4497 | try gop.value_ptr.ensureUnusedCapacity(gpa, entry.value_ptr.count()); |
| 4498 | for (entry.value_ptr.keys()) |local_index| { |
| 4499 | gop.value_ptr.putAssumeCapacityNoClobber(local_index, {}); |
| 4500 | } |
| 4501 | } else gop.value_ptr.* = entry.value_ptr.move(); |
| 4496 | 4502 | } |
| 4497 | 4503 | deinitFreeLocalsMap(gpa, new_free_locals); |
| 4498 | 4504 | new_free_locals.* = old_free_locals.move(); |
| ... | ... | @@ -4522,6 +4528,10 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { |
| 4522 | 4528 | // that we can notice and use them in the else branch. Any new locals must |
| 4523 | 4529 | // necessarily be free already after the then branch is complete. |
| 4524 | 4530 | const pre_locals_len = @intCast(LocalIndex, f.locals.items.len); |
| 4531 | // Remember how many allocs there were before entering the then branch so |
| 4532 | // that we can notice and make sure not to use them in the else branch. |
| 4533 | // Any new allocs must be removed from the free list. |
| 4534 | const pre_allocs_len = @intCast(LocalIndex, f.allocs.count()); |
| 4525 | 4535 | const pre_clone_depth = f.free_locals_clone_depth; |
| 4526 | 4536 | f.free_locals_clone_depth = @intCast(LoopDepth, f.free_locals_stack.items.len); |
| 4527 | 4537 | |
| ... | ... | @@ -4552,7 +4562,7 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { |
| 4552 | 4562 | try die(f, inst, Air.indexToRef(operand)); |
| 4553 | 4563 | } |
| 4554 | 4564 | |
| 4555 | | try noticeBranchFrees(f, pre_locals_len, inst); |
| 4565 | try noticeBranchFrees(f, pre_locals_len, pre_allocs_len, inst); |
| 4556 | 4566 | |
| 4557 | 4567 | if (needs_else) { |
| 4558 | 4568 | try genBody(f, else_body); |
| ... | ... | @@ -4627,6 +4637,10 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { |
| 4627 | 4637 | // we can notice and use them in subsequent branches. Any new locals must |
| 4628 | 4638 | // necessarily be free already after the previous branch is complete. |
| 4629 | 4639 | const pre_locals_len = @intCast(LocalIndex, f.locals.items.len); |
| 4640 | // Remember how many allocs there were before entering each branch so that |
| 4641 | // we can notice and make sure not to use them in subsequent branches. |
| 4642 | // Any new allocs must be removed from the free list. |
| 4643 | const pre_allocs_len = @intCast(LocalIndex, f.allocs.count()); |
| 4630 | 4644 | const pre_clone_depth = f.free_locals_clone_depth; |
| 4631 | 4645 | f.free_locals_clone_depth = @intCast(LoopDepth, f.free_locals_stack.items.len); |
| 4632 | 4646 | |
| ... | ... | @@ -4647,7 +4661,7 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { |
| 4647 | 4661 | try genBody(f, case_body); |
| 4648 | 4662 | } |
| 4649 | 4663 | |
| 4650 | | try noticeBranchFrees(f, pre_locals_len, inst); |
| 4664 | try noticeBranchFrees(f, pre_locals_len, pre_allocs_len, inst); |
| 4651 | 4665 | } else { |
| 4652 | 4666 | for (liveness.deaths[case_i]) |operand| { |
| 4653 | 4667 | try die(f, inst, Air.indexToRef(operand)); |
| ... | ... | @@ -7441,21 +7455,16 @@ fn freeLocal(f: *Function, inst: Air.Inst.Index, local_index: LocalIndex, ref_in |
| 7441 | 7455 | const local = &f.locals.items[local_index]; |
| 7442 | 7456 | log.debug("%{d}: freeing t{d} (operand %{d})", .{ inst, local_index, ref_inst }); |
| 7443 | 7457 | if (local.loop_depth < f.free_locals_clone_depth) return; |
| 7444 | | const gop = try f.free_locals_stack.items[local.loop_depth].getOrPutContext( |
| 7445 | | gpa, |
| 7446 | | local.ty, |
| 7447 | | f.tyHashCtx(), |
| 7448 | | ); |
| 7458 | const gop = try f.free_locals_stack.items[local.loop_depth].getOrPut(gpa, local.getType()); |
| 7449 | 7459 | if (!gop.found_existing) gop.value_ptr.* = .{}; |
| 7450 | 7460 | if (std.debug.runtime_safety) { |
| 7451 | | // If this trips, it means a local is being inserted into the |
| 7452 | | // free_locals map while it already exists in the map, which is not |
| 7453 | | // allowed. |
| 7454 | | assert(mem.indexOfScalar(LocalIndex, gop.value_ptr.items, local_index) == null); |
| 7455 | 7461 | // If this trips, an unfreeable allocation was attempted to be freed. |
| 7456 | 7462 | assert(!f.allocs.contains(local_index)); |
| 7457 | 7463 | } |
| 7458 | | try gop.value_ptr.append(gpa, local_index); |
| 7464 | // If this trips, it means a local is being inserted into the |
| 7465 | // free_locals map while it already exists in the map, which is not |
| 7466 | // allowed. |
| 7467 | try gop.value_ptr.putNoClobber(gpa, local_index, {}); |
| 7459 | 7468 | } |
| 7460 | 7469 | |
| 7461 | 7470 | const BigTomb = struct { |
| ... | ... | @@ -7504,14 +7513,36 @@ fn deinitFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) void { |
| 7504 | 7513 | map.deinit(gpa); |
| 7505 | 7514 | } |
| 7506 | 7515 | |
| 7507 | | fn noticeBranchFrees(f: *Function, pre_locals_len: LocalIndex, inst: Air.Inst.Index) !void { |
| 7516 | fn noticeBranchFrees( |
| 7517 | f: *Function, |
| 7518 | pre_locals_len: LocalIndex, |
| 7519 | pre_allocs_len: LocalIndex, |
| 7520 | inst: Air.Inst.Index, |
| 7521 | ) !void { |
| 7522 | const free_locals = f.getFreeLocals(); |
| 7523 | |
| 7508 | 7524 | for (f.locals.items[pre_locals_len..], pre_locals_len..) |*local, local_i| { |
| 7509 | 7525 | const local_index = @intCast(LocalIndex, local_i); |
| 7510 | | if (f.allocs.contains(local_index)) continue; // allocs are not freeable |
| 7526 | if (f.allocs.contains(local_index)) { |
| 7527 | if (std.debug.runtime_safety) { |
| 7528 | // new allocs are no longer freeable, so make sure they aren't in the free list |
| 7529 | if (free_locals.getPtr(local.getType())) |locals_list| { |
| 7530 | assert(!locals_list.contains(local_index)); |
| 7531 | } |
| 7532 | } |
| 7533 | continue; |
| 7534 | } |
| 7511 | 7535 | |
| 7512 | 7536 | // free more deeply nested locals from other branches at current depth |
| 7513 | 7537 | assert(local.loop_depth >= f.free_locals_stack.items.len - 1); |
| 7514 | 7538 | local.loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1); |
| 7515 | 7539 | try freeLocal(f, inst, local_index, 0); |
| 7516 | 7540 | } |
| 7541 | |
| 7542 | for (f.allocs.keys()[pre_allocs_len..]) |local_i| { |
| 7543 | const local_index = @intCast(LocalIndex, local_i); |
| 7544 | const local = &f.locals.items[local_index]; |
| 7545 | // new allocs are no longer freeable, so remove them from the free list |
| 7546 | if (free_locals.getPtr(local.getType())) |locals_list| _ = locals_list.swapRemove(local_index); |
| 7547 | } |
| 7517 | 7548 | } |