authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-07 11:12:44-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-04-07 11:12:44-04:00
log48f98494fda57c5eca7b4ac899046c1dd285e471
treefd96476ec302a30c51f24ac91fc5ea5e2b6637d9
parent086639630800fc52bd727163e85d174ec1ac1103
parenta7f674d6c1ba3d72a324aa918929c5beb36a8306
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15195 from mlugg/fix/liveness-loop-defer-deaths

Liveness: defer deaths of externally-scoped instructions in loop bodies

11 files changed, 806 insertions(+), 139 deletions(-)

src/Liveness.zig+619-1
...@@ -25,6 +25,7 @@ tomb_bits: []usize,...@@ -25,6 +25,7 @@ tomb_bits: []usize,
25/// array. The meaning of the data depends on the AIR tag.25/// array. The meaning of the data depends on the AIR tag.
26/// * `cond_br` - points to a `CondBr` in `extra` at this index.26/// * `cond_br` - points to a `CondBr` in `extra` at this index.
27/// * `switch_br` - points to a `SwitchBr` in `extra` at this index.27/// * `switch_br` - points to a `SwitchBr` in `extra` at this index.
28/// * `loop` - points to a `Loop` in `extra` at this index.
28/// * `asm`, `call`, `aggregate_init` - the value is a set of bits which are the extra tomb29/// * `asm`, `call`, `aggregate_init` - the value is a set of bits which are the extra tomb
29/// bits of operands.30/// bits of operands.
30/// The main tomb bits are still used and the extra ones are starting with the lsb of the31/// The main tomb bits are still used and the extra ones are starting with the lsb of the
...@@ -51,6 +52,11 @@ pub const SwitchBr = struct {...@@ -51,6 +52,11 @@ pub const SwitchBr = struct {
51 else_death_count: u32,52 else_death_count: u32,
52};53};
5354
55/// Trailing is the set of instructions whose lifetimes end at the end of the loop body.
56pub const Loop = struct {
57 death_count: u32,
58};
59
54pub fn analyze(gpa: Allocator, air: Air) Allocator.Error!Liveness {60pub fn analyze(gpa: Allocator, air: Air) Allocator.Error!Liveness {
55 const tracy = trace(@src());61 const tracy = trace(@src());
56 defer tracy.end();62 defer tracy.end();
...@@ -76,6 +82,11 @@ pub fn analyze(gpa: Allocator, air: Air) Allocator.Error!Liveness {...@@ -76,6 +82,11 @@ pub fn analyze(gpa: Allocator, air: Air) Allocator.Error!Liveness {
76 const main_body = air.getMainBody();82 const main_body = air.getMainBody();
77 try a.table.ensureTotalCapacity(gpa, @intCast(u32, main_body.len));83 try a.table.ensureTotalCapacity(gpa, @intCast(u32, main_body.len));
78 try analyzeWithContext(&a, null, main_body);84 try analyzeWithContext(&a, null, main_body);
85 {
86 var to_remove: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .{};
87 defer to_remove.deinit(gpa);
88 try removeDeaths(&a, &to_remove, main_body);
89 }
79 return Liveness{90 return Liveness{
80 .tomb_bits = a.tomb_bits,91 .tomb_bits = a.tomb_bits,
81 .special = a.special,92 .special = a.special,
...@@ -650,6 +661,18 @@ pub fn getSwitchBr(l: Liveness, gpa: Allocator, inst: Air.Inst.Index, cases_len:...@@ -650,6 +661,18 @@ pub fn getSwitchBr(l: Liveness, gpa: Allocator, inst: Air.Inst.Index, cases_len:
650 };661 };
651}662}
652663
664pub const LoopSlice = struct {
665 deaths: []const Air.Inst.Index,
666};
667
668pub fn getLoop(l: Liveness, inst: Air.Inst.Index) LoopSlice {
669 const index: usize = l.special.get(inst) orelse return .{
670 .deaths = &.{},
671 };
672 const death_count = l.extra[index];
673 return .{ .deaths = l.extra[index + 1 ..][0..death_count] };
674}
675
653pub fn deinit(l: *Liveness, gpa: Allocator) void {676pub fn deinit(l: *Liveness, gpa: Allocator) void {
654 gpa.free(l.tomb_bits);677 gpa.free(l.tomb_bits);
655 gpa.free(l.extra);678 gpa.free(l.extra);
...@@ -1138,7 +1161,39 @@ fn analyzeInst(...@@ -1138,7 +1161,39 @@ fn analyzeInst(
1138 .loop => {1161 .loop => {
1139 const extra = a.air.extraData(Air.Block, inst_datas[inst].ty_pl.payload);1162 const extra = a.air.extraData(Air.Block, inst_datas[inst].ty_pl.payload);
1140 const body = a.air.extra[extra.end..][0..extra.data.body_len];1163 const body = a.air.extra[extra.end..][0..extra.data.body_len];
1141 try analyzeWithContext(a, new_set, body);1164
1165 var body_table: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .{};
1166 defer body_table.deinit(gpa);
1167
1168 // Instructions outside the loop body cannot die within the loop, since further loop
1169 // iterations may occur. Track deaths from the loop body - we'll remove all of these
1170 // retroactively, and add them to our extra data.
1171
1172 try analyzeWithContext(a, &body_table, body);
1173
1174 if (new_set) |ns| {
1175 try ns.ensureUnusedCapacity(gpa, body_table.count());
1176 var it = body_table.keyIterator();
1177 while (it.next()) |key| {
1178 _ = ns.putAssumeCapacity(key.*, {});
1179 }
1180 }
1181
1182 try a.extra.ensureUnusedCapacity(gpa, std.meta.fields(Loop).len + body_table.count());
1183 const extra_index = a.addExtraAssumeCapacity(Loop{
1184 .death_count = body_table.count(),
1185 });
1186 {
1187 var it = body_table.keyIterator();
1188 while (it.next()) |key| {
1189 a.extra.appendAssumeCapacity(key.*);
1190 }
1191 }
1192 try a.special.put(gpa, inst, extra_index);
1193
1194 // We'll remove invalid deaths in a separate pass after main liveness analysis. See
1195 // removeDeaths for more details.
1196
1142 return; // Loop has no operands and it is always unreferenced.1197 return; // Loop has no operands and it is always unreferenced.
1143 },1198 },
1144 .@"try" => {1199 .@"try" => {
...@@ -1412,3 +1467,566 @@ const ExtraTombs = struct {...@@ -1412,3 +1467,566 @@ const ExtraTombs = struct {
1412 et.big_tomb_bits_extra.deinit(et.analysis.gpa);1467 et.big_tomb_bits_extra.deinit(et.analysis.gpa);
1413 }1468 }
1414};1469};
1470
1471/// Remove any deaths invalidated by the deaths from an enclosing `loop`. Reshuffling deaths stored
1472/// in `extra` causes it to become non-dense, but that's fine - we won't remove too much data.
1473/// Making it dense would be a lot more work - it'd require recomputing every index in `special`.
1474fn removeDeaths(
1475 a: *Analysis,
1476 to_remove: *std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1477 body: []const Air.Inst.Index,
1478) error{OutOfMemory}!void {
1479 for (body) |inst| {
1480 try removeInstDeaths(a, to_remove, inst);
1481 }
1482}
1483
1484fn removeInstDeaths(
1485 a: *Analysis,
1486 to_remove: *std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1487 inst: Air.Inst.Index,
1488) !void {
1489 const inst_tags = a.air.instructions.items(.tag);
1490 const inst_datas = a.air.instructions.items(.data);
1491
1492 switch (inst_tags[inst]) {
1493 .add,
1494 .add_optimized,
1495 .addwrap,
1496 .addwrap_optimized,
1497 .add_sat,
1498 .sub,
1499 .sub_optimized,
1500 .subwrap,
1501 .subwrap_optimized,
1502 .sub_sat,
1503 .mul,
1504 .mul_optimized,
1505 .mulwrap,
1506 .mulwrap_optimized,
1507 .mul_sat,
1508 .div_float,
1509 .div_float_optimized,
1510 .div_trunc,
1511 .div_trunc_optimized,
1512 .div_floor,
1513 .div_floor_optimized,
1514 .div_exact,
1515 .div_exact_optimized,
1516 .rem,
1517 .rem_optimized,
1518 .mod,
1519 .mod_optimized,
1520 .bit_and,
1521 .bit_or,
1522 .xor,
1523 .cmp_lt,
1524 .cmp_lt_optimized,
1525 .cmp_lte,
1526 .cmp_lte_optimized,
1527 .cmp_eq,
1528 .cmp_eq_optimized,
1529 .cmp_gte,
1530 .cmp_gte_optimized,
1531 .cmp_gt,
1532 .cmp_gt_optimized,
1533 .cmp_neq,
1534 .cmp_neq_optimized,
1535 .bool_and,
1536 .bool_or,
1537 .store,
1538 .array_elem_val,
1539 .slice_elem_val,
1540 .ptr_elem_val,
1541 .shl,
1542 .shl_exact,
1543 .shl_sat,
1544 .shr,
1545 .shr_exact,
1546 .atomic_store_unordered,
1547 .atomic_store_monotonic,
1548 .atomic_store_release,
1549 .atomic_store_seq_cst,
1550 .set_union_tag,
1551 .min,
1552 .max,
1553 => {
1554 const o = inst_datas[inst].bin_op;
1555 removeOperandDeaths(a, to_remove, inst, .{ o.lhs, o.rhs, .none });
1556 },
1557
1558 .vector_store_elem => {
1559 const o = inst_datas[inst].vector_store_elem;
1560 const extra = a.air.extraData(Air.Bin, o.payload).data;
1561 removeOperandDeaths(a, to_remove, inst, .{ o.vector_ptr, extra.lhs, extra.rhs });
1562 },
1563
1564 .arg,
1565 .alloc,
1566 .ret_ptr,
1567 .constant,
1568 .const_ty,
1569 .trap,
1570 .breakpoint,
1571 .dbg_stmt,
1572 .dbg_inline_begin,
1573 .dbg_inline_end,
1574 .dbg_block_begin,
1575 .dbg_block_end,
1576 .unreach,
1577 .fence,
1578 .ret_addr,
1579 .frame_addr,
1580 .wasm_memory_size,
1581 .err_return_trace,
1582 .save_err_return_trace_index,
1583 .c_va_start,
1584 .work_item_id,
1585 .work_group_size,
1586 .work_group_id,
1587 => {},
1588
1589 .not,
1590 .bitcast,
1591 .load,
1592 .fpext,
1593 .fptrunc,
1594 .intcast,
1595 .trunc,
1596 .optional_payload,
1597 .optional_payload_ptr,
1598 .optional_payload_ptr_set,
1599 .errunion_payload_ptr_set,
1600 .wrap_optional,
1601 .unwrap_errunion_payload,
1602 .unwrap_errunion_err,
1603 .unwrap_errunion_payload_ptr,
1604 .unwrap_errunion_err_ptr,
1605 .wrap_errunion_payload,
1606 .wrap_errunion_err,
1607 .slice_ptr,
1608 .slice_len,
1609 .ptr_slice_len_ptr,
1610 .ptr_slice_ptr_ptr,
1611 .struct_field_ptr_index_0,
1612 .struct_field_ptr_index_1,
1613 .struct_field_ptr_index_2,
1614 .struct_field_ptr_index_3,
1615 .array_to_slice,
1616 .float_to_int,
1617 .float_to_int_optimized,
1618 .int_to_float,
1619 .get_union_tag,
1620 .clz,
1621 .ctz,
1622 .popcount,
1623 .byte_swap,
1624 .bit_reverse,
1625 .splat,
1626 .error_set_has_value,
1627 .addrspace_cast,
1628 .c_va_arg,
1629 .c_va_copy,
1630 => {
1631 const o = inst_datas[inst].ty_op;
1632 removeOperandDeaths(a, to_remove, inst, .{ o.operand, .none, .none });
1633 },
1634
1635 .is_null,
1636 .is_non_null,
1637 .is_null_ptr,
1638 .is_non_null_ptr,
1639 .is_err,
1640 .is_non_err,
1641 .is_err_ptr,
1642 .is_non_err_ptr,
1643 .ptrtoint,
1644 .bool_to_int,
1645 .ret,
1646 .ret_load,
1647 .is_named_enum_value,
1648 .tag_name,
1649 .error_name,
1650 .sqrt,
1651 .sin,
1652 .cos,
1653 .tan,
1654 .exp,
1655 .exp2,
1656 .log,
1657 .log2,
1658 .log10,
1659 .fabs,
1660 .floor,
1661 .ceil,
1662 .round,
1663 .trunc_float,
1664 .neg,
1665 .neg_optimized,
1666 .cmp_lt_errors_len,
1667 .set_err_return_trace,
1668 .c_va_end,
1669 => {
1670 const operand = inst_datas[inst].un_op;
1671 removeOperandDeaths(a, to_remove, inst, .{ operand, .none, .none });
1672 },
1673
1674 .add_with_overflow,
1675 .sub_with_overflow,
1676 .mul_with_overflow,
1677 .shl_with_overflow,
1678 .ptr_add,
1679 .ptr_sub,
1680 .ptr_elem_ptr,
1681 .slice_elem_ptr,
1682 .slice,
1683 => {
1684 const ty_pl = inst_datas[inst].ty_pl;
1685 const extra = a.air.extraData(Air.Bin, ty_pl.payload).data;
1686 removeOperandDeaths(a, to_remove, inst, .{ extra.lhs, extra.rhs, .none });
1687 },
1688
1689 .dbg_var_ptr,
1690 .dbg_var_val,
1691 => {
1692 const operand = inst_datas[inst].pl_op.operand;
1693 removeOperandDeaths(a, to_remove, inst, .{ operand, .none, .none });
1694 },
1695
1696 .prefetch => {
1697 const prefetch = inst_datas[inst].prefetch;
1698 removeOperandDeaths(a, to_remove, inst, .{ prefetch.ptr, .none, .none });
1699 },
1700
1701 .call, .call_always_tail, .call_never_tail, .call_never_inline => {
1702 const inst_data = inst_datas[inst].pl_op;
1703 const callee = inst_data.operand;
1704 const extra = a.air.extraData(Air.Call, inst_data.payload);
1705 const args = @ptrCast([]const Air.Inst.Ref, a.air.extra[extra.end..][0..extra.data.args_len]);
1706
1707 var death_remover = BigTombDeathRemover.init(a, to_remove, inst);
1708 death_remover.feed(callee);
1709 for (args) |operand| {
1710 death_remover.feed(operand);
1711 }
1712 death_remover.finish();
1713 },
1714 .select => {
1715 const pl_op = inst_datas[inst].pl_op;
1716 const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1717 removeOperandDeaths(a, to_remove, inst, .{ pl_op.operand, extra.lhs, extra.rhs });
1718 },
1719 .shuffle => {
1720 const extra = a.air.extraData(Air.Shuffle, inst_datas[inst].ty_pl.payload).data;
1721 removeOperandDeaths(a, to_remove, inst, .{ extra.a, extra.b, .none });
1722 },
1723 .reduce, .reduce_optimized => {
1724 const reduce = inst_datas[inst].reduce;
1725 removeOperandDeaths(a, to_remove, inst, .{ reduce.operand, .none, .none });
1726 },
1727 .cmp_vector, .cmp_vector_optimized => {
1728 const extra = a.air.extraData(Air.VectorCmp, inst_datas[inst].ty_pl.payload).data;
1729 removeOperandDeaths(a, to_remove, inst, .{ extra.lhs, extra.rhs, .none });
1730 },
1731 .aggregate_init => {
1732 const ty_pl = inst_datas[inst].ty_pl;
1733 const aggregate_ty = a.air.getRefType(ty_pl.ty);
1734 const len = @intCast(usize, aggregate_ty.arrayLen());
1735 const elements = @ptrCast([]const Air.Inst.Ref, a.air.extra[ty_pl.payload..][0..len]);
1736
1737 var death_remover = BigTombDeathRemover.init(a, to_remove, inst);
1738 for (elements) |elem| {
1739 death_remover.feed(elem);
1740 }
1741 death_remover.finish();
1742 },
1743 .union_init => {
1744 const extra = a.air.extraData(Air.UnionInit, inst_datas[inst].ty_pl.payload).data;
1745 removeOperandDeaths(a, to_remove, inst, .{ extra.init, .none, .none });
1746 },
1747 .struct_field_ptr, .struct_field_val => {
1748 const extra = a.air.extraData(Air.StructField, inst_datas[inst].ty_pl.payload).data;
1749 removeOperandDeaths(a, to_remove, inst, .{ extra.struct_operand, .none, .none });
1750 },
1751 .field_parent_ptr => {
1752 const extra = a.air.extraData(Air.FieldParentPtr, inst_datas[inst].ty_pl.payload).data;
1753 removeOperandDeaths(a, to_remove, inst, .{ extra.field_ptr, .none, .none });
1754 },
1755 .cmpxchg_strong, .cmpxchg_weak => {
1756 const extra = a.air.extraData(Air.Cmpxchg, inst_datas[inst].ty_pl.payload).data;
1757 removeOperandDeaths(a, to_remove, inst, .{ extra.ptr, extra.expected_value, extra.new_value });
1758 },
1759 .mul_add => {
1760 const pl_op = inst_datas[inst].pl_op;
1761 const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1762 removeOperandDeaths(a, to_remove, inst, .{ extra.lhs, extra.rhs, pl_op.operand });
1763 },
1764 .atomic_load => {
1765 const ptr = inst_datas[inst].atomic_load.ptr;
1766 removeOperandDeaths(a, to_remove, inst, .{ ptr, .none, .none });
1767 },
1768 .atomic_rmw => {
1769 const pl_op = inst_datas[inst].pl_op;
1770 const extra = a.air.extraData(Air.AtomicRmw, pl_op.payload).data;
1771 removeOperandDeaths(a, to_remove, inst, .{ pl_op.operand, extra.operand, .none });
1772 },
1773 .memset,
1774 .memcpy,
1775 => {
1776 const pl_op = inst_datas[inst].pl_op;
1777 const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1778 removeOperandDeaths(a, to_remove, inst, .{ pl_op.operand, extra.lhs, extra.rhs });
1779 },
1780
1781 .br => {
1782 const br = inst_datas[inst].br;
1783 removeOperandDeaths(a, to_remove, inst, .{ br.operand, .none, .none });
1784 },
1785 .assembly => {
1786 const extra = a.air.extraData(Air.Asm, inst_datas[inst].ty_pl.payload);
1787 var extra_i: usize = extra.end;
1788 const outputs = @ptrCast([]const Air.Inst.Ref, a.air.extra[extra_i..][0..extra.data.outputs_len]);
1789 extra_i += outputs.len;
1790 const inputs = @ptrCast([]const Air.Inst.Ref, a.air.extra[extra_i..][0..extra.data.inputs_len]);
1791 extra_i += inputs.len;
1792
1793 var death_remover = BigTombDeathRemover.init(a, to_remove, inst);
1794 for (outputs) |output| {
1795 if (output != .none) {
1796 death_remover.feed(output);
1797 }
1798 }
1799 for (inputs) |input| {
1800 death_remover.feed(input);
1801 }
1802 death_remover.finish();
1803 },
1804 .block => {
1805 const extra = a.air.extraData(Air.Block, inst_datas[inst].ty_pl.payload);
1806 const body = a.air.extra[extra.end..][0..extra.data.body_len];
1807 try removeDeaths(a, to_remove, body);
1808 },
1809 .loop => {
1810 const extra = a.air.extraData(Air.Block, inst_datas[inst].ty_pl.payload);
1811 const body = a.air.extra[extra.end..][0..extra.data.body_len];
1812
1813 const liveness_extra_idx = a.special.get(inst) orelse {
1814 try removeDeaths(a, to_remove, body);
1815 return;
1816 };
1817
1818 const death_count = a.extra.items[liveness_extra_idx];
1819 var deaths = a.extra.items[liveness_extra_idx + 1 ..][0..death_count];
1820
1821 // Remove any deaths in `to_remove` from this loop's deaths
1822 deaths.len = removeExtraDeaths(to_remove, deaths);
1823 a.extra.items[liveness_extra_idx] = @intCast(u32, deaths.len);
1824
1825 // Temporarily add any deaths of ours to `to_remove`
1826 try to_remove.ensureUnusedCapacity(a.gpa, @intCast(u32, deaths.len));
1827 for (deaths) |d| {
1828 to_remove.putAssumeCapacity(d, {});
1829 }
1830 try removeDeaths(a, to_remove, body);
1831 for (deaths) |d| {
1832 _ = to_remove.remove(d);
1833 }
1834 },
1835 .@"try" => {
1836 const pl_op = inst_datas[inst].pl_op;
1837 const extra = a.air.extraData(Air.Try, pl_op.payload);
1838 const body = a.air.extra[extra.end..][0..extra.data.body_len];
1839 try removeDeaths(a, to_remove, body);
1840 removeOperandDeaths(a, to_remove, inst, .{ pl_op.operand, .none, .none });
1841 },
1842 .try_ptr => {
1843 const extra = a.air.extraData(Air.TryPtr, inst_datas[inst].ty_pl.payload);
1844 const body = a.air.extra[extra.end..][0..extra.data.body_len];
1845 try removeDeaths(a, to_remove, body);
1846 removeOperandDeaths(a, to_remove, inst, .{ extra.data.ptr, .none, .none });
1847 },
1848 .cond_br => {
1849 const inst_data = inst_datas[inst].pl_op;
1850 const condition = inst_data.operand;
1851 const extra = a.air.extraData(Air.CondBr, inst_data.payload);
1852 const then_body = a.air.extra[extra.end..][0..extra.data.then_body_len];
1853 const else_body = a.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len];
1854
1855 if (a.special.get(inst)) |liveness_extra_idx| {
1856 const then_death_count = a.extra.items[liveness_extra_idx + 0];
1857 const else_death_count = a.extra.items[liveness_extra_idx + 1];
1858 var then_deaths = a.extra.items[liveness_extra_idx + 2 ..][0..then_death_count];
1859 var else_deaths = a.extra.items[liveness_extra_idx + 2 + then_death_count ..][0..else_death_count];
1860
1861 const new_then_death_count = removeExtraDeaths(to_remove, then_deaths);
1862 const new_else_death_count = removeExtraDeaths(to_remove, else_deaths);
1863
1864 a.extra.items[liveness_extra_idx + 0] = new_then_death_count;
1865 a.extra.items[liveness_extra_idx + 1] = new_else_death_count;
1866
1867 if (new_then_death_count < then_death_count) {
1868 // `else` deaths need to be moved earlier in `extra`
1869 const src = a.extra.items[liveness_extra_idx + 2 + then_death_count ..];
1870 const dest = a.extra.items[liveness_extra_idx + 2 + new_then_death_count ..];
1871 std.mem.copy(u32, dest, src[0..new_else_death_count]);
1872 }
1873 }
1874
1875 try removeDeaths(a, to_remove, then_body);
1876 try removeDeaths(a, to_remove, else_body);
1877
1878 removeOperandDeaths(a, to_remove, inst, .{ condition, .none, .none });
1879 },
1880 .switch_br => {
1881 const pl_op = inst_datas[inst].pl_op;
1882 const condition = pl_op.operand;
1883 const switch_br = a.air.extraData(Air.SwitchBr, pl_op.payload);
1884
1885 var air_extra_index: usize = switch_br.end;
1886 for (0..switch_br.data.cases_len) |_| {
1887 const case = a.air.extraData(Air.SwitchBr.Case, air_extra_index);
1888 const case_body = a.air.extra[case.end + case.data.items_len ..][0..case.data.body_len];
1889 air_extra_index = case.end + case.data.items_len + case_body.len;
1890 try removeDeaths(a, to_remove, case_body);
1891 }
1892 { // else
1893 const else_body = a.air.extra[air_extra_index..][0..switch_br.data.else_body_len];
1894 try removeDeaths(a, to_remove, else_body);
1895 }
1896
1897 if (a.special.get(inst)) |liveness_extra_idx| {
1898 const else_death_count = a.extra.items[liveness_extra_idx];
1899 var read_idx = liveness_extra_idx + 1;
1900 var write_idx = read_idx; // write_idx <= read_idx always
1901 for (0..switch_br.data.cases_len) |_| {
1902 const case_death_count = a.extra.items[read_idx];
1903 const case_deaths = a.extra.items[read_idx + 1 ..][0..case_death_count];
1904 const new_death_count = removeExtraDeaths(to_remove, case_deaths);
1905 a.extra.items[write_idx] = new_death_count;
1906 if (write_idx < read_idx) {
1907 std.mem.copy(u32, a.extra.items[write_idx + 1 ..], a.extra.items[read_idx + 1 ..][0..new_death_count]);
1908 }
1909 read_idx += 1 + case_death_count;
1910 write_idx += 1 + new_death_count;
1911 }
1912 const else_deaths = a.extra.items[read_idx..][0..else_death_count];
1913 const new_else_death_count = removeExtraDeaths(to_remove, else_deaths);
1914 a.extra.items[liveness_extra_idx] = new_else_death_count;
1915 if (write_idx < read_idx) {
1916 std.mem.copy(u32, a.extra.items[write_idx..], a.extra.items[read_idx..][0..new_else_death_count]);
1917 }
1918 }
1919
1920 removeOperandDeaths(a, to_remove, inst, .{ condition, .none, .none });
1921 },
1922 .wasm_memory_grow => {
1923 const pl_op = inst_datas[inst].pl_op;
1924 removeOperandDeaths(a, to_remove, inst, .{ pl_op.operand, .none, .none });
1925 },
1926 }
1927}
1928
1929fn removeOperandDeaths(
1930 a: *Analysis,
1931 to_remove: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1932 inst: Air.Inst.Index,
1933 operands: [bpi - 1]Air.Inst.Ref,
1934) void {
1935 const usize_index = (inst * bpi) / @bitSizeOf(usize);
1936
1937 const cur_tomb = @truncate(Bpi, a.tomb_bits[usize_index] >>
1938 @intCast(Log2Int(usize), (inst % (@bitSizeOf(usize) / bpi)) * bpi));
1939
1940 var toggle_bits: Bpi = 0;
1941
1942 for (operands, 0..) |op_ref, i| {
1943 const mask = @as(Bpi, 1) << @intCast(OperandInt, i);
1944 const op_int = @enumToInt(op_ref);
1945 if (op_int < Air.Inst.Ref.typed_value_map.len) continue;
1946 const operand: Air.Inst.Index = op_int - @intCast(u32, Air.Inst.Ref.typed_value_map.len);
1947 if ((cur_tomb & mask) != 0 and to_remove.contains(operand)) {
1948 log.debug("remove death of %{} in %{}", .{ operand, inst });
1949 toggle_bits ^= mask;
1950 }
1951 }
1952
1953 a.tomb_bits[usize_index] ^= @as(usize, toggle_bits) <<
1954 @intCast(Log2Int(usize), (inst % (@bitSizeOf(usize) / bpi)) * bpi);
1955}
1956
1957fn removeExtraDeaths(
1958 to_remove: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1959 deaths: []Air.Inst.Index,
1960) u32 {
1961 var new_len = @intCast(u32, deaths.len);
1962 var i: usize = 0;
1963 while (i < new_len) {
1964 if (to_remove.contains(deaths[i])) {
1965 log.debug("remove extra death of %{}", .{deaths[i]});
1966 deaths[i] = deaths[new_len - 1];
1967 new_len -= 1;
1968 } else {
1969 i += 1;
1970 }
1971 }
1972 return new_len;
1973}
1974
1975const BigTombDeathRemover = struct {
1976 a: *Analysis,
1977 to_remove: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1978 inst: Air.Inst.Index,
1979
1980 operands: [bpi - 1]Air.Inst.Ref = .{.none} ** (bpi - 1),
1981 next_oper: OperandInt = 0,
1982
1983 bit_index: u32 = 0,
1984 // Initialized once we finish the small tomb operands: see `feed`
1985 extra_start: u32 = undefined,
1986 extra_offset: u32 = 0,
1987
1988 fn init(a: *Analysis, to_remove: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void), inst: Air.Inst.Index) BigTombDeathRemover {
1989 return .{
1990 .a = a,
1991 .to_remove = to_remove,
1992 .inst = inst,
1993 };
1994 }
1995
1996 fn feed(dr: *BigTombDeathRemover, operand: Air.Inst.Ref) void {
1997 if (dr.next_oper < bpi - 1) {
1998 dr.operands[dr.next_oper] = operand;
1999 dr.next_oper += 1;
2000 if (dr.next_oper == bpi - 1) {
2001 removeOperandDeaths(dr.a, dr.to_remove, dr.inst, dr.operands);
2002 if (dr.a.special.get(dr.inst)) |idx| dr.extra_start = idx;
2003 }
2004 return;
2005 }
2006
2007 defer dr.bit_index += 1;
2008
2009 const op_int = @enumToInt(operand);
2010 if (op_int < Air.Inst.Ref.typed_value_map.len) return;
2011
2012 const op_inst: Air.Inst.Index = op_int - @intCast(u32, Air.Inst.Ref.typed_value_map.len);
2013
2014 while (dr.bit_index - dr.extra_offset * 31 >= 31) {
2015 dr.extra_offset += 1;
2016 }
2017 const dies = @truncate(u1, dr.a.extra.items[dr.extra_start + dr.extra_offset] >>
2018 @intCast(u5, dr.bit_index - dr.extra_offset * 31)) != 0;
2019
2020 if (dies and dr.to_remove.contains(op_inst)) {
2021 log.debug("remove big death of %{}", .{op_inst});
2022 dr.a.extra.items[dr.extra_start + dr.extra_offset] ^=
2023 (@as(u32, 1) << @intCast(u5, dr.bit_index - dr.extra_offset * 31));
2024 }
2025 }
2026
2027 fn finish(dr: *BigTombDeathRemover) void {
2028 if (dr.next_oper < bpi) {
2029 removeOperandDeaths(dr.a, dr.to_remove, dr.inst, dr.operands);
2030 }
2031 }
2032};
src/arch/aarch64/CodeGen.zig+8
...@@ -5000,9 +5000,17 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {...@@ -5000,9 +5000,17 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
5000 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;5000 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
5001 const loop = self.air.extraData(Air.Block, ty_pl.payload);5001 const loop = self.air.extraData(Air.Block, ty_pl.payload);
5002 const body = self.air.extra[loop.end..][0..loop.data.body_len];5002 const body = self.air.extra[loop.end..][0..loop.data.body_len];
5003 const liveness_loop = self.liveness.getLoop(inst);
5003 const start_index = @intCast(u32, self.mir_instructions.len);5004 const start_index = @intCast(u32, self.mir_instructions.len);
5005
5004 try self.genBody(body);5006 try self.genBody(body);
5005 try self.jump(start_index);5007 try self.jump(start_index);
5008
5009 try self.ensureProcessDeathCapacity(liveness_loop.deaths.len);
5010 for (liveness_loop.deaths) |operand| {
5011 self.processDeath(operand);
5012 }
5013
5006 return self.finishAirBookkeeping();5014 return self.finishAirBookkeeping();
5007}5015}
50085016
src/arch/arm/CodeGen.zig+8
...@@ -4923,9 +4923,17 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {...@@ -4923,9 +4923,17 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
4923 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;4923 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
4924 const loop = self.air.extraData(Air.Block, ty_pl.payload);4924 const loop = self.air.extraData(Air.Block, ty_pl.payload);
4925 const body = self.air.extra[loop.end..][0..loop.data.body_len];4925 const body = self.air.extra[loop.end..][0..loop.data.body_len];
4926 const liveness_loop = self.liveness.getLoop(inst);
4926 const start_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);4927 const start_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);
4928
4927 try self.genBody(body);4929 try self.genBody(body);
4928 try self.jump(start_index);4930 try self.jump(start_index);
4931
4932 try self.ensureProcessDeathCapacity(liveness_loop.deaths.len);
4933 for (liveness_loop.deaths) |operand| {
4934 self.processDeath(operand);
4935 }
4936
4929 return self.finishAirBookkeeping();4937 return self.finishAirBookkeeping();
4930}4938}
49314939
src/arch/sparc64/CodeGen.zig+8
...@@ -1750,9 +1750,17 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {...@@ -1750,9 +1750,17 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
1750 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;1750 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1751 const loop = self.air.extraData(Air.Block, ty_pl.payload);1751 const loop = self.air.extraData(Air.Block, ty_pl.payload);
1752 const body = self.air.extra[loop.end .. loop.end + loop.data.body_len];1752 const body = self.air.extra[loop.end .. loop.end + loop.data.body_len];
1753 const liveness_loop = self.liveness.getLoop(inst);
1753 const start = @intCast(u32, self.mir_instructions.len);1754 const start = @intCast(u32, self.mir_instructions.len);
1755
1754 try self.genBody(body);1756 try self.genBody(body);
1755 try self.jump(start);1757 try self.jump(start);
1758
1759 try self.ensureProcessDeathCapacity(liveness_loop.deaths.len);
1760 for (liveness_loop.deaths) |operand| {
1761 self.processDeath(operand);
1762 }
1763
1756 return self.finishAirBookkeeping();1764 return self.finishAirBookkeeping();
1757}1765}
17581766
src/arch/wasm/CodeGen.zig+6
...@@ -3042,6 +3042,7 @@ fn airLoop(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -3042,6 +3042,7 @@ fn airLoop(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3042 const ty_pl = func.air.instructions.items(.data)[inst].ty_pl;3042 const ty_pl = func.air.instructions.items(.data)[inst].ty_pl;
3043 const loop = func.air.extraData(Air.Block, ty_pl.payload);3043 const loop = func.air.extraData(Air.Block, ty_pl.payload);
3044 const body = func.air.extra[loop.end..][0..loop.data.body_len];3044 const body = func.air.extra[loop.end..][0..loop.data.body_len];
3045 const liveness_loop = func.liveness.getLoop(inst);
30453046
3046 // result type of loop is always 'noreturn', meaning we can always3047 // result type of loop is always 'noreturn', meaning we can always
3047 // emit the wasm type 'block_empty'.3048 // emit the wasm type 'block_empty'.
...@@ -3052,6 +3053,11 @@ fn airLoop(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -3052,6 +3053,11 @@ fn airLoop(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3052 try func.addLabel(.br, 0);3053 try func.addLabel(.br, 0);
3053 try func.endBlock();3054 try func.endBlock();
30543055
3056 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, @intCast(u32, liveness_loop.deaths.len));
3057 for (liveness_loop.deaths) |death| {
3058 func.processDeath(Air.indexToRef(death));
3059 }
3060
3055 func.finishAir(inst, .none, &.{});3061 func.finishAir(inst, .none, &.{});
3056}3062}
30573063
src/arch/x86_64/CodeGen.zig+7
...@@ -6185,6 +6185,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {...@@ -6185,6 +6185,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
6185 const loop = self.air.extraData(Air.Block, ty_pl.payload);6185 const loop = self.air.extraData(Air.Block, ty_pl.payload);
6186 const body = self.air.extra[loop.end..][0..loop.data.body_len];6186 const body = self.air.extra[loop.end..][0..loop.data.body_len];
6187 const jmp_target = @intCast(u32, self.mir_instructions.len);6187 const jmp_target = @intCast(u32, self.mir_instructions.len);
6188 const liveness_loop = self.liveness.getLoop(inst);
61886189
6189 {6190 {
6190 try self.branch_stack.append(.{});6191 try self.branch_stack.append(.{});
...@@ -6208,6 +6209,12 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {...@@ -6208,6 +6209,12 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
6208 try self.canonicaliseBranches(true, &dummy_branch, &branch, true, false);6209 try self.canonicaliseBranches(true, &dummy_branch, &branch, true, false);
62096210
6210 _ = try self.asmJmpReloc(jmp_target);6211 _ = try self.asmJmpReloc(jmp_target);
6212
6213 try self.ensureProcessDeathCapacity(liveness_loop.deaths.len);
6214 for (liveness_loop.deaths) |operand| {
6215 self.processDeath(operand);
6216 }
6217
6211 return self.finishAirBookkeeping();6218 return self.finishAirBookkeeping();
6212}6219}
62136220
src/codegen/c.zig+122-131
...@@ -55,7 +55,7 @@ const BlockData = struct {...@@ -55,7 +55,7 @@ const BlockData = struct {
55 result: CValue,55 result: CValue,
56};56};
5757
58pub const CValueMap = std.AutoHashMap(Air.Inst.Ref, CValue);58pub const CValueMap = std.AutoHashMap(Air.Inst.Index, CValue);
5959
60pub const LazyFnKey = union(enum) {60pub const LazyFnKey = union(enum) {
61 tag_name: Decl.Index,61 tag_name: Decl.Index,
...@@ -77,9 +77,8 @@ pub const LazyFnMap = std.AutoArrayHashMapUnmanaged(LazyFnKey, LazyFnValue);...@@ -77,9 +77,8 @@ pub const LazyFnMap = std.AutoArrayHashMapUnmanaged(LazyFnKey, LazyFnValue);
77const LoopDepth = u16;77const LoopDepth = u16;
78const Local = struct {78const Local = struct {
79 cty_idx: CType.Index,79 cty_idx: CType.Index,
80 /// How many loops the last definition was nested in.
81 loop_depth: LoopDepth,
82 alignas: CType.AlignAs,80 alignas: CType.AlignAs,
81 is_in_clone: bool,
8382
84 pub fn getType(local: Local) LocalType {83 pub fn getType(local: Local) LocalType {
85 return .{ .cty_idx = local.cty_idx, .alignas = local.alignas };84 return .{ .cty_idx = local.cty_idx, .alignas = local.alignas };
...@@ -90,7 +89,6 @@ const LocalIndex = u16;...@@ -90,7 +89,6 @@ const LocalIndex = u16;
90const LocalType = struct { cty_idx: CType.Index, alignas: CType.AlignAs };89const LocalType = struct { cty_idx: CType.Index, alignas: CType.AlignAs };
91const LocalsList = std.AutoArrayHashMapUnmanaged(LocalIndex, void);90const LocalsList = std.AutoArrayHashMapUnmanaged(LocalIndex, void);
92const LocalsMap = std.AutoArrayHashMapUnmanaged(LocalType, LocalsList);91const LocalsMap = std.AutoArrayHashMapUnmanaged(LocalType, LocalsList);
93const LocalsStack = std.ArrayListUnmanaged(LocalsMap);
9492
95const ValueRenderLocation = enum {93const ValueRenderLocation = enum {
96 FunctionArgument,94 FunctionArgument,
...@@ -279,41 +277,43 @@ pub const Function = struct {...@@ -279,41 +277,43 @@ pub const Function = struct {
279 /// Which locals are available for reuse, based on Type.277 /// Which locals are available for reuse, based on Type.
280 /// Only locals in the last stack entry are available for reuse,278 /// Only locals in the last stack entry are available for reuse,
281 /// other entries will become available on loop exit.279 /// other entries will become available on loop exit.
282 free_locals_stack: LocalsStack = .{},280 free_locals_map: LocalsMap = .{},
283 free_locals_clone_depth: LoopDepth = 0,281 is_in_clone: bool = false,
284 /// Locals which will not be freed by Liveness. This is used after a282 /// Locals which will not be freed by Liveness. This is used after a
285 /// Function body is lowered in order to make `free_locals_stack` have283 /// Function body is lowered in order to make `free_locals_map` have
286 /// 100% of the locals within so that it can be used to render the block284 /// 100% of the locals within so that it can be used to render the block
287 /// of variable declarations at the top of a function, sorted descending285 /// of variable declarations at the top of a function, sorted descending
288 /// by type alignment.286 /// by type alignment.
289 /// The value is whether the alloc is static or not.287 /// The value is whether the alloc is static or not.
290 allocs: std.AutoArrayHashMapUnmanaged(LocalIndex, bool) = .{},288 allocs: std.AutoArrayHashMapUnmanaged(LocalIndex, bool) = .{},
291 /// Needed for memory used by the keys of free_locals_stack entries.289 /// Needed for memory used by the keys of free_locals_map entries.
292 arena: std.heap.ArenaAllocator,290 arena: std.heap.ArenaAllocator,
293291
294 fn resolveInst(f: *Function, inst: Air.Inst.Ref) !CValue {292 fn resolveInst(f: *Function, ref: Air.Inst.Ref) !CValue {
295 const gop = try f.value_map.getOrPut(inst);293 if (Air.refToIndex(ref)) |inst| {
296 if (gop.found_existing) return gop.value_ptr.*;294 const gop = try f.value_map.getOrPut(inst);
297295 if (gop.found_existing) return gop.value_ptr.*;
298 const val = f.air.value(inst).?;
299 const ty = f.air.typeOf(inst);
300
301 const result: CValue = if (lowersToArray(ty, f.object.dg.module.getTarget())) result: {
302 const writer = f.object.code_header.writer();
303 const alignment = 0;
304 const decl_c_value = try f.allocLocalValue(ty, alignment);
305 const gpa = f.object.dg.gpa;
306 try f.allocs.put(gpa, decl_c_value.new_local, true);
307 try writer.writeAll("static ");
308 try f.object.dg.renderTypeAndName(writer, ty, decl_c_value, Const, alignment, .complete);
309 try writer.writeAll(" = ");
310 try f.object.dg.renderValue(writer, ty, val, .StaticInitializer);
311 try writer.writeAll(";\n ");
312 break :result decl_c_value;
313 } else .{ .constant = inst };
314296
315 gop.value_ptr.* = result;297 const val = f.air.value(ref).?;
316 return result;298 const ty = f.air.typeOf(ref);
299
300 const result: CValue = if (lowersToArray(ty, f.object.dg.module.getTarget())) result: {
301 const writer = f.object.code_header.writer();
302 const alignment = 0;
303 const decl_c_value = try f.allocLocalValue(ty, alignment);
304 const gpa = f.object.dg.gpa;
305 try f.allocs.put(gpa, decl_c_value.new_local, true);
306 try writer.writeAll("static ");
307 try f.object.dg.renderTypeAndName(writer, ty, decl_c_value, Const, alignment, .complete);
308 try writer.writeAll(" = ");
309 try f.object.dg.renderValue(writer, ty, val, .StaticInitializer);
310 try writer.writeAll(";\n ");
311 break :result decl_c_value;
312 } else .{ .constant = ref };
313
314 gop.value_ptr.* = result;
315 return result;
316 } else return .{ .constant = ref };
317 }317 }
318318
319 fn wantSafety(f: *Function) bool {319 fn wantSafety(f: *Function) bool {
...@@ -323,18 +323,14 @@ pub const Function = struct {...@@ -323,18 +323,14 @@ pub const Function = struct {
323 };323 };
324 }324 }
325325
326 fn getFreeLocals(f: *Function) *LocalsMap {
327 return &f.free_locals_stack.items[f.free_locals_stack.items.len - 1];
328 }
329
330 /// Skips the reuse logic.326 /// Skips the reuse logic.
331 fn allocLocalValue(f: *Function, ty: Type, alignment: u32) !CValue {327 fn allocLocalValue(f: *Function, ty: Type, alignment: u32) !CValue {
332 const gpa = f.object.dg.gpa;328 const gpa = f.object.dg.gpa;
333 const target = f.object.dg.module.getTarget();329 const target = f.object.dg.module.getTarget();
334 try f.locals.append(gpa, .{330 try f.locals.append(gpa, .{
335 .cty_idx = try f.typeToIndex(ty, .complete),331 .cty_idx = try f.typeToIndex(ty, .complete),
336 .loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1),
337 .alignas = CType.AlignAs.init(alignment, ty.abiAlignment(target)),332 .alignas = CType.AlignAs.init(alignment, ty.abiAlignment(target)),
333 .is_in_clone = f.is_in_clone,
338 });334 });
339 return .{ .new_local = @intCast(LocalIndex, f.locals.items.len - 1) };335 return .{ .new_local = @intCast(LocalIndex, f.locals.items.len - 1) };
340 }336 }
...@@ -348,13 +344,11 @@ pub const Function = struct {...@@ -348,13 +344,11 @@ pub const Function = struct {
348 /// Only allocates the local; does not print anything.344 /// Only allocates the local; does not print anything.
349 fn allocAlignedLocal(f: *Function, ty: Type, _: CQualifiers, alignment: u32) !CValue {345 fn allocAlignedLocal(f: *Function, ty: Type, _: CQualifiers, alignment: u32) !CValue {
350 const target = f.object.dg.module.getTarget();346 const target = f.object.dg.module.getTarget();
351 if (f.getFreeLocals().getPtr(.{347 if (f.free_locals_map.getPtr(.{
352 .cty_idx = try f.typeToIndex(ty, .complete),348 .cty_idx = try f.typeToIndex(ty, .complete),
353 .alignas = CType.AlignAs.init(alignment, ty.abiAlignment(target)),349 .alignas = CType.AlignAs.init(alignment, ty.abiAlignment(target)),
354 })) |locals_list| {350 })) |locals_list| {
355 if (locals_list.popOrNull()) |local_entry| {351 if (locals_list.popOrNull()) |local_entry| {
356 const local = &f.locals.items[local_entry.key];
357 local.loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1);
358 return .{ .new_local = local_entry.key };352 return .{ .new_local = local_entry.key };
359 }353 }
360 }354 }
...@@ -485,10 +479,7 @@ pub const Function = struct {...@@ -485,10 +479,7 @@ pub const Function = struct {
485 const gpa = f.object.dg.gpa;479 const gpa = f.object.dg.gpa;
486 f.allocs.deinit(gpa);480 f.allocs.deinit(gpa);
487 f.locals.deinit(gpa);481 f.locals.deinit(gpa);
488 for (f.free_locals_stack.items) |*free_locals| {482 deinitFreeLocalsMap(gpa, &f.free_locals_map);
489 deinitFreeLocalsMap(gpa, free_locals);
490 }
491 f.free_locals_stack.deinit(gpa);
492 f.blocks.deinit(gpa);483 f.blocks.deinit(gpa);
493 f.value_map.deinit();484 f.value_map.deinit();
494 f.lazy_fns.deinit(gpa);485 f.lazy_fns.deinit(gpa);
...@@ -2592,8 +2583,7 @@ pub fn genFunc(f: *Function) !void {...@@ -2592,8 +2583,7 @@ pub fn genFunc(f: *Function) !void {
2592 o.code_header.appendSliceAssumeCapacity("{\n ");2583 o.code_header.appendSliceAssumeCapacity("{\n ");
2593 const empty_header_len = o.code_header.items.len;2584 const empty_header_len = o.code_header.items.len;
25942585
2595 f.free_locals_stack.clearRetainingCapacity();2586 f.free_locals_map.clearRetainingCapacity();
2596 try f.free_locals_stack.append(gpa, .{});
25972587
2598 const main_body = f.air.getMainBody();2588 const main_body = f.air.getMainBody();
2599 try genBody(f, main_body);2589 try genBody(f, main_body);
...@@ -2605,7 +2595,8 @@ pub fn genFunc(f: *Function) !void {...@@ -2605,7 +2595,8 @@ pub fn genFunc(f: *Function) !void {
2605 // Liveness analysis, however, locals from alloc instructions will be2595 // Liveness analysis, however, locals from alloc instructions will be
2606 // missing. These are added now to complete the map. Then we can sort by2596 // missing. These are added now to complete the map. Then we can sort by
2607 // alignment, descending.2597 // alignment, descending.
2608 const free_locals = f.getFreeLocals();2598 const free_locals = &f.free_locals_map;
2599 assert(f.value_map.count() == 0); // there must not be any unfreed locals
2609 for (f.allocs.keys(), f.allocs.values()) |local_index, value| {2600 for (f.allocs.keys(), f.allocs.values()) |local_index, value| {
2610 if (value) continue; // static2601 if (value) continue; // static
2611 const local = f.locals.items[local_index];2602 const local = f.locals.items[local_index];
...@@ -3007,7 +2998,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,...@@ -3007,7 +2998,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
3007 if (result_value == .new_local) {2998 if (result_value == .new_local) {
3008 log.debug("map %{d} to t{d}", .{ inst, result_value.new_local });2999 log.debug("map %{d} to t{d}", .{ inst, result_value.new_local });
3009 }3000 }
3010 try f.value_map.putNoClobber(Air.indexToRef(inst), switch (result_value) {3001 try f.value_map.putNoClobber(inst, switch (result_value) {
3011 .none => continue,3002 .none => continue,
3012 .new_local => |i| .{ .local = i },3003 .new_local => |i| .{ .local = i },
3013 else => result_value,3004 else => result_value,
...@@ -3093,17 +3084,21 @@ fn airPtrElemPtr(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3093,17 +3084,21 @@ fn airPtrElemPtr(f: *Function, inst: Air.Inst.Index) !CValue {
3093 const child_ty = ptr_ty.childType();3084 const child_ty = ptr_ty.childType();
30943085
3095 const ptr = try f.resolveInst(bin_op.lhs);3086 const ptr = try f.resolveInst(bin_op.lhs);
3096 if (!child_ty.hasRuntimeBitsIgnoreComptime()) {
3097 if (f.liveness.operandDies(inst, 1)) try die(f, inst, bin_op.rhs);
3098 return ptr;
3099 }
3100 const index = try f.resolveInst(bin_op.rhs);3087 const index = try f.resolveInst(bin_op.rhs);
3101 try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });3088 try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });
31023089
3103 const writer = f.object.writer();3090 const writer = f.object.writer();
3104 const local = try f.allocLocal(inst, f.air.typeOfIndex(inst));3091 const local = try f.allocLocal(inst, f.air.typeOfIndex(inst));
3105 try f.writeCValue(writer, local, .Other);3092 try f.writeCValue(writer, local, .Other);
3106 try writer.writeAll(" = (");3093 try writer.writeAll(" = ");
3094
3095 if (!child_ty.hasRuntimeBitsIgnoreComptime()) {
3096 try f.writeCValue(writer, ptr, .Initializer);
3097 try writer.writeAll(";\n");
3098 return local;
3099 }
3100
3101 try writer.writeByte('(');
3107 try f.renderType(writer, inst_ty);3102 try f.renderType(writer, inst_ty);
3108 try writer.writeAll(")&(");3103 try writer.writeAll(")&(");
3109 if (ptr_ty.ptrSize() == .One) {3104 if (ptr_ty.ptrSize() == .One) {
...@@ -3229,12 +3224,11 @@ fn airArrayElemVal(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3229,12 +3224,11 @@ fn airArrayElemVal(f: *Function, inst: Air.Inst.Index) !CValue {
3229}3224}
32303225
3231fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {3226fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {
3232 const inst_ty = f.air.typeOfIndex(inst);3227 if (f.liveness.isUnused(inst)) return .none;
32333228
3229 const inst_ty = f.air.typeOfIndex(inst);
3234 const elem_type = inst_ty.elemType();3230 const elem_type = inst_ty.elemType();
3235 if (!elem_type.isFnOrHasRuntimeBitsIgnoreComptime()) {3231 if (!elem_type.isFnOrHasRuntimeBitsIgnoreComptime()) return .{ .undef = inst_ty };
3236 return .{ .undef = inst_ty };
3237 }
32383232
3239 const target = f.object.dg.module.getTarget();3233 const target = f.object.dg.module.getTarget();
3240 const local = try f.allocAlignedLocal(3234 const local = try f.allocAlignedLocal(
...@@ -3249,12 +3243,11 @@ fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3249,12 +3243,11 @@ fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {
3249}3243}
32503244
3251fn airRetPtr(f: *Function, inst: Air.Inst.Index) !CValue {3245fn airRetPtr(f: *Function, inst: Air.Inst.Index) !CValue {
3252 const inst_ty = f.air.typeOfIndex(inst);3246 if (f.liveness.isUnused(inst)) return .none;
32533247
3248 const inst_ty = f.air.typeOfIndex(inst);
3254 const elem_ty = inst_ty.elemType();3249 const elem_ty = inst_ty.elemType();
3255 if (!elem_ty.isFnOrHasRuntimeBitsIgnoreComptime()) {3250 if (!elem_ty.isFnOrHasRuntimeBitsIgnoreComptime()) return .{ .undef = inst_ty };
3256 return .{ .undef = inst_ty };
3257 }
32583251
3259 const target = f.object.dg.module.getTarget();3252 const target = f.object.dg.module.getTarget();
3260 const local = try f.allocAlignedLocal(3253 const local = try f.allocAlignedLocal(
...@@ -3274,10 +3267,22 @@ fn airArg(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3274,10 +3267,22 @@ fn airArg(f: *Function, inst: Air.Inst.Index) !CValue {
32743267
3275 const i = f.next_arg_index;3268 const i = f.next_arg_index;
3276 f.next_arg_index += 1;3269 f.next_arg_index += 1;
3277 return if (inst_cty != try f.typeToIndex(inst_ty, .complete))3270 const result: CValue = if (inst_cty != try f.typeToIndex(inst_ty, .complete))
3278 .{ .arg_array = i }3271 .{ .arg_array = i }
3279 else3272 else
3280 .{ .arg = i };3273 .{ .arg = i };
3274
3275 if (f.liveness.isUnused(inst)) {
3276 const writer = f.object.writer();
3277 try writer.writeByte('(');
3278 try f.renderType(writer, Type.void);
3279 try writer.writeByte(')');
3280 try f.writeCValue(writer, result, .Other);
3281 try writer.writeAll(";\n");
3282 return .none;
3283 }
3284
3285 return result;
3281}3286}
32823287
3283fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {3288fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
...@@ -4191,21 +4196,23 @@ fn airCall(...@@ -4191,21 +4196,23 @@ fn airCall(
4191 var lowered_ret_buf: LowerFnRetTyBuffer = undefined;4196 var lowered_ret_buf: LowerFnRetTyBuffer = undefined;
4192 const lowered_ret_ty = lowerFnRetTy(ret_ty, &lowered_ret_buf, target);4197 const lowered_ret_ty = lowerFnRetTy(ret_ty, &lowered_ret_buf, target);
41934198
4194 const result_local = if (modifier == .always_tail) r: {4199 const result_local = result: {
4195 try writer.writeAll("zig_always_tail return ");4200 if (modifier == .always_tail) {
4196 break :r .none;4201 try writer.writeAll("zig_always_tail return ");
4197 } else if (!lowered_ret_ty.hasRuntimeBitsIgnoreComptime())4202 break :result .none;
4198 .none4203 } else if (!lowered_ret_ty.hasRuntimeBitsIgnoreComptime()) {
4199 else if (f.liveness.isUnused(inst)) r: {4204 break :result .none;
4200 try writer.writeByte('(');4205 } else if (f.liveness.isUnused(inst)) {
4201 try f.renderType(writer, Type.void);4206 try writer.writeByte('(');
4202 try writer.writeByte(')');4207 try f.renderType(writer, Type.void);
4203 break :r .none;4208 try writer.writeByte(')');
4204 } else r: {4209 break :result .none;
4205 const local = try f.allocLocal(inst, try lowered_ret_ty.copy(f.arena.allocator()));4210 } else {
4206 try f.writeCValue(writer, local, .Other);4211 const local = try f.allocLocal(inst, try lowered_ret_ty.copy(f.arena.allocator()));
4207 try writer.writeAll(" = ");4212 try f.writeCValue(writer, local, .Other);
4208 break :r local;4213 try writer.writeAll(" = ");
4214 break :result local;
4215 }
4209 };4216 };
42104217
4211 callee: {4218 callee: {
...@@ -4250,9 +4257,9 @@ fn airCall(...@@ -4250,9 +4257,9 @@ fn airCall(
4250 }4257 }
4251 try writer.writeAll(");\n");4258 try writer.writeAll(");\n");
42524259
4253 const result = r: {4260 const result = result: {
4254 if (result_local == .none or !lowersToArray(ret_ty, target))4261 if (result_local == .none or !lowersToArray(ret_ty, target))
4255 break :r result_local;4262 break :result result_local;
42564263
4257 const array_local = try f.allocLocal(inst, ret_ty);4264 const array_local = try f.allocLocal(inst, ret_ty);
4258 try writer.writeAll("memcpy(");4265 try writer.writeAll("memcpy(");
...@@ -4263,7 +4270,7 @@ fn airCall(...@@ -4263,7 +4270,7 @@ fn airCall(
4263 try f.renderType(writer, ret_ty);4270 try f.renderType(writer, ret_ty);
4264 try writer.writeAll("));\n");4271 try writer.writeAll("));\n");
4265 try freeLocal(f, inst, result_local.new_local, 0);4272 try freeLocal(f, inst, result_local.new_local, 0);
4266 break :r array_local;4273 break :result array_local;
4267 };4274 };
42684275
4269 return result;4276 return result;
...@@ -4480,7 +4487,7 @@ fn airBitcast(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4480,7 +4487,7 @@ fn airBitcast(f: *Function, inst: Air.Inst.Index) !CValue {
4480 {4487 {
4481 try f.writeCValue(writer, local, .Other);4488 try f.writeCValue(writer, local, .Other);
4482 try writer.writeAll(" = ");4489 try writer.writeAll(" = ");
4483 try f.writeCValue(writer, operand, .Other);4490 try f.writeCValue(writer, operand, .Initializer);
4484 try writer.writeAll(";\n");4491 try writer.writeAll(";\n");
4485 return local;4492 return local;
4486 }4493 }
...@@ -4630,30 +4637,16 @@ fn airLoop(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4630,30 +4637,16 @@ fn airLoop(f: *Function, inst: Air.Inst.Index) !CValue {
4630 const ty_pl = f.air.instructions.items(.data)[inst].ty_pl;4637 const ty_pl = f.air.instructions.items(.data)[inst].ty_pl;
4631 const loop = f.air.extraData(Air.Block, ty_pl.payload);4638 const loop = f.air.extraData(Air.Block, ty_pl.payload);
4632 const body = f.air.extra[loop.end..][0..loop.data.body_len];4639 const body = f.air.extra[loop.end..][0..loop.data.body_len];
4640 const liveness_loop = f.liveness.getLoop(inst);
4633 const writer = f.object.writer();4641 const writer = f.object.writer();
46344642
4635 const gpa = f.object.dg.gpa;
4636 try f.free_locals_stack.insert(gpa, f.free_locals_stack.items.len - 1, .{});
4637
4638 try writer.writeAll("for (;;) ");4643 try writer.writeAll("for (;;) ");
4639 try genBody(f, body);4644 try genBody(f, body);
4640 try writer.writeByte('\n');4645 try writer.writeByte('\n');
46414646
4642 var old_free_locals = f.free_locals_stack.pop();4647 for (liveness_loop.deaths) |operand| {
4643 defer deinitFreeLocalsMap(gpa, &old_free_locals);4648 try die(f, inst, Air.indexToRef(operand));
4644 const new_free_locals = f.getFreeLocals();
4645 var it = new_free_locals.iterator();
4646 while (it.next()) |entry| {
4647 const gop = try old_free_locals.getOrPut(gpa, entry.key_ptr.*);
4648 if (gop.found_existing) {
4649 try gop.value_ptr.ensureUnusedCapacity(gpa, entry.value_ptr.count());
4650 for (entry.value_ptr.keys()) |local_index| {
4651 gop.value_ptr.putAssumeCapacityNoClobber(local_index, {});
4652 }
4653 } else gop.value_ptr.* = entry.value_ptr.move();
4654 }4649 }
4655 deinitFreeLocalsMap(gpa, new_free_locals);
4656 new_free_locals.* = old_free_locals.move();
46574650
4658 return .none;4651 return .none;
4659}4652}
...@@ -4673,7 +4666,7 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4673,7 +4666,7 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {
4673 const gpa = f.object.dg.gpa;4666 const gpa = f.object.dg.gpa;
4674 var cloned_map = try f.value_map.clone();4667 var cloned_map = try f.value_map.clone();
4675 defer cloned_map.deinit();4668 defer cloned_map.deinit();
4676 var cloned_frees = try cloneFreeLocalsMap(gpa, f.getFreeLocals());4669 var cloned_frees = try cloneFreeLocalsMap(gpa, &f.free_locals_map);
4677 defer deinitFreeLocalsMap(gpa, &cloned_frees);4670 defer deinitFreeLocalsMap(gpa, &cloned_frees);
46784671
4679 // Remember how many locals there were before entering the then branch so4672 // Remember how many locals there were before entering the then branch so
...@@ -4684,8 +4677,8 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4684,8 +4677,8 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {
4684 // that we can notice and make sure not to use them in the else branch.4677 // that we can notice and make sure not to use them in the else branch.
4685 // Any new allocs must be removed from the free list.4678 // Any new allocs must be removed from the free list.
4686 const pre_allocs_len = @intCast(LocalIndex, f.allocs.count());4679 const pre_allocs_len = @intCast(LocalIndex, f.allocs.count());
4687 const pre_clone_depth = f.free_locals_clone_depth;4680 const was_in_clone = f.is_in_clone;
4688 f.free_locals_clone_depth = @intCast(LoopDepth, f.free_locals_stack.items.len);4681 f.is_in_clone = true;
46894682
4690 for (liveness_condbr.then_deaths) |operand| {4683 for (liveness_condbr.then_deaths) |operand| {
4691 try die(f, inst, Air.indexToRef(operand));4684 try die(f, inst, Air.indexToRef(operand));
...@@ -4706,10 +4699,10 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4706,10 +4699,10 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {
47064699
4707 f.value_map.deinit();4700 f.value_map.deinit();
4708 f.value_map = cloned_map.move();4701 f.value_map = cloned_map.move();
4709 const free_locals = f.getFreeLocals();4702 const free_locals = &f.free_locals_map;
4710 deinitFreeLocalsMap(gpa, free_locals);4703 deinitFreeLocalsMap(gpa, free_locals);
4711 free_locals.* = cloned_frees.move();4704 free_locals.* = cloned_frees.move();
4712 f.free_locals_clone_depth = pre_clone_depth;4705 f.is_in_clone = was_in_clone;
4713 for (liveness_condbr.else_deaths) |operand| {4706 for (liveness_condbr.else_deaths) |operand| {
4714 try die(f, inst, Air.indexToRef(operand));4707 try die(f, inst, Air.indexToRef(operand));
4715 }4708 }
...@@ -4781,7 +4774,7 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4781,7 +4774,7 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
4781 if (case_i != last_case_i) {4774 if (case_i != last_case_i) {
4782 const old_value_map = f.value_map;4775 const old_value_map = f.value_map;
4783 f.value_map = try old_value_map.clone();4776 f.value_map = try old_value_map.clone();
4784 var free_locals = f.getFreeLocals();4777 var free_locals = &f.free_locals_map;
4785 const old_free_locals = free_locals.*;4778 const old_free_locals = free_locals.*;
4786 free_locals.* = try cloneFreeLocalsMap(gpa, free_locals);4779 free_locals.* = try cloneFreeLocalsMap(gpa, free_locals);
47874780
...@@ -4793,14 +4786,13 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4793,14 +4786,13 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
4793 // we can notice and make sure not to use them in subsequent branches.4786 // we can notice and make sure not to use them in subsequent branches.
4794 // Any new allocs must be removed from the free list.4787 // Any new allocs must be removed from the free list.
4795 const pre_allocs_len = @intCast(LocalIndex, f.allocs.count());4788 const pre_allocs_len = @intCast(LocalIndex, f.allocs.count());
4796 const pre_clone_depth = f.free_locals_clone_depth;4789 const was_in_clone = f.is_in_clone;
4797 f.free_locals_clone_depth = @intCast(LoopDepth, f.free_locals_stack.items.len);4790 f.is_in_clone = true;
47984791
4799 {4792 {
4800 defer {4793 defer {
4801 f.free_locals_clone_depth = pre_clone_depth;4794 f.is_in_clone = was_in_clone;
4802 f.value_map.deinit();4795 f.value_map.deinit();
4803 free_locals = f.getFreeLocals();
4804 deinitFreeLocalsMap(gpa, free_locals);4796 deinitFreeLocalsMap(gpa, free_locals);
4805 f.value_map = old_value_map;4797 f.value_map = old_value_map;
4806 free_locals.* = old_free_locals;4798 free_locals.* = old_free_locals;
...@@ -4862,8 +4854,8 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -4862,8 +4854,8 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
4862 const inputs = @ptrCast([]const Air.Inst.Ref, f.air.extra[extra_i..][0..extra.data.inputs_len]);4854 const inputs = @ptrCast([]const Air.Inst.Ref, f.air.extra[extra_i..][0..extra.data.inputs_len]);
4863 extra_i += inputs.len;4855 extra_i += inputs.len;
48644856
4865 const result = r: {4857 const result = result: {
4866 if (!is_volatile and f.liveness.isUnused(inst)) break :r .none;4858 if (!is_volatile and f.liveness.isUnused(inst)) break :result .none;
48674859
4868 const writer = f.object.writer();4860 const writer = f.object.writer();
4869 const inst_ty = f.air.typeOfIndex(inst);4861 const inst_ty = f.air.typeOfIndex(inst);
...@@ -5091,7 +5083,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -5091,7 +5083,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
5091 }5083 }
5092 }5084 }
50935085
5094 break :r local;5086 break :result if (f.liveness.isUnused(inst)) .none else local;
5095 };5087 };
50965088
5097 var bt = iterateBigTomb(f, inst);5089 var bt = iterateBigTomb(f, inst);
...@@ -7063,21 +7055,22 @@ fn airUnionInit(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -7063,21 +7055,22 @@ fn airUnionInit(f: *Function, inst: Air.Inst.Index) !CValue {
70637055
7064fn airPrefetch(f: *Function, inst: Air.Inst.Index) !CValue {7056fn airPrefetch(f: *Function, inst: Air.Inst.Index) !CValue {
7065 const prefetch = f.air.instructions.items(.data)[inst].prefetch;7057 const prefetch = f.air.instructions.items(.data)[inst].prefetch;
7066 switch (prefetch.cache) {7058
7067 .data => {},
7068 // The available prefetch intrinsics do not accept a cache argument; only
7069 // address, rw, and locality. So unless the cache is data, we do not lower
7070 // this instruction.
7071 .instruction => return .none,
7072 }
7073 const ptr = try f.resolveInst(prefetch.ptr);7059 const ptr = try f.resolveInst(prefetch.ptr);
7074 try reap(f, inst, &.{prefetch.ptr});7060 try reap(f, inst, &.{prefetch.ptr});
7061
7075 const writer = f.object.writer();7062 const writer = f.object.writer();
7076 try writer.writeAll("zig_prefetch(");7063 switch (prefetch.cache) {
7077 try f.writeCValue(writer, ptr, .FunctionArgument);7064 .data => {
7078 try writer.print(", {d}, {d});\n", .{7065 try writer.writeAll("zig_prefetch(");
7079 @enumToInt(prefetch.rw), prefetch.locality,7066 try f.writeCValue(writer, ptr, .FunctionArgument);
7080 });7067 try writer.print(", {d}, {d});\n", .{ @enumToInt(prefetch.rw), prefetch.locality });
7068 },
7069 // The available prefetch intrinsics do not accept a cache argument; only
7070 // address, rw, and locality.
7071 .instruction => {},
7072 }
7073
7081 return .none;7074 return .none;
7082}7075}
70837076
...@@ -7857,8 +7850,8 @@ fn reap(f: *Function, inst: Air.Inst.Index, operands: []const Air.Inst.Ref) !voi...@@ -7857,8 +7850,8 @@ fn reap(f: *Function, inst: Air.Inst.Index, operands: []const Air.Inst.Ref) !voi
78577850
7858fn die(f: *Function, inst: Air.Inst.Index, ref: Air.Inst.Ref) !void {7851fn die(f: *Function, inst: Air.Inst.Index, ref: Air.Inst.Ref) !void {
7859 const ref_inst = Air.refToIndex(ref) orelse return;7852 const ref_inst = Air.refToIndex(ref) orelse return;
7853 const c_value = (f.value_map.fetchRemove(ref_inst) orelse return).value;
7860 if (f.air.instructions.items(.tag)[ref_inst] == .constant) return;7854 if (f.air.instructions.items(.tag)[ref_inst] == .constant) return;
7861 const c_value = (f.value_map.fetchRemove(ref) orelse return).value;
7862 const local_index = switch (c_value) {7855 const local_index = switch (c_value) {
7863 .local, .new_local => |l| l,7856 .local, .new_local => |l| l,
7864 else => return,7857 else => return,
...@@ -7870,8 +7863,8 @@ fn freeLocal(f: *Function, inst: Air.Inst.Index, local_index: LocalIndex, ref_in...@@ -7870,8 +7863,8 @@ fn freeLocal(f: *Function, inst: Air.Inst.Index, local_index: LocalIndex, ref_in
7870 const gpa = f.object.dg.gpa;7863 const gpa = f.object.dg.gpa;
7871 const local = &f.locals.items[local_index];7864 const local = &f.locals.items[local_index];
7872 log.debug("%{d}: freeing t{d} (operand %{d})", .{ inst, local_index, ref_inst });7865 log.debug("%{d}: freeing t{d} (operand %{d})", .{ inst, local_index, ref_inst });
7873 if (local.loop_depth < f.free_locals_clone_depth) return;7866 if (f.is_in_clone != local.is_in_clone) return;
7874 const gop = try f.free_locals_stack.items[local.loop_depth].getOrPut(gpa, local.getType());7867 const gop = try f.free_locals_map.getOrPut(gpa, local.getType());
7875 if (!gop.found_existing) gop.value_ptr.* = .{};7868 if (!gop.found_existing) gop.value_ptr.* = .{};
7876 if (std.debug.runtime_safety) {7869 if (std.debug.runtime_safety) {
7877 // If this trips, an unfreeable allocation was attempted to be freed.7870 // If this trips, an unfreeable allocation was attempted to be freed.
...@@ -7935,23 +7928,21 @@ fn noticeBranchFrees(...@@ -7935,23 +7928,21 @@ fn noticeBranchFrees(
7935 pre_allocs_len: LocalIndex,7928 pre_allocs_len: LocalIndex,
7936 inst: Air.Inst.Index,7929 inst: Air.Inst.Index,
7937) !void {7930) !void {
7938 const free_locals = f.getFreeLocals();
7939
7940 for (f.locals.items[pre_locals_len..], pre_locals_len..) |*local, local_i| {7931 for (f.locals.items[pre_locals_len..], pre_locals_len..) |*local, local_i| {
7941 const local_index = @intCast(LocalIndex, local_i);7932 const local_index = @intCast(LocalIndex, local_i);
7942 if (f.allocs.contains(local_index)) {7933 if (f.allocs.contains(local_index)) {
7943 if (std.debug.runtime_safety) {7934 if (std.debug.runtime_safety) {
7944 // new allocs are no longer freeable, so make sure they aren't in the free list7935 // new allocs are no longer freeable, so make sure they aren't in the free list
7945 if (free_locals.getPtr(local.getType())) |locals_list| {7936 if (f.free_locals_map.getPtr(local.getType())) |locals_list| {
7946 assert(!locals_list.contains(local_index));7937 assert(!locals_list.contains(local_index));
7947 }7938 }
7948 }7939 }
7949 continue;7940 continue;
7950 }7941 }
79517942
7952 // free more deeply nested locals from other branches at current depth7943 // free cloned locals from other branches at current cloned-ness
7953 assert(local.loop_depth >= f.free_locals_stack.items.len - 1);7944 std.debug.assert(local.is_in_clone or !f.is_in_clone);
7954 local.loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1);7945 local.is_in_clone = f.is_in_clone;
7955 try freeLocal(f, inst, local_index, 0);7946 try freeLocal(f, inst, local_index, 0);
7956 }7947 }
79577948
...@@ -7959,6 +7950,6 @@ fn noticeBranchFrees(...@@ -7959,6 +7950,6 @@ fn noticeBranchFrees(
7959 const local_index = @intCast(LocalIndex, local_i);7950 const local_index = @intCast(LocalIndex, local_i);
7960 const local = &f.locals.items[local_index];7951 const local = &f.locals.items[local_index];
7961 // new allocs are no longer freeable, so remove them from the free list7952 // new allocs are no longer freeable, so remove them from the free list
7962 if (free_locals.getPtr(local.getType())) |locals_list| _ = locals_list.swapRemove(local_index);7953 if (f.free_locals_map.getPtr(local.getType())) |locals_list| _ = locals_list.swapRemove(local_index);
7963 }7954 }
7964}7955}
src/print_air.zig+28-3
...@@ -267,9 +267,9 @@ const Writer = struct {...@@ -267,9 +267,9 @@ const Writer = struct {
267 .c_va_copy,267 .c_va_copy,
268 => try w.writeTyOp(s, inst),268 => try w.writeTyOp(s, inst),
269269
270 .block,270 .block => try w.writeBlock(s, inst),
271 .loop,271
272 => try w.writeBlock(s, inst),272 .loop => try w.writeLoop(s, inst),
273273
274 .slice,274 .slice,
275 .slice_elem_ptr,275 .slice_elem_ptr,
...@@ -401,6 +401,31 @@ const Writer = struct {...@@ -401,6 +401,31 @@ const Writer = struct {
401 try s.writeAll("}");401 try s.writeAll("}");
402 }402 }
403403
404 fn writeLoop(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
405 const ty_pl = w.air.instructions.items(.data)[inst].ty_pl;
406 const extra = w.air.extraData(Air.Block, ty_pl.payload);
407 const body = w.air.extra[extra.end..][0..extra.data.body_len];
408 const liveness_loop = w.liveness.getLoop(inst);
409
410 try w.writeType(s, w.air.getRefType(ty_pl.ty));
411 if (w.skip_body) return s.writeAll(", ...");
412 try s.writeAll(", {\n");
413 const old_indent = w.indent;
414 w.indent += 2;
415 try w.writeBody(s, body);
416 if (liveness_loop.deaths.len != 0) {
417 try s.writeByteNTimes(' ', w.indent);
418 for (liveness_loop.deaths, 0..) |operand, i| {
419 if (i != 0) try s.writeAll(" ");
420 try s.print("%{d}!", .{operand});
421 }
422 try s.writeAll("\n");
423 }
424 w.indent = old_indent;
425 try s.writeByteNTimes(' ', w.indent);
426 try s.writeAll("}");
427 }
428
404 fn writeAggregateInit(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {429 fn writeAggregateInit(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
405 const ty_pl = w.air.instructions.items(.data)[inst].ty_pl;430 const ty_pl = w.air.instructions.items(.data)[inst].ty_pl;
406 const vector_ty = w.air.getRefType(ty_pl.ty);431 const vector_ty = w.air.getRefType(ty_pl.ty);
test/behavior/const_slice_child.zig-1
...@@ -7,7 +7,6 @@ const expect = testing.expect;...@@ -7,7 +7,6 @@ const expect = testing.expect;
7var argv: [*]const [*]const u8 = undefined;7var argv: [*]const [*]const u8 = undefined;
88
9test "const slice child" {9test "const slice child" {
10 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
11 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO10 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
12 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO11 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
13 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO12 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/for.zig-2
...@@ -227,7 +227,6 @@ test "else continue outer for" {...@@ -227,7 +227,6 @@ test "else continue outer for" {
227227
228test "for loop with else branch" {228test "for loop with else branch" {
229 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO229 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
230 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
231230
232 {231 {
233 var x = [_]u32{ 1, 2 };232 var x = [_]u32{ 1, 2 };
...@@ -377,7 +376,6 @@ test "raw pointer and slice" {...@@ -377,7 +376,6 @@ test "raw pointer and slice" {
377test "raw pointer and counter" {376test "raw pointer and counter" {
378 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO377 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
379 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO378 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
380 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
381 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO379 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
382380
383 var buf: [10]u8 = undefined;381 var buf: [10]u8 = undefined;
test/behavior/slice.zig-1
...@@ -238,7 +238,6 @@ test "C pointer" {...@@ -238,7 +238,6 @@ test "C pointer" {
238test "C pointer slice access" {238test "C pointer slice access" {
239 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;239 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
240 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;240 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
241 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
242 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO241 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
243242
244 var buf: [10]u32 = [1]u32{42} ** 10;243 var buf: [10]u32 = [1]u32{42} ** 10;