authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-04-12 07:05:58-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-04-12 11:22:12-07:00
logb0edd8752a00ea191decf302d9802b853d85fd4c
tree67d5bd98e2eff521205d91ea85329e414164bd23
parent17631cb2d30bf2d7b10401cf8f784a599bade5c5

Liveness: modify encoding to support over 32 operands

Prior to this, Liveness encoded `asm`, `call`, and `aggregate_init` with a single 32-bit integer, allowing up to 35 operands (3 are provided by the regular tomb_bits). However, the Zig language allows function calls with more than 35 arguments, inline assembly with more than 35 inputs, and anonymous tuples with more than 35 elements. The new encoding stores an index to the extra array instead of the bits directly, and then as many extra elements as needed to encode all the operands. The MSB is used as a flag to tell which element is the last one, allowing for 31 bits per element. Prior to this, print_air did not bother correctly printing tombstones for these instructions; now it does. In addition to updating the BigTomb iteration logic in the machine code backends, this commit extracts the common logic into the Liveness namespace.

8 files changed, 240 insertions(+), 90 deletions(-)

src/Liveness.zig+66-5
...@@ -178,11 +178,50 @@ pub fn deinit(l: *Liveness, gpa: Allocator) void {...@@ -178,11 +178,50 @@ pub fn deinit(l: *Liveness, gpa: Allocator) void {
178 l.* = undefined;178 l.* = undefined;
179}179}
180180
181pub fn iterateBigTomb(l: Liveness, inst: Air.Inst.Index) BigTomb {
182 return .{
183 .tomb_bits = l.getTombBits(inst),
184 .extra_start = l.special.get(inst) orelse 0,
185 .extra_offset = 0,
186 .extra = l.extra,
187 .bit_index = 0,
188 };
189}
190
181/// How many tomb bits per AIR instruction.191/// How many tomb bits per AIR instruction.
182pub const bpi = 4;192pub const bpi = 4;
183pub const Bpi = std.meta.Int(.unsigned, bpi);193pub const Bpi = std.meta.Int(.unsigned, bpi);
184pub const OperandInt = std.math.Log2Int(Bpi);194pub const OperandInt = std.math.Log2Int(Bpi);
185195
196/// Useful for decoders of Liveness information.
197pub const BigTomb = struct {
198 tomb_bits: Liveness.Bpi,
199 bit_index: u32,
200 extra_start: u32,
201 extra_offset: u32,
202 extra: []const u32,
203
204 /// Returns whether the next operand dies.
205 pub fn feed(bt: *BigTomb) bool {
206 const this_bit_index = bt.bit_index;
207 bt.bit_index += 1;
208
209 const small_tombs = Liveness.bpi - 1;
210 if (this_bit_index < small_tombs) {
211 const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
212 return dies;
213 }
214
215 const big_bit_index = this_bit_index - small_tombs;
216 while (big_bit_index - bt.extra_offset * 31 >= 31) {
217 bt.extra_offset += 1;
218 }
219 const dies = @truncate(u1, bt.extra[bt.extra_start + bt.extra_offset] >>
220 @intCast(u5, big_bit_index - bt.extra_offset * 31)) != 0;
221 return dies;
222 }
223};
224
186/// In-progress data; on successful analysis converted into `Liveness`.225/// In-progress data; on successful analysis converted into `Liveness`.
187const Analysis = struct {226const Analysis = struct {
188 gpa: Allocator,227 gpa: Allocator,
...@@ -428,6 +467,7 @@ fn analyzeInst(...@@ -428,6 +467,7 @@ fn analyzeInst(
428 .inst = inst,467 .inst = inst,
429 .main_tomb = main_tomb,468 .main_tomb = main_tomb,
430 };469 };
470 defer extra_tombs.deinit();
431 try extra_tombs.feed(callee);471 try extra_tombs.feed(callee);
432 for (args) |arg| {472 for (args) |arg| {
433 try extra_tombs.feed(arg);473 try extra_tombs.feed(arg);
...@@ -468,6 +508,7 @@ fn analyzeInst(...@@ -468,6 +508,7 @@ fn analyzeInst(
468 .inst = inst,508 .inst = inst,
469 .main_tomb = main_tomb,509 .main_tomb = main_tomb,
470 };510 };
511 defer extra_tombs.deinit();
471 for (elements) |elem| {512 for (elements) |elem| {
472 try extra_tombs.feed(elem);513 try extra_tombs.feed(elem);
473 }514 }
...@@ -555,6 +596,7 @@ fn analyzeInst(...@@ -555,6 +596,7 @@ fn analyzeInst(
555 .inst = inst,596 .inst = inst,
556 .main_tomb = main_tomb,597 .main_tomb = main_tomb,
557 };598 };
599 defer extra_tombs.deinit();
558 for (outputs) |output| {600 for (outputs) |output| {
559 if (output != .none) {601 if (output != .none) {
560 try extra_tombs.feed(output);602 try extra_tombs.feed(output);
...@@ -790,10 +832,10 @@ const ExtraTombs = struct {...@@ -790,10 +832,10 @@ const ExtraTombs = struct {
790 bit_index: usize = 0,832 bit_index: usize = 0,
791 tomb_bits: Bpi = 0,833 tomb_bits: Bpi = 0,
792 big_tomb_bits: u32 = 0,834 big_tomb_bits: u32 = 0,
835 big_tomb_bits_extra: std.ArrayListUnmanaged(u32) = .{},
793836
794 fn feed(et: *ExtraTombs, op_ref: Air.Inst.Ref) !void {837 fn feed(et: *ExtraTombs, op_ref: Air.Inst.Ref) !void {
795 const this_bit_index = et.bit_index;838 const this_bit_index = et.bit_index;
796 assert(this_bit_index < 32); // TODO mechanism for when there are greater than 32 operands
797 et.bit_index += 1;839 et.bit_index += 1;
798 const gpa = et.analysis.gpa;840 const gpa = et.analysis.gpa;
799 const op_index = Air.refToIndex(op_ref) orelse return;841 const op_index = Air.refToIndex(op_ref) orelse return;
...@@ -801,18 +843,37 @@ const ExtraTombs = struct {...@@ -801,18 +843,37 @@ const ExtraTombs = struct {
801 if (prev == null) {843 if (prev == null) {
802 // Death.844 // Death.
803 if (et.new_set) |ns| try ns.putNoClobber(gpa, op_index, {});845 if (et.new_set) |ns| try ns.putNoClobber(gpa, op_index, {});
804 if (this_bit_index < bpi - 1) {846 const available_tomb_bits = bpi - 1;
847 if (this_bit_index < available_tomb_bits) {
805 et.tomb_bits |= @as(Bpi, 1) << @intCast(OperandInt, this_bit_index);848 et.tomb_bits |= @as(Bpi, 1) << @intCast(OperandInt, this_bit_index);
806 } else {849 } else {
807 const big_bit_index = this_bit_index - (bpi - 1);850 const big_bit_index = this_bit_index - available_tomb_bits;
808 et.big_tomb_bits |= @as(u32, 1) << @intCast(u5, big_bit_index);851 while (big_bit_index >= (et.big_tomb_bits_extra.items.len + 1) * 31) {
852 // We need another element in the extra array.
853 try et.big_tomb_bits_extra.append(gpa, et.big_tomb_bits);
854 et.big_tomb_bits = 0;
855 } else {
856 const final_bit_index = big_bit_index - et.big_tomb_bits_extra.items.len * 31;
857 et.big_tomb_bits |= @as(u32, 1) << @intCast(u5, final_bit_index);
858 }
809 }859 }
810 }860 }
811 }861 }
812862
813 fn finish(et: *ExtraTombs) !void {863 fn finish(et: *ExtraTombs) !void {
814 et.tomb_bits |= @as(Bpi, @boolToInt(et.main_tomb)) << (bpi - 1);864 et.tomb_bits |= @as(Bpi, @boolToInt(et.main_tomb)) << (bpi - 1);
865 // Signal the terminal big_tomb_bits element.
866 et.big_tomb_bits |= @as(u32, 1) << 31;
867
815 et.analysis.storeTombBits(et.inst, et.tomb_bits);868 et.analysis.storeTombBits(et.inst, et.tomb_bits);
816 try et.analysis.special.put(et.analysis.gpa, et.inst, et.big_tomb_bits);869 const extra_index = @intCast(u32, et.analysis.extra.items.len);
870 try et.analysis.extra.ensureUnusedCapacity(et.analysis.gpa, et.big_tomb_bits_extra.items.len + 1);
871 try et.analysis.special.put(et.analysis.gpa, et.inst, extra_index);
872 et.analysis.extra.appendSliceAssumeCapacity(et.big_tomb_bits_extra.items);
873 et.analysis.extra.appendAssumeCapacity(et.big_tomb_bits);
874 }
875
876 fn deinit(et: *ExtraTombs) void {
877 et.big_tomb_bits_extra.deinit(et.analysis.gpa);
817 }878 }
818};879};
src/arch/aarch64/CodeGen.zig+5-21
...@@ -202,26 +202,12 @@ const BlockData = struct {...@@ -202,26 +202,12 @@ const BlockData = struct {
202const BigTomb = struct {202const BigTomb = struct {
203 function: *Self,203 function: *Self,
204 inst: Air.Inst.Index,204 inst: Air.Inst.Index,
205 tomb_bits: Liveness.Bpi,205 lbt: Liveness.BigTomb,
206 big_tomb_bits: u32,
207 bit_index: usize,
208206
209 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {207 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
210 const this_bit_index = bt.bit_index;208 const dies = bt.lbt.feed();
211 bt.bit_index += 1;209 const op_index = Air.refToIndex(op_ref) orelse return;
212210 if (!dies) return;
213 const op_int = @enumToInt(op_ref);
214 if (op_int < Air.Inst.Ref.typed_value_map.len) return;
215 const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
216
217 if (this_bit_index < Liveness.bpi - 1) {
218 const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
219 if (!dies) return;
220 } else {
221 const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
222 const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
223 if (!dies) return;
224 }
225 bt.function.processDeath(op_index);211 bt.function.processDeath(op_index);
226 }212 }
227213
...@@ -3291,9 +3277,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT...@@ -3291,9 +3277,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT
3291 return BigTomb{3277 return BigTomb{
3292 .function = self,3278 .function = self,
3293 .inst = inst,3279 .inst = inst,
3294 .tomb_bits = self.liveness.getTombBits(inst),3280 .lbt = self.liveness.iterateBigTomb(inst),
3295 .big_tomb_bits = self.liveness.special.get(inst) orelse 0,
3296 .bit_index = 0,
3297 };3281 };
3298}3282}
32993283
src/arch/arm/CodeGen.zig+5-21
...@@ -224,26 +224,12 @@ const BlockData = struct {...@@ -224,26 +224,12 @@ const BlockData = struct {
224const BigTomb = struct {224const BigTomb = struct {
225 function: *Self,225 function: *Self,
226 inst: Air.Inst.Index,226 inst: Air.Inst.Index,
227 tomb_bits: Liveness.Bpi,227 lbt: Liveness.BigTomb,
228 big_tomb_bits: u32,
229 bit_index: usize,
230228
231 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {229 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
232 const this_bit_index = bt.bit_index;230 const dies = bt.lbt.feed();
233 bt.bit_index += 1;231 const op_index = Air.refToIndex(op_ref) orelse return;
234232 if (!dies) return;
235 const op_int = @enumToInt(op_ref);
236 if (op_int < Air.Inst.Ref.typed_value_map.len) return;
237 const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
238
239 if (this_bit_index < Liveness.bpi - 1) {
240 const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
241 if (!dies) return;
242 } else {
243 const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
244 const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
245 if (!dies) return;
246 }
247 bt.function.processDeath(op_index);233 bt.function.processDeath(op_index);
248 }234 }
249235
...@@ -4076,9 +4062,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT...@@ -4076,9 +4062,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT
4076 return BigTomb{4062 return BigTomb{
4077 .function = self,4063 .function = self,
4078 .inst = inst,4064 .inst = inst,
4079 .tomb_bits = self.liveness.getTombBits(inst),4065 .lbt = self.liveness.iterateBigTomb(inst),
4080 .big_tomb_bits = self.liveness.special.get(inst) orelse 0,
4081 .bit_index = 0,
4082 };4066 };
4083}4067}
40844068
src/arch/riscv64/CodeGen.zig+5-21
...@@ -194,26 +194,12 @@ const Reloc = union(enum) {...@@ -194,26 +194,12 @@ const Reloc = union(enum) {
194const BigTomb = struct {194const BigTomb = struct {
195 function: *Self,195 function: *Self,
196 inst: Air.Inst.Index,196 inst: Air.Inst.Index,
197 tomb_bits: Liveness.Bpi,197 lbt: Liveness.BigTomb,
198 big_tomb_bits: u32,
199 bit_index: usize,
200198
201 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {199 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
202 const this_bit_index = bt.bit_index;200 const dies = bt.lbt.feed();
203 bt.bit_index += 1;201 const op_index = Air.refToIndex(op_ref) orelse return;
204202 if (!dies) return;
205 const op_int = @enumToInt(op_ref);
206 if (op_int < Air.Inst.Ref.typed_value_map.len) return;
207 const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
208
209 if (this_bit_index < Liveness.bpi - 1) {
210 const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
211 if (!dies) return;
212 } else {
213 const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
214 const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
215 if (!dies) return;
216 }
217 bt.function.processDeath(op_index);203 bt.function.processDeath(op_index);
218 }204 }
219205
...@@ -2198,9 +2184,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT...@@ -2198,9 +2184,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT
2198 return BigTomb{2184 return BigTomb{
2199 .function = self,2185 .function = self,
2200 .inst = inst,2186 .inst = inst,
2201 .tomb_bits = self.liveness.getTombBits(inst),2187 .lbt = self.liveness.iterateBigTomb(inst),
2202 .big_tomb_bits = self.liveness.special.get(inst) orelse 0,
2203 .bit_index = 0,
2204 };2188 };
2205}2189}
22062190
src/arch/x86_64/CodeGen.zig+4-18
...@@ -272,24 +272,12 @@ const BlockData = struct {...@@ -272,24 +272,12 @@ const BlockData = struct {
272const BigTomb = struct {272const BigTomb = struct {
273 function: *Self,273 function: *Self,
274 inst: Air.Inst.Index,274 inst: Air.Inst.Index,
275 tomb_bits: Liveness.Bpi,275 lbt: Liveness.BigTomb,
276 big_tomb_bits: u32,
277 bit_index: usize,
278276
279 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {277 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
280 const this_bit_index = bt.bit_index;278 const dies = bt.lbt.feed();
281 bt.bit_index += 1;
282
283 const op_index = Air.refToIndex(op_ref) orelse return;279 const op_index = Air.refToIndex(op_ref) orelse return;
284280 if (!dies) return;
285 if (this_bit_index < Liveness.bpi - 1) {
286 const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
287 if (!dies) return;
288 } else {
289 const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
290 const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
291 if (!dies) return;
292 }
293 bt.function.processDeath(op_index);281 bt.function.processDeath(op_index);
294 }282 }
295283
...@@ -4845,9 +4833,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT...@@ -4845,9 +4833,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT
4845 return BigTomb{4833 return BigTomb{
4846 .function = self,4834 .function = self,
4847 .inst = inst,4835 .inst = inst,
4848 .tomb_bits = self.liveness.getTombBits(inst),4836 .lbt = self.liveness.iterateBigTomb(inst),
4849 .big_tomb_bits = self.liveness.special.get(inst) orelse 0,
4850 .bit_index = 0,
4851 };4837 };
4852}4838}
48534839
src/codegen/c.zig+1-1
...@@ -1348,7 +1348,7 @@ pub const DeclGen = struct {...@@ -1348,7 +1348,7 @@ pub const DeclGen = struct {
1348 return w.writeAll(name);1348 return w.writeAll(name);
1349 },1349 },
1350 .ErrorSet => {1350 .ErrorSet => {
1351 comptime std.debug.assert(Type.initTag(.anyerror).abiSize(builtin.target) == 2);1351 comptime assert(Type.initTag(.anyerror).abiSize(builtin.target) == 2);
1352 return w.writeAll("uint16_t");1352 return w.writeAll("uint16_t");
1353 },1353 },
1354 .ErrorUnion => {1354 .ErrorUnion => {
src/print_air.zig+13-3
...@@ -724,11 +724,21 @@ const Writer = struct {...@@ -724,11 +724,21 @@ const Writer = struct {
724 op_index: usize,724 op_index: usize,
725 operand: Air.Inst.Ref,725 operand: Air.Inst.Ref,
726 ) @TypeOf(s).Error!void {726 ) @TypeOf(s).Error!void {
727 const dies = if (op_index < Liveness.bpi - 1)727 const small_tomb_bits = Liveness.bpi - 1;
728 const dies = if (op_index < small_tomb_bits)
728 w.liveness.operandDies(inst, @intCast(Liveness.OperandInt, op_index))729 w.liveness.operandDies(inst, @intCast(Liveness.OperandInt, op_index))
729 else blk: {730 else blk: {
730 // TODO731 var extra_index = w.liveness.special.get(inst).?;
731 break :blk false;732 var tomb_op_index: usize = small_tomb_bits;
733 while (true) {
734 const bits = w.liveness.extra[extra_index];
735 if (op_index < tomb_op_index + 31) {
736 break :blk @truncate(u1, bits >> @intCast(u5, op_index - tomb_op_index)) != 0;
737 }
738 if ((bits >> 31) != 0) break :blk false;
739 extra_index += 1;
740 tomb_op_index += 31;
741 } else unreachable;
732 };742 };
733 return w.writeInstRef(s, operand, dies);743 return w.writeInstRef(s, operand, dies);
734 }744 }
test/behavior/call.zig+141
...@@ -118,3 +118,144 @@ test "result location of function call argument through runtime condition and st...@@ -118,3 +118,144 @@ test "result location of function call argument through runtime condition and st
118 .e = if (!runtime) .a else .b,118 .e = if (!runtime) .a else .b,
119 });119 });
120}120}
121
122test "function call with 40 arguments" {
123 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
124
125 const S = struct {
126 fn doTheTest(thirty_nine: i32) !void {
127 const result = add(
128 0,
129 1,
130 2,
131 3,
132 4,
133 5,
134 6,
135 7,
136 8,
137 9,
138 10,
139 11,
140 12,
141 13,
142 14,
143 15,
144 16,
145 17,
146 18,
147 19,
148 20,
149 21,
150 22,
151 23,
152 24,
153 25,
154 26,
155 27,
156 28,
157 29,
158 30,
159 31,
160 32,
161 33,
162 34,
163 35,
164 36,
165 37,
166 38,
167 thirty_nine,
168 40,
169 );
170 try expect(result == 820);
171 try expect(thirty_nine == 39);
172 }
173
174 fn add(
175 a0: i32,
176 a1: i32,
177 a2: i32,
178 a3: i32,
179 a4: i32,
180 a5: i32,
181 a6: i32,
182 a7: i32,
183 a8: i32,
184 a9: i32,
185 a10: i32,
186 a11: i32,
187 a12: i32,
188 a13: i32,
189 a14: i32,
190 a15: i32,
191 a16: i32,
192 a17: i32,
193 a18: i32,
194 a19: i32,
195 a20: i32,
196 a21: i32,
197 a22: i32,
198 a23: i32,
199 a24: i32,
200 a25: i32,
201 a26: i32,
202 a27: i32,
203 a28: i32,
204 a29: i32,
205 a30: i32,
206 a31: i32,
207 a32: i32,
208 a33: i32,
209 a34: i32,
210 a35: i32,
211 a36: i32,
212 a37: i32,
213 a38: i32,
214 a39: i32,
215 a40: i32,
216 ) i32 {
217 return a0 +
218 a1 +
219 a2 +
220 a3 +
221 a4 +
222 a5 +
223 a6 +
224 a7 +
225 a8 +
226 a9 +
227 a10 +
228 a11 +
229 a12 +
230 a13 +
231 a14 +
232 a15 +
233 a16 +
234 a17 +
235 a18 +
236 a19 +
237 a20 +
238 a21 +
239 a22 +
240 a23 +
241 a24 +
242 a25 +
243 a26 +
244 a27 +
245 a28 +
246 a29 +
247 a30 +
248 a31 +
249 a32 +
250 a33 +
251 a34 +
252 a35 +
253 a36 +
254 a37 +
255 a38 +
256 a39 +
257 a40;
258 }
259 };
260 try S.doTheTest(39);
261}