authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-04-12 07:05:58-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-04-12 11:22:12-07:00
logb0edd8752a00ea191decf302d9802b853d85fd4c
tree67d5bd98e2eff521205d91ea85329e414164bd23
parent17631cb2d30bf2d7b10401cf8f784a599bade5c5

Liveness: modify encoding to support over 32 operands

Prior to this, Liveness encoded `asm`, `call`, and `aggregate_init` with a single 32-bit integer, allowing up to 35 operands (3 are provided by the regular tomb_bits). However, the Zig language allows function calls with more than 35 arguments, inline assembly with more than 35 inputs, and anonymous tuples with more than 35 elements. The new encoding stores an index to the extra array instead of the bits directly, and then as many extra elements as needed to encode all the operands. The MSB is used as a flag to tell which element is the last one, allowing for 31 bits per element. Prior to this, print_air did not bother correctly printing tombstones for these instructions; now it does. In addition to updating the BigTomb iteration logic in the machine code backends, this commit extracts the common logic into the Liveness namespace.

8 files changed, 240 insertions(+), 90 deletions(-)

src/Liveness.zig+66-5
......@@ -178,11 +178,50 @@ pub fn deinit(l: *Liveness, gpa: Allocator) void {
178178 l.* = undefined;
179179}
180180
181pub fn iterateBigTomb(l: Liveness, inst: Air.Inst.Index) BigTomb {
182 return .{
183 .tomb_bits = l.getTombBits(inst),
184 .extra_start = l.special.get(inst) orelse 0,
185 .extra_offset = 0,
186 .extra = l.extra,
187 .bit_index = 0,
188 };
189}
190
181191/// How many tomb bits per AIR instruction.
182192pub const bpi = 4;
183193pub const Bpi = std.meta.Int(.unsigned, bpi);
184194pub const OperandInt = std.math.Log2Int(Bpi);
185195
196/// Useful for decoders of Liveness information.
197pub const BigTomb = struct {
198 tomb_bits: Liveness.Bpi,
199 bit_index: u32,
200 extra_start: u32,
201 extra_offset: u32,
202 extra: []const u32,
203
204 /// Returns whether the next operand dies.
205 pub fn feed(bt: *BigTomb) bool {
206 const this_bit_index = bt.bit_index;
207 bt.bit_index += 1;
208
209 const small_tombs = Liveness.bpi - 1;
210 if (this_bit_index < small_tombs) {
211 const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
212 return dies;
213 }
214
215 const big_bit_index = this_bit_index - small_tombs;
216 while (big_bit_index - bt.extra_offset * 31 >= 31) {
217 bt.extra_offset += 1;
218 }
219 const dies = @truncate(u1, bt.extra[bt.extra_start + bt.extra_offset] >>
220 @intCast(u5, big_bit_index - bt.extra_offset * 31)) != 0;
221 return dies;
222 }
223};
224
186225/// In-progress data; on successful analysis converted into `Liveness`.
187226const Analysis = struct {
188227 gpa: Allocator,
......@@ -428,6 +467,7 @@ fn analyzeInst(
428467 .inst = inst,
429468 .main_tomb = main_tomb,
430469 };
470 defer extra_tombs.deinit();
431471 try extra_tombs.feed(callee);
432472 for (args) |arg| {
433473 try extra_tombs.feed(arg);
......@@ -468,6 +508,7 @@ fn analyzeInst(
468508 .inst = inst,
469509 .main_tomb = main_tomb,
470510 };
511 defer extra_tombs.deinit();
471512 for (elements) |elem| {
472513 try extra_tombs.feed(elem);
473514 }
......@@ -555,6 +596,7 @@ fn analyzeInst(
555596 .inst = inst,
556597 .main_tomb = main_tomb,
557598 };
599 defer extra_tombs.deinit();
558600 for (outputs) |output| {
559601 if (output != .none) {
560602 try extra_tombs.feed(output);
......@@ -790,10 +832,10 @@ const ExtraTombs = struct {
790832 bit_index: usize = 0,
791833 tomb_bits: Bpi = 0,
792834 big_tomb_bits: u32 = 0,
835 big_tomb_bits_extra: std.ArrayListUnmanaged(u32) = .{},
793836
794837 fn feed(et: *ExtraTombs, op_ref: Air.Inst.Ref) !void {
795838 const this_bit_index = et.bit_index;
796 assert(this_bit_index < 32); // TODO mechanism for when there are greater than 32 operands
797839 et.bit_index += 1;
798840 const gpa = et.analysis.gpa;
799841 const op_index = Air.refToIndex(op_ref) orelse return;
......@@ -801,18 +843,37 @@ const ExtraTombs = struct {
801843 if (prev == null) {
802844 // Death.
803845 if (et.new_set) |ns| try ns.putNoClobber(gpa, op_index, {});
804 if (this_bit_index < bpi - 1) {
846 const available_tomb_bits = bpi - 1;
847 if (this_bit_index < available_tomb_bits) {
805848 et.tomb_bits |= @as(Bpi, 1) << @intCast(OperandInt, this_bit_index);
806849 } else {
807 const big_bit_index = this_bit_index - (bpi - 1);
808 et.big_tomb_bits |= @as(u32, 1) << @intCast(u5, big_bit_index);
850 const big_bit_index = this_bit_index - available_tomb_bits;
851 while (big_bit_index >= (et.big_tomb_bits_extra.items.len + 1) * 31) {
852 // We need another element in the extra array.
853 try et.big_tomb_bits_extra.append(gpa, et.big_tomb_bits);
854 et.big_tomb_bits = 0;
855 } else {
856 const final_bit_index = big_bit_index - et.big_tomb_bits_extra.items.len * 31;
857 et.big_tomb_bits |= @as(u32, 1) << @intCast(u5, final_bit_index);
858 }
809859 }
810860 }
811861 }
812862
813863 fn finish(et: *ExtraTombs) !void {
814864 et.tomb_bits |= @as(Bpi, @boolToInt(et.main_tomb)) << (bpi - 1);
865 // Signal the terminal big_tomb_bits element.
866 et.big_tomb_bits |= @as(u32, 1) << 31;
867
815868 et.analysis.storeTombBits(et.inst, et.tomb_bits);
816 try et.analysis.special.put(et.analysis.gpa, et.inst, et.big_tomb_bits);
869 const extra_index = @intCast(u32, et.analysis.extra.items.len);
870 try et.analysis.extra.ensureUnusedCapacity(et.analysis.gpa, et.big_tomb_bits_extra.items.len + 1);
871 try et.analysis.special.put(et.analysis.gpa, et.inst, extra_index);
872 et.analysis.extra.appendSliceAssumeCapacity(et.big_tomb_bits_extra.items);
873 et.analysis.extra.appendAssumeCapacity(et.big_tomb_bits);
874 }
875
876 fn deinit(et: *ExtraTombs) void {
877 et.big_tomb_bits_extra.deinit(et.analysis.gpa);
817878 }
818879};
src/arch/aarch64/CodeGen.zig+5-21
......@@ -202,26 +202,12 @@ const BlockData = struct {
202202const BigTomb = struct {
203203 function: *Self,
204204 inst: Air.Inst.Index,
205 tomb_bits: Liveness.Bpi,
206 big_tomb_bits: u32,
207 bit_index: usize,
205 lbt: Liveness.BigTomb,
208206
209207 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
210 const this_bit_index = bt.bit_index;
211 bt.bit_index += 1;
212
213 const op_int = @enumToInt(op_ref);
214 if (op_int < Air.Inst.Ref.typed_value_map.len) return;
215 const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
216
217 if (this_bit_index < Liveness.bpi - 1) {
218 const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
219 if (!dies) return;
220 } else {
221 const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
222 const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
223 if (!dies) return;
224 }
208 const dies = bt.lbt.feed();
209 const op_index = Air.refToIndex(op_ref) orelse return;
210 if (!dies) return;
225211 bt.function.processDeath(op_index);
226212 }
227213
......@@ -3291,9 +3277,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT
32913277 return BigTomb{
32923278 .function = self,
32933279 .inst = inst,
3294 .tomb_bits = self.liveness.getTombBits(inst),
3295 .big_tomb_bits = self.liveness.special.get(inst) orelse 0,
3296 .bit_index = 0,
3280 .lbt = self.liveness.iterateBigTomb(inst),
32973281 };
32983282}
32993283
src/arch/arm/CodeGen.zig+5-21
......@@ -224,26 +224,12 @@ const BlockData = struct {
224224const BigTomb = struct {
225225 function: *Self,
226226 inst: Air.Inst.Index,
227 tomb_bits: Liveness.Bpi,
228 big_tomb_bits: u32,
229 bit_index: usize,
227 lbt: Liveness.BigTomb,
230228
231229 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
232 const this_bit_index = bt.bit_index;
233 bt.bit_index += 1;
234
235 const op_int = @enumToInt(op_ref);
236 if (op_int < Air.Inst.Ref.typed_value_map.len) return;
237 const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
238
239 if (this_bit_index < Liveness.bpi - 1) {
240 const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
241 if (!dies) return;
242 } else {
243 const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
244 const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
245 if (!dies) return;
246 }
230 const dies = bt.lbt.feed();
231 const op_index = Air.refToIndex(op_ref) orelse return;
232 if (!dies) return;
247233 bt.function.processDeath(op_index);
248234 }
249235
......@@ -4076,9 +4062,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT
40764062 return BigTomb{
40774063 .function = self,
40784064 .inst = inst,
4079 .tomb_bits = self.liveness.getTombBits(inst),
4080 .big_tomb_bits = self.liveness.special.get(inst) orelse 0,
4081 .bit_index = 0,
4065 .lbt = self.liveness.iterateBigTomb(inst),
40824066 };
40834067}
40844068
src/arch/riscv64/CodeGen.zig+5-21
......@@ -194,26 +194,12 @@ const Reloc = union(enum) {
194194const BigTomb = struct {
195195 function: *Self,
196196 inst: Air.Inst.Index,
197 tomb_bits: Liveness.Bpi,
198 big_tomb_bits: u32,
199 bit_index: usize,
197 lbt: Liveness.BigTomb,
200198
201199 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
202 const this_bit_index = bt.bit_index;
203 bt.bit_index += 1;
204
205 const op_int = @enumToInt(op_ref);
206 if (op_int < Air.Inst.Ref.typed_value_map.len) return;
207 const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
208
209 if (this_bit_index < Liveness.bpi - 1) {
210 const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
211 if (!dies) return;
212 } else {
213 const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
214 const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
215 if (!dies) return;
216 }
200 const dies = bt.lbt.feed();
201 const op_index = Air.refToIndex(op_ref) orelse return;
202 if (!dies) return;
217203 bt.function.processDeath(op_index);
218204 }
219205
......@@ -2198,9 +2184,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT
21982184 return BigTomb{
21992185 .function = self,
22002186 .inst = inst,
2201 .tomb_bits = self.liveness.getTombBits(inst),
2202 .big_tomb_bits = self.liveness.special.get(inst) orelse 0,
2203 .bit_index = 0,
2187 .lbt = self.liveness.iterateBigTomb(inst),
22042188 };
22052189}
22062190
src/arch/x86_64/CodeGen.zig+4-18
......@@ -272,24 +272,12 @@ const BlockData = struct {
272272const BigTomb = struct {
273273 function: *Self,
274274 inst: Air.Inst.Index,
275 tomb_bits: Liveness.Bpi,
276 big_tomb_bits: u32,
277 bit_index: usize,
275 lbt: Liveness.BigTomb,
278276
279277 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
280 const this_bit_index = bt.bit_index;
281 bt.bit_index += 1;
282
278 const dies = bt.lbt.feed();
283279 const op_index = Air.refToIndex(op_ref) orelse return;
284
285 if (this_bit_index < Liveness.bpi - 1) {
286 const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
287 if (!dies) return;
288 } else {
289 const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
290 const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
291 if (!dies) return;
292 }
280 if (!dies) return;
293281 bt.function.processDeath(op_index);
294282 }
295283
......@@ -4845,9 +4833,7 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT
48454833 return BigTomb{
48464834 .function = self,
48474835 .inst = inst,
4848 .tomb_bits = self.liveness.getTombBits(inst),
4849 .big_tomb_bits = self.liveness.special.get(inst) orelse 0,
4850 .bit_index = 0,
4836 .lbt = self.liveness.iterateBigTomb(inst),
48514837 };
48524838}
48534839
src/codegen/c.zig+1-1
......@@ -1348,7 +1348,7 @@ pub const DeclGen = struct {
13481348 return w.writeAll(name);
13491349 },
13501350 .ErrorSet => {
1351 comptime std.debug.assert(Type.initTag(.anyerror).abiSize(builtin.target) == 2);
1351 comptime assert(Type.initTag(.anyerror).abiSize(builtin.target) == 2);
13521352 return w.writeAll("uint16_t");
13531353 },
13541354 .ErrorUnion => {
src/print_air.zig+13-3
......@@ -724,11 +724,21 @@ const Writer = struct {
724724 op_index: usize,
725725 operand: Air.Inst.Ref,
726726 ) @TypeOf(s).Error!void {
727 const dies = if (op_index < Liveness.bpi - 1)
727 const small_tomb_bits = Liveness.bpi - 1;
728 const dies = if (op_index < small_tomb_bits)
728729 w.liveness.operandDies(inst, @intCast(Liveness.OperandInt, op_index))
729730 else blk: {
730 // TODO
731 break :blk false;
731 var extra_index = w.liveness.special.get(inst).?;
732 var tomb_op_index: usize = small_tomb_bits;
733 while (true) {
734 const bits = w.liveness.extra[extra_index];
735 if (op_index < tomb_op_index + 31) {
736 break :blk @truncate(u1, bits >> @intCast(u5, op_index - tomb_op_index)) != 0;
737 }
738 if ((bits >> 31) != 0) break :blk false;
739 extra_index += 1;
740 tomb_op_index += 31;
741 } else unreachable;
732742 };
733743 return w.writeInstRef(s, operand, dies);
734744 }
test/behavior/call.zig+141
......@@ -118,3 +118,144 @@ test "result location of function call argument through runtime condition and st
118118 .e = if (!runtime) .a else .b,
119119 });
120120}
121
122test "function call with 40 arguments" {
123 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
124
125 const S = struct {
126 fn doTheTest(thirty_nine: i32) !void {
127 const result = add(
128 0,
129 1,
130 2,
131 3,
132 4,
133 5,
134 6,
135 7,
136 8,
137 9,
138 10,
139 11,
140 12,
141 13,
142 14,
143 15,
144 16,
145 17,
146 18,
147 19,
148 20,
149 21,
150 22,
151 23,
152 24,
153 25,
154 26,
155 27,
156 28,
157 29,
158 30,
159 31,
160 32,
161 33,
162 34,
163 35,
164 36,
165 37,
166 38,
167 thirty_nine,
168 40,
169 );
170 try expect(result == 820);
171 try expect(thirty_nine == 39);
172 }
173
174 fn add(
175 a0: i32,
176 a1: i32,
177 a2: i32,
178 a3: i32,
179 a4: i32,
180 a5: i32,
181 a6: i32,
182 a7: i32,
183 a8: i32,
184 a9: i32,
185 a10: i32,
186 a11: i32,
187 a12: i32,
188 a13: i32,
189 a14: i32,
190 a15: i32,
191 a16: i32,
192 a17: i32,
193 a18: i32,
194 a19: i32,
195 a20: i32,
196 a21: i32,
197 a22: i32,
198 a23: i32,
199 a24: i32,
200 a25: i32,
201 a26: i32,
202 a27: i32,
203 a28: i32,
204 a29: i32,
205 a30: i32,
206 a31: i32,
207 a32: i32,
208 a33: i32,
209 a34: i32,
210 a35: i32,
211 a36: i32,
212 a37: i32,
213 a38: i32,
214 a39: i32,
215 a40: i32,
216 ) i32 {
217 return a0 +
218 a1 +
219 a2 +
220 a3 +
221 a4 +
222 a5 +
223 a6 +
224 a7 +
225 a8 +
226 a9 +
227 a10 +
228 a11 +
229 a12 +
230 a13 +
231 a14 +
232 a15 +
233 a16 +
234 a17 +
235 a18 +
236 a19 +
237 a20 +
238 a21 +
239 a22 +
240 a23 +
241 a24 +
242 a25 +
243 a26 +
244 a27 +
245 a28 +
246 a29 +
247 a30 +
248 a31 +
249 a32 +
250 a33 +
251 a34 +
252 a35 +
253 a36 +
254 a37 +
255 a38 +
256 a39 +
257 a40;
258 }
259 };
260 try S.doTheTest(39);
261}