authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-09-17 18:12:17-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-09-18 00:09:30-07:00
logc970dbdfeca60de227bba53b155c8374ae90f919
tree5e5a06f78fdb339d98af490b348759575d9c8b52
parent48e2ba3b3c1e224b59e3c97ed462ac88df4d8a4b

LLVM: cache LLVM struct field indexes

This is an optimization to avoid O(N) field index lookups. It's also nicer in terms of DRY; the only tradeoff is memory usage.

1 files changed, 63 insertions(+), 107 deletions(-)

src/codegen/llvm.zig+63-107
...@@ -825,6 +825,18 @@ pub const Object = struct {...@@ -825,6 +825,18 @@ pub const Object = struct {
825 /// Memoizes a null `?usize` value.825 /// Memoizes a null `?usize` value.
826 null_opt_usize: Builder.Constant,826 null_opt_usize: Builder.Constant,
827827
828 /// When an LLVM struct type is created, an entry is inserted into this
829 /// table for every zig source field of the struct that has a corresponding
830 /// LLVM struct field. comptime fields and 0 bit fields are not included.
831 /// The value is the LLVM struct field index.
832 /// This is denormalized data.
833 struct_field_map: std.AutoHashMapUnmanaged(ZigStructField, c_uint),
834
835 const ZigStructField = struct {
836 struct_ty: InternPool.Index,
837 field_index: u32,
838 };
839
828 pub const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);840 pub const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);
829841
830 /// This is an ArrayHashMap as opposed to a HashMap because in `flushModule` we842 /// This is an ArrayHashMap as opposed to a HashMap because in `flushModule` we
...@@ -979,6 +991,7 @@ pub const Object = struct {...@@ -979,6 +991,7 @@ pub const Object = struct {
979 .error_name_table = .none,991 .error_name_table = .none,
980 .extern_collisions = .{},992 .extern_collisions = .{},
981 .null_opt_usize = .no_init,993 .null_opt_usize = .no_init,
994 .struct_field_map = .{},
982 };995 };
983 }996 }
984997
...@@ -994,6 +1007,7 @@ pub const Object = struct {...@@ -994,6 +1007,7 @@ pub const Object = struct {
994 self.type_map.deinit(gpa);1007 self.type_map.deinit(gpa);
995 self.extern_collisions.deinit(gpa);1008 self.extern_collisions.deinit(gpa);
996 self.builder.deinit();1009 self.builder.deinit();
1010 self.struct_field_map.deinit(gpa);
997 self.* = undefined;1011 self.* = undefined;
998 }1012 }
9991013
...@@ -3274,7 +3288,10 @@ pub const Object = struct {...@@ -3274,7 +3288,10 @@ pub const Object = struct {
32743288
3275 var llvm_field_types = std.ArrayListUnmanaged(Builder.Type){};3289 var llvm_field_types = std.ArrayListUnmanaged(Builder.Type){};
3276 defer llvm_field_types.deinit(o.gpa);3290 defer llvm_field_types.deinit(o.gpa);
3291 // Although we can estimate how much capacity to add, these cannot be
3292 // relied upon because of the recursive calls to lowerType below.
3277 try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_obj.fields.count());3293 try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_obj.fields.count());
3294 try o.struct_field_map.ensureUnusedCapacity(o.gpa, @intCast(struct_obj.fields.count()));
32783295
3279 comptime assert(struct_layout_version == 2);3296 comptime assert(struct_layout_version == 2);
3280 var offset: u64 = 0;3297 var offset: u64 = 0;
...@@ -3296,6 +3313,10 @@ pub const Object = struct {...@@ -3296,6 +3313,10 @@ pub const Object = struct {
3296 o.gpa,3313 o.gpa,
3297 try o.builder.arrayType(padding_len, .i8),3314 try o.builder.arrayType(padding_len, .i8),
3298 );3315 );
3316 try o.struct_field_map.put(o.gpa, .{
3317 .struct_ty = t.toIntern(),
3318 .field_index = field_and_index.index,
3319 }, @intCast(llvm_field_types.items.len));
3299 try llvm_field_types.append(o.gpa, try o.lowerType(field.ty));3320 try llvm_field_types.append(o.gpa, try o.lowerType(field.ty));
33003321
3301 offset += field.ty.abiSize(mod);3322 offset += field.ty.abiSize(mod);
...@@ -3319,7 +3340,10 @@ pub const Object = struct {...@@ -3319,7 +3340,10 @@ pub const Object = struct {
3319 .anon_struct_type => |anon_struct_type| {3340 .anon_struct_type => |anon_struct_type| {
3320 var llvm_field_types: std.ArrayListUnmanaged(Builder.Type) = .{};3341 var llvm_field_types: std.ArrayListUnmanaged(Builder.Type) = .{};
3321 defer llvm_field_types.deinit(o.gpa);3342 defer llvm_field_types.deinit(o.gpa);
3343 // Although we can estimate how much capacity to add, these cannot be
3344 // relied upon because of the recursive calls to lowerType below.
3322 try llvm_field_types.ensureUnusedCapacity(o.gpa, anon_struct_type.types.len);3345 try llvm_field_types.ensureUnusedCapacity(o.gpa, anon_struct_type.types.len);
3346 try o.struct_field_map.ensureUnusedCapacity(o.gpa, anon_struct_type.types.len);
33233347
3324 comptime assert(struct_layout_version == 2);3348 comptime assert(struct_layout_version == 2);
3325 var offset: u64 = 0;3349 var offset: u64 = 0;
...@@ -3328,7 +3352,8 @@ pub const Object = struct {...@@ -3328,7 +3352,8 @@ pub const Object = struct {
3328 for (3352 for (
3329 anon_struct_type.types.get(ip),3353 anon_struct_type.types.get(ip),
3330 anon_struct_type.values.get(ip),3354 anon_struct_type.values.get(ip),
3331 ) |field_ty, field_val| {3355 0..,
3356 ) |field_ty, field_val, field_index| {
3332 if (field_val != .none or !field_ty.toType().hasRuntimeBits(mod)) continue;3357 if (field_val != .none or !field_ty.toType().hasRuntimeBits(mod)) continue;
33333358
3334 const field_align = field_ty.toType().abiAlignment(mod);3359 const field_align = field_ty.toType().abiAlignment(mod);
...@@ -3341,6 +3366,10 @@ pub const Object = struct {...@@ -3341,6 +3366,10 @@ pub const Object = struct {
3341 o.gpa,3366 o.gpa,
3342 try o.builder.arrayType(padding_len, .i8),3367 try o.builder.arrayType(padding_len, .i8),
3343 );3368 );
3369 try o.struct_field_map.put(o.gpa, .{
3370 .struct_ty = t.toIntern(),
3371 .field_index = @intCast(field_index),
3372 }, @intCast(llvm_field_types.items.len));
3344 try llvm_field_types.append(o.gpa, try o.lowerType(field_ty.toType()));3373 try llvm_field_types.append(o.gpa, try o.lowerType(field_ty.toType()));
33453374
3346 offset += field_ty.toType().abiSize(mod);3375 offset += field_ty.toType().abiSize(mod);
...@@ -4237,9 +4266,9 @@ pub const Object = struct {...@@ -4237,9 +4266,9 @@ pub const Object = struct {
4237 try o.lowerType(parent_ty),4266 try o.lowerType(parent_ty),
4238 parent_ptr,4267 parent_ptr,
4239 null,4268 null,
4240 if (llvmField(parent_ty, field_index, mod)) |llvm_field| &.{4269 if (o.llvmFieldIndex(parent_ty, field_index)) |llvm_field_index| &.{
4241 try o.builder.intConst(.i32, 0),4270 try o.builder.intConst(.i32, 0),
4242 try o.builder.intConst(.i32, llvm_field.index),4271 try o.builder.intConst(.i32, llvm_field_index),
4243 } else &.{4272 } else &.{
4244 try o.builder.intConst(.i32, @intFromBool(4273 try o.builder.intConst(.i32, @intFromBool(
4245 parent_ty.hasRuntimeBitsIgnoreComptime(mod),4274 parent_ty.hasRuntimeBitsIgnoreComptime(mod),
...@@ -4396,6 +4425,13 @@ pub const Object = struct {...@@ -4396,6 +4425,13 @@ pub const Object = struct {
4396 try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = alignment }, &o.builder);4425 try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = alignment }, &o.builder);
4397 if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);4426 if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);
4398 }4427 }
4428
4429 fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint {
4430 return o.struct_field_map.get(.{
4431 .struct_ty = struct_ty.toIntern(),
4432 .field_index = @intCast(field_index),
4433 });
4434 }
4399};4435};
44004436
4401pub const DeclGen = struct {4437pub const DeclGen = struct {
...@@ -6142,7 +6178,7 @@ pub const FuncGen = struct {...@@ -6142,7 +6178,7 @@ pub const FuncGen = struct {
6142 return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");6178 return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
6143 },6179 },
6144 else => {6180 else => {
6145 const llvm_field_index = llvmField(struct_ty, field_index, mod).?.index;6181 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
6146 return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, "");6182 return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, "");
6147 },6183 },
6148 },6184 },
...@@ -6169,23 +6205,25 @@ pub const FuncGen = struct {...@@ -6169,23 +6205,25 @@ pub const FuncGen = struct {
61696205
6170 switch (struct_ty.zigTypeTag(mod)) {6206 switch (struct_ty.zigTypeTag(mod)) {
6171 .Struct => {6207 .Struct => {
6172 assert(struct_ty.containerLayout(mod) != .Packed);6208 const layout = struct_ty.containerLayout(mod);
6173 const llvm_field = llvmField(struct_ty, field_index, mod).?;6209 assert(layout != .Packed);
6174 const struct_llvm_ty = try o.lowerType(struct_ty);6210 const struct_llvm_ty = try o.lowerType(struct_ty);
6211 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
6175 const field_ptr =6212 const field_ptr =
6176 try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field.index, "");6213 try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, "");
6214 const alignment = struct_ty.structFieldAlign(field_index, mod);
6177 const field_ptr_ty = try mod.ptrType(.{6215 const field_ptr_ty = try mod.ptrType(.{
6178 .child = llvm_field.ty.toIntern(),6216 .child = field_ty.toIntern(),
6179 .flags = .{6217 .flags = .{
6180 .alignment = InternPool.Alignment.fromNonzeroByteUnits(llvm_field.alignment),6218 .alignment = InternPool.Alignment.fromNonzeroByteUnits(alignment),
6181 },6219 },
6182 });6220 });
6183 if (isByRef(field_ty, mod)) {6221 if (isByRef(field_ty, mod)) {
6184 if (canElideLoad(self, body_tail))6222 if (canElideLoad(self, body_tail))
6185 return field_ptr;6223 return field_ptr;
61866224
6187 assert(llvm_field.alignment != 0);6225 assert(alignment != 0);
6188 const field_alignment = Builder.Alignment.fromByteUnits(llvm_field.alignment);6226 const field_alignment = Builder.Alignment.fromByteUnits(alignment);
6189 return self.loadByRef(field_ptr, field_ty, field_alignment, .normal);6227 return self.loadByRef(field_ptr, field_ty, field_alignment, .normal);
6190 } else {6228 } else {
6191 return self.load(field_ptr, field_ptr_ty);6229 return self.load(field_ptr, field_ptr_ty);
...@@ -7080,15 +7118,17 @@ pub const FuncGen = struct {...@@ -7080,15 +7118,17 @@ pub const FuncGen = struct {
7080 const field_index = ty_pl.payload;7118 const field_index = ty_pl.payload;
70817119
7082 const mod = o.module;7120 const mod = o.module;
7083 const llvm_field = llvmField(struct_ty, field_index, mod).?;
7084 const struct_llvm_ty = try o.lowerType(struct_ty);7121 const struct_llvm_ty = try o.lowerType(struct_ty);
7122 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
7085 assert(self.err_ret_trace != .none);7123 assert(self.err_ret_trace != .none);
7086 const field_ptr =7124 const field_ptr =
7087 try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field.index, "");7125 try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, "");
7126 const field_alignment = struct_ty.structFieldAlign(field_index, mod);
7127 const field_ty = struct_ty.structFieldType(field_index, mod);
7088 const field_ptr_ty = try mod.ptrType(.{7128 const field_ptr_ty = try mod.ptrType(.{
7089 .child = llvm_field.ty.toIntern(),7129 .child = field_ty.toIntern(),
7090 .flags = .{7130 .flags = .{
7091 .alignment = InternPool.Alignment.fromNonzeroByteUnits(llvm_field.alignment),7131 .alignment = InternPool.Alignment.fromNonzeroByteUnits(field_alignment),
7092 },7132 },
7093 });7133 });
7094 return self.load(field_ptr, field_ptr_ty);7134 return self.load(field_ptr, field_ptr_ty);
...@@ -7640,8 +7680,8 @@ pub const FuncGen = struct {...@@ -7640,8 +7680,8 @@ pub const FuncGen = struct {
7640 const result_val = try self.wip.extractValue(results, &.{0}, "");7680 const result_val = try self.wip.extractValue(results, &.{0}, "");
7641 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");7681 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
76427682
7643 const result_index = llvmField(inst_ty, 0, mod).?.index;7683 const result_index = o.llvmFieldIndex(inst_ty, 0).?;
7644 const overflow_index = llvmField(inst_ty, 1, mod).?.index;7684 const overflow_index = o.llvmFieldIndex(inst_ty, 1).?;
76457685
7646 if (isByRef(inst_ty, mod)) {7686 if (isByRef(inst_ty, mod)) {
7647 const result_alignment = Builder.Alignment.fromByteUnits(inst_ty.abiAlignment(mod));7687 const result_alignment = Builder.Alignment.fromByteUnits(inst_ty.abiAlignment(mod));
...@@ -7998,8 +8038,8 @@ pub const FuncGen = struct {...@@ -7998,8 +8038,8 @@ pub const FuncGen = struct {
79988038
7999 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");8039 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
80008040
8001 const result_index = llvmField(dest_ty, 0, mod).?.index;8041 const result_index = o.llvmFieldIndex(dest_ty, 0).?;
8002 const overflow_index = llvmField(dest_ty, 1, mod).?.index;8042 const overflow_index = o.llvmFieldIndex(dest_ty, 1).?;
80038043
8004 if (isByRef(dest_ty, mod)) {8044 if (isByRef(dest_ty, mod)) {
8005 const result_alignment = Builder.Alignment.fromByteUnits(dest_ty.abiAlignment(mod));8045 const result_alignment = Builder.Alignment.fromByteUnits(dest_ty.abiAlignment(mod));
...@@ -9616,7 +9656,7 @@ pub const FuncGen = struct {...@@ -9616,7 +9656,7 @@ pub const FuncGen = struct {
9616 if ((try result_ty.structFieldValueComptime(mod, i)) != null) continue;9656 if ((try result_ty.structFieldValueComptime(mod, i)) != null) continue;
96179657
9618 const llvm_elem = try self.resolveInst(elem);9658 const llvm_elem = try self.resolveInst(elem);
9619 const llvm_i = llvmField(result_ty, i, mod).?.index;9659 const llvm_i = o.llvmFieldIndex(result_ty, i).?;
9620 const field_ptr =9660 const field_ptr =
9621 try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, "");9661 try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, "");
9622 const field_ptr_ty = try mod.ptrType(.{9662 const field_ptr_ty = try mod.ptrType(.{
...@@ -9637,7 +9677,7 @@ pub const FuncGen = struct {...@@ -9637,7 +9677,7 @@ pub const FuncGen = struct {
9637 if ((try result_ty.structFieldValueComptime(mod, i)) != null) continue;9677 if ((try result_ty.structFieldValueComptime(mod, i)) != null) continue;
96389678
9639 const llvm_elem = try self.resolveInst(elem);9679 const llvm_elem = try self.resolveInst(elem);
9640 const llvm_i = llvmField(result_ty, i, mod).?.index;9680 const llvm_i = o.llvmFieldIndex(result_ty, i).?;
9641 result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, "");9681 result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, "");
9642 }9682 }
9643 return result;9683 return result;
...@@ -10059,8 +10099,8 @@ pub const FuncGen = struct {...@@ -10059,8 +10099,8 @@ pub const FuncGen = struct {
10059 else => {10099 else => {
10060 const struct_llvm_ty = try o.lowerPtrElemTy(struct_ty);10100 const struct_llvm_ty = try o.lowerPtrElemTy(struct_ty);
1006110101
10062 if (llvmField(struct_ty, field_index, mod)) |llvm_field| {10102 if (o.llvmFieldIndex(struct_ty, field_index)) |llvm_field_index| {
10063 return self.wip.gepStruct(struct_llvm_ty, struct_ptr, llvm_field.index, "");10103 return self.wip.gepStruct(struct_llvm_ty, struct_ptr, llvm_field_index, "");
10064 } else {10104 } else {
10065 // If we found no index then this means this is a zero sized field at the10105 // If we found no index then this means this is a zero sized field at the
10066 // end of the struct. Treat our struct pointer as an array of two and get10106 // end of the struct. Treat our struct pointer as an array of two and get
...@@ -10527,90 +10567,6 @@ fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, targ...@@ -10527,90 +10567,6 @@ fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, targ
10527 };10567 };
10528}10568}
1052910569
10530const LlvmField = struct {
10531 index: c_uint,
10532 ty: Type,
10533 alignment: u32,
10534};
10535
10536/// Take into account 0 bit fields and padding. Returns null if an llvm
10537/// field could not be found.
10538/// This only happens if you want the field index of a zero sized field at
10539/// the end of the struct.
10540fn llvmField(ty: Type, field_index: usize, mod: *Module) ?LlvmField {
10541 // Detects where we inserted extra padding fields so that we can skip
10542 // over them in this function.
10543 comptime assert(struct_layout_version == 2);
10544 var offset: u64 = 0;
10545 var big_align: u32 = 0;
10546
10547 const ip = &mod.intern_pool;
10548 const struct_type = switch (ip.indexToKey(ty.toIntern())) {
10549 .anon_struct_type => |tuple| {
10550 var llvm_field_index: c_uint = 0;
10551 for (tuple.types.get(ip), tuple.values.get(ip), 0..) |field_ty, field_val, i| {
10552 if (field_val != .none or !field_ty.toType().hasRuntimeBits(mod)) continue;
10553
10554 const field_align = field_ty.toType().abiAlignment(mod);
10555 big_align = @max(big_align, field_align);
10556 const prev_offset = offset;
10557 offset = std.mem.alignForward(u64, offset, field_align);
10558
10559 const padding_len = offset - prev_offset;
10560 if (padding_len > 0) {
10561 llvm_field_index += 1;
10562 }
10563
10564 if (field_index <= i) {
10565 return .{
10566 .index = llvm_field_index,
10567 .ty = field_ty.toType(),
10568 .alignment = field_align,
10569 };
10570 }
10571
10572 llvm_field_index += 1;
10573 offset += field_ty.toType().abiSize(mod);
10574 }
10575 return null;
10576 },
10577 .struct_type => |s| s,
10578 else => unreachable,
10579 };
10580 const struct_obj = mod.structPtrUnwrap(struct_type.index).?;
10581 const layout = struct_obj.layout;
10582 assert(layout != .Packed);
10583
10584 var llvm_field_index: c_uint = 0;
10585 var it = struct_obj.runtimeFieldIterator(mod);
10586 while (it.next()) |field_and_index| {
10587 const field = field_and_index.field;
10588 const field_align = field.alignment(mod, layout);
10589 big_align = @max(big_align, field_align);
10590 const prev_offset = offset;
10591 offset = std.mem.alignForward(u64, offset, field_align);
10592
10593 const padding_len = offset - prev_offset;
10594 if (padding_len > 0) {
10595 llvm_field_index += 1;
10596 }
10597
10598 if (field_index == field_and_index.index) {
10599 return .{
10600 .index = llvm_field_index,
10601 .ty = field.ty,
10602 .alignment = field_align,
10603 };
10604 }
10605
10606 llvm_field_index += 1;
10607 offset += field.ty.abiSize(mod);
10608 } else {
10609 // We did not find an llvm field that corresponds to this zig field.
10610 return null;
10611 }
10612}
10613
10614fn firstParamSRet(fn_info: InternPool.Key.FuncType, mod: *Module) bool {10570fn firstParamSRet(fn_info: InternPool.Key.FuncType, mod: *Module) bool {
10615 const return_type = fn_info.return_type.toType();10571 const return_type = fn_info.return_type.toType();
10616 if (!return_type.hasRuntimeBitsIgnoreComptime(mod)) return false;10572 if (!return_type.hasRuntimeBitsIgnoreComptime(mod)) return false;