authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-05-30 00:05:55-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-06-10 20:47:57-07:00
log61978c8c9473bc06fa1fde75e37374dd330ed614
treec3e8a8a8ed707505df163de9fcdba018b5687bdf
parent66f83f27a2904180bae7797a7c87c6eddc7eebff

InternPool: eliminate indexToKey call graph cycle

Recursion makes this hot function more difficult to profile and optimize. I measured a 1.05x speedup vs the previous commit with the (set of passing) behavior tests. This commit was the last in a series, and the main thing it needed to do was make InternPool.typeOf not call indexToKey(). This required adding a type field to the runtime_value encoding even though it is technically redundant. This could have been avoided with a loop inside typeOf, but I wanted to keep the machine code of that hot function as simple as possible. The variable encoding is still responsible for a relatively small slice of the InternPool data size. I added a function that provides the payload type corresponding to the InternPool.Tag type, which allows for some handy inline switch prongs. Let's start moving the structs that are specific to InternPool.Tag into the corresponding namespace. This will provide type safety if the encoding of InternPool changes for these types later.

1 files changed, 243 insertions(+), 57 deletions(-)

src/InternPool.zig+243-57
......@@ -222,6 +222,11 @@ pub const Key = union(enum) {
222222 /// A comptime function call with a memoized result.
223223 memoized_call: Key.MemoizedCall,
224224
225 pub const TypeValue = struct {
226 ty: Index,
227 val: Index,
228 };
229
225230 pub const IntType = std.builtin.Type.Int;
226231
227232 pub const ErrorUnionType = struct {
......@@ -1372,7 +1377,7 @@ pub const Index = enum(u32) {
13721377 },
13731378
13741379 undef: DataIsIndex,
1375 runtime_value: DataIsIndex,
1380 runtime_value: struct { data: *Tag.TypeValue },
13761381 simple_value: struct { data: SimpleValue },
13771382 ptr_decl: struct { data: *PtrDecl },
13781383 ptr_mut_decl: struct { data: *PtrMutDecl },
......@@ -1383,7 +1388,7 @@ pub const Index = enum(u32) {
13831388 ptr_elem: struct { data: *PtrBaseIndex },
13841389 ptr_field: struct { data: *PtrBaseIndex },
13851390 ptr_slice: struct { data: *PtrSlice },
1386 opt_payload: struct { data: *TypeValue },
1391 opt_payload: struct { data: *Tag.TypeValue },
13871392 opt_null: DataIsIndex,
13881393 int_u8: struct { data: u8 },
13891394 int_u16: struct { data: u16 },
......@@ -1399,7 +1404,7 @@ pub const Index = enum(u32) {
13991404 int_lazy_size: struct { data: *IntLazy },
14001405 error_set_error: struct { data: *Key.Error },
14011406 error_union_error: struct { data: *Key.Error },
1402 error_union_payload: struct { data: *TypeValue },
1407 error_union_payload: struct { data: *Tag.TypeValue },
14031408 enum_literal: struct { data: NullTerminatedString },
14041409 enum_tag: struct { data: *Key.EnumTag },
14051410 float_f16: struct { data: f16 },
......@@ -1410,7 +1415,7 @@ pub const Index = enum(u32) {
14101415 float_c_longdouble_f80: struct { data: *Float80 },
14111416 float_c_longdouble_f128: struct { data: *Float128 },
14121417 float_comptime_float: struct { data: *Float128 },
1413 variable: struct { data: *Variable },
1418 variable: struct { data: *Tag.Variable },
14141419 extern_func: struct { data: *Key.ExternFunc },
14151420 func: struct { data: *Key.Func },
14161421 only_possible_value: DataIsIndex,
......@@ -1769,7 +1774,7 @@ pub const Tag = enum(u8) {
17691774 undef,
17701775 /// A wrapper for values which are comptime-known but should
17711776 /// semantically be runtime-known.
1772 /// `data` is `Index` of the value.
1777 /// data is extra index of `TypeValue`.
17731778 runtime_value,
17741779 /// A value that can be represented with only an enum tag.
17751780 /// data is SimpleValue enum value.
......@@ -1924,8 +1929,117 @@ pub const Tag = enum(u8) {
19241929 /// data is extra index to `Key.MemoizedDecl`
19251930 memoized_decl,
19261931 /// A memoized comptime function call result.
1927 /// data is extra index to `MemoizedFunc`
1932 /// data is extra index to `MemoizedCall`
19281933 memoized_call,
1934
1935 const ErrorUnionType = Key.ErrorUnionType;
1936 const OpaqueType = Key.OpaqueType;
1937 const TypeValue = Key.TypeValue;
1938 const Error = Key.Error;
1939 const EnumTag = Key.EnumTag;
1940 const ExternFunc = Key.ExternFunc;
1941 const Func = Key.Func;
1942 const Union = Key.Union;
1943 const MemoizedDecl = Key.MemoizedDecl;
1944
1945 fn Payload(comptime tag: Tag) type {
1946 return switch (tag) {
1947 .type_int_signed => unreachable,
1948 .type_int_unsigned => unreachable,
1949 .type_array_big => Array,
1950 .type_array_small => Vector,
1951 .type_vector => Vector,
1952 .type_pointer => Pointer,
1953 .type_slice => unreachable,
1954 .type_optional => unreachable,
1955 .type_anyframe => unreachable,
1956 .type_error_union => ErrorUnionType,
1957 .type_error_set => ErrorSet,
1958 .type_inferred_error_set => unreachable,
1959 .type_enum_auto => EnumAuto,
1960 .type_enum_explicit => EnumExplicit,
1961 .type_enum_nonexhaustive => EnumExplicit,
1962 .simple_type => unreachable,
1963 .type_opaque => OpaqueType,
1964 .type_struct => unreachable,
1965 .type_struct_ns => unreachable,
1966 .type_struct_anon => TypeStructAnon,
1967 .type_tuple_anon => TypeStructAnon,
1968 .type_union_tagged => unreachable,
1969 .type_union_untagged => unreachable,
1970 .type_union_safety => unreachable,
1971 .type_function => TypeFunction,
1972
1973 .undef => unreachable,
1974 .runtime_value => TypeValue,
1975 .simple_value => unreachable,
1976 .ptr_decl => PtrDecl,
1977 .ptr_mut_decl => PtrMutDecl,
1978 .ptr_comptime_field => PtrComptimeField,
1979 .ptr_int => PtrBase,
1980 .ptr_eu_payload => PtrBase,
1981 .ptr_opt_payload => PtrBase,
1982 .ptr_elem => PtrBaseIndex,
1983 .ptr_field => PtrBaseIndex,
1984 .ptr_slice => PtrSlice,
1985 .opt_payload => TypeValue,
1986 .opt_null => unreachable,
1987 .int_u8 => unreachable,
1988 .int_u16 => unreachable,
1989 .int_u32 => unreachable,
1990 .int_i32 => unreachable,
1991 .int_usize => unreachable,
1992 .int_comptime_int_u32 => unreachable,
1993 .int_comptime_int_i32 => unreachable,
1994 .int_small => IntSmall,
1995 .int_positive => unreachable,
1996 .int_negative => unreachable,
1997 .int_lazy_align => IntLazy,
1998 .int_lazy_size => IntLazy,
1999 .error_set_error => Error,
2000 .error_union_error => Error,
2001 .error_union_payload => TypeValue,
2002 .enum_literal => unreachable,
2003 .enum_tag => EnumTag,
2004 .float_f16 => unreachable,
2005 .float_f32 => unreachable,
2006 .float_f64 => unreachable,
2007 .float_f80 => unreachable,
2008 .float_f128 => unreachable,
2009 .float_c_longdouble_f80 => unreachable,
2010 .float_c_longdouble_f128 => unreachable,
2011 .float_comptime_float => unreachable,
2012 .variable => Variable,
2013 .extern_func => ExternFunc,
2014 .func => Func,
2015 .only_possible_value => unreachable,
2016 .union_value => Union,
2017 .bytes => Bytes,
2018 .aggregate => Aggregate,
2019 .repeated => Repeated,
2020 .memoized_decl => MemoizedDecl,
2021 .memoized_call => MemoizedCall,
2022 };
2023 }
2024
2025 pub const Variable = struct {
2026 ty: Index,
2027 /// May be `none`.
2028 init: Index,
2029 decl: Module.Decl.Index,
2030 /// Library name if specified.
2031 /// For example `extern "c" var stderrp = ...` would have 'c' as library name.
2032 lib_name: OptionalNullTerminatedString,
2033 flags: Flags,
2034
2035 pub const Flags = packed struct(u32) {
2036 is_extern: bool,
2037 is_const: bool,
2038 is_threadlocal: bool,
2039 is_weak_linkage: bool,
2040 _: u28 = 0,
2041 };
2042 };
19292043};
19302044
19312045/// Trailing:
......@@ -2137,11 +2251,6 @@ pub const Array = struct {
21372251 }
21382252};
21392253
2140pub const TypeValue = struct {
2141 ty: Index,
2142 val: Index,
2143};
2144
21452254/// Trailing:
21462255/// 0. field name: NullTerminatedString for each fields_len; declaration order
21472256/// 1. tag value: Index for each fields_len; declaration order
......@@ -2190,25 +2299,6 @@ pub const PackedU64 = packed struct(u64) {
21902299 }
21912300};
21922301
2193pub const Variable = struct {
2194 /// This is a value if has_init is true, otherwise a type.
2195 init: Index,
2196 decl: Module.Decl.Index,
2197 /// Library name if specified.
2198 /// For example `extern "c" var stderrp = ...` would have 'c' as library name.
2199 lib_name: OptionalNullTerminatedString,
2200 flags: Flags,
2201
2202 pub const Flags = packed struct(u32) {
2203 has_init: bool,
2204 is_extern: bool,
2205 is_const: bool,
2206 is_threadlocal: bool,
2207 is_weak_linkage: bool,
2208 _: u27 = 0,
2209 };
2210};
2211
22122302pub const PtrDecl = struct {
22132303 ty: Index,
22142304 decl: Module.Decl.Index,
......@@ -2569,19 +2659,13 @@ pub fn indexToKey(ip: *const InternPool, index: Index) Key {
25692659 .type_function => .{ .func_type = ip.indexToKeyFuncType(data) },
25702660
25712661 .undef => .{ .undef = @intToEnum(Index, data) },
2572 .runtime_value => {
2573 const val = @intToEnum(Index, data);
2574 return .{ .runtime_value = .{
2575 .ty = ip.typeOf(val),
2576 .val = val,
2577 } };
2578 },
2662 .runtime_value => .{ .runtime_value = ip.extraData(Tag.TypeValue, data) },
25792663 .opt_null => .{ .opt = .{
25802664 .ty = @intToEnum(Index, data),
25812665 .val = .none,
25822666 } },
25832667 .opt_payload => {
2584 const extra = ip.extraData(TypeValue, data);
2668 const extra = ip.extraData(Tag.TypeValue, data);
25852669 return .{ .opt = .{
25862670 .ty = extra.ty,
25872671 .val = extra.val,
......@@ -2806,10 +2890,10 @@ pub fn indexToKey(ip: *const InternPool, index: Index) Key {
28062890 .storage = .{ .f128 = ip.extraData(Float128, data).get() },
28072891 } },
28082892 .variable => {
2809 const extra = ip.extraData(Variable, data);
2893 const extra = ip.extraData(Tag.Variable, data);
28102894 return .{ .variable = .{
2811 .ty = if (extra.flags.has_init) ip.typeOf(extra.init) else extra.init,
2812 .init = if (extra.flags.has_init) extra.init else .none,
2895 .ty = extra.ty,
2896 .init = extra.init,
28132897 .decl = extra.decl,
28142898 .lib_name = extra.lib_name,
28152899 .is_extern = extra.flags.is_extern,
......@@ -2887,7 +2971,7 @@ pub fn indexToKey(ip: *const InternPool, index: Index) Key {
28872971 } };
28882972 },
28892973 .error_union_payload => {
2890 const extra = ip.extraData(TypeValue, data);
2974 const extra = ip.extraData(Tag.TypeValue, data);
28912975 return .{ .error_union = .{
28922976 .ty = extra.ty,
28932977 .val = .{ .payload = extra.val },
......@@ -3124,7 +3208,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {
31243208 assert(runtime_value.ty == ip.typeOf(runtime_value.val));
31253209 ip.items.appendAssumeCapacity(.{
31263210 .tag = .runtime_value,
3127 .data = @enumToInt(runtime_value.val),
3211 .data = try ip.addExtra(gpa, runtime_value),
31283212 });
31293213 },
31303214
......@@ -3266,12 +3350,12 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {
32663350 if (has_init) assert(variable.ty == ip.typeOf(variable.init));
32673351 ip.items.appendAssumeCapacity(.{
32683352 .tag = .variable,
3269 .data = try ip.addExtra(gpa, Variable{
3270 .init = if (has_init) variable.init else variable.ty,
3353 .data = try ip.addExtra(gpa, Tag.Variable{
3354 .ty = variable.ty,
3355 .init = variable.init,
32713356 .decl = variable.decl,
32723357 .lib_name = variable.lib_name,
32733358 .flags = .{
3274 .has_init = has_init,
32753359 .is_extern = variable.is_extern,
32763360 .is_const = variable.is_const,
32773361 .is_threadlocal = variable.is_threadlocal,
......@@ -3431,7 +3515,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {
34313515 .data = @enumToInt(opt.ty),
34323516 } else .{
34333517 .tag = .opt_payload,
3434 .data = try ip.addExtra(gpa, TypeValue{
3518 .data = try ip.addExtra(gpa, Tag.TypeValue{
34353519 .ty = opt.ty,
34363520 .val = opt.val,
34373521 }),
......@@ -3642,7 +3726,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {
36423726 },
36433727 .payload => |payload| .{
36443728 .tag = .error_union_payload,
3645 .data = try ip.addExtra(gpa, TypeValue{
3729 .data = try ip.addExtra(gpa, Tag.TypeValue{
36463730 .ty = error_union.ty,
36473731 .val = payload,
36483732 }),
......@@ -4222,7 +4306,7 @@ fn addExtraAssumeCapacity(ip: *InternPool, extra: anytype) u32 {
42224306 TypeFunction.Flags => @bitCast(u32, @field(extra, field.name)),
42234307 Pointer.PackedOffset => @bitCast(u32, @field(extra, field.name)),
42244308 Pointer.VectorIndex => @enumToInt(@field(extra, field.name)),
4225 Variable.Flags => @bitCast(u32, @field(extra, field.name)),
4309 Tag.Variable.Flags => @bitCast(u32, @field(extra, field.name)),
42264310 else => @compileError("bad field type: " ++ @typeName(field.type)),
42274311 });
42284312 }
......@@ -4290,7 +4374,7 @@ fn extraDataTrail(ip: InternPool, comptime T: type, index: usize) struct { data:
42904374 TypeFunction.Flags => @bitCast(TypeFunction.Flags, int32),
42914375 Pointer.PackedOffset => @bitCast(Pointer.PackedOffset, int32),
42924376 Pointer.VectorIndex => @intToEnum(Pointer.VectorIndex, int32),
4293 Variable.Flags => @bitCast(Variable.Flags, int32),
4377 Tag.Variable.Flags => @bitCast(Tag.Variable.Flags, int32),
42944378 else => @compileError("bad field type: " ++ @typeName(field.type)),
42954379 };
42964380 }
......@@ -4737,7 +4821,7 @@ pub fn isAggregateType(ip: InternPool, ty: Index) bool {
47374821/// The is only legal because the initializer is not part of the hash.
47384822pub fn mutateVarInit(ip: *InternPool, index: Index, init_index: Index) void {
47394823 assert(ip.items.items(.tag)[@enumToInt(index)] == .variable);
4740 const field_index = inline for (@typeInfo(Variable).Struct.fields, 0..) |field, field_index| {
4824 const field_index = inline for (@typeInfo(Tag.Variable).Struct.fields, 0..) |field, field_index| {
47414825 if (comptime std.mem.eql(u8, field.name, "init")) break field_index;
47424826 } else unreachable;
47434827 ip.extra.items[ip.items.items(.data)[@enumToInt(index)] + field_index] = @enumToInt(init_index);
......@@ -4847,7 +4931,7 @@ fn dumpFallible(ip: InternPool, arena: Allocator) anyerror!void {
48474931 },
48484932
48494933 .undef => 0,
4850 .runtime_value => 0,
4934 .runtime_value => @sizeOf(Tag.TypeValue),
48514935 .simple_type => 0,
48524936 .simple_value => 0,
48534937 .ptr_decl => @sizeOf(PtrDecl),
......@@ -4860,7 +4944,7 @@ fn dumpFallible(ip: InternPool, arena: Allocator) anyerror!void {
48604944 .ptr_field => @sizeOf(PtrBaseIndex),
48614945 .ptr_slice => @sizeOf(PtrSlice),
48624946 .opt_null => 0,
4863 .opt_payload => @sizeOf(TypeValue),
4947 .opt_payload => @sizeOf(Tag.TypeValue),
48644948 .int_u8 => 0,
48654949 .int_u16 => 0,
48664950 .int_u32 => 0,
......@@ -4880,7 +4964,7 @@ fn dumpFallible(ip: InternPool, arena: Allocator) anyerror!void {
48804964 .int_lazy_align, .int_lazy_size => @sizeOf(IntLazy),
48814965
48824966 .error_set_error, .error_union_error => @sizeOf(Key.Error),
4883 .error_union_payload => @sizeOf(TypeValue),
4967 .error_union_payload => @sizeOf(Tag.TypeValue),
48844968 .enum_literal => 0,
48854969 .enum_tag => @sizeOf(Key.EnumTag),
48864970
......@@ -4905,7 +4989,7 @@ fn dumpFallible(ip: InternPool, arena: Allocator) anyerror!void {
49054989 .float_c_longdouble_f80 => @sizeOf(Float80),
49064990 .float_c_longdouble_f128 => @sizeOf(Float128),
49074991 .float_comptime_float => @sizeOf(Float128),
4908 .variable => @sizeOf(Variable) + @sizeOf(Module.Decl),
4992 .variable => @sizeOf(Tag.Variable) + @sizeOf(Module.Decl),
49094993 .extern_func => @sizeOf(Key.ExternFunc) + @sizeOf(Module.Decl),
49104994 .func => @sizeOf(Key.Func) + @sizeOf(Module.Fn) + @sizeOf(Module.Decl),
49114995 .only_possible_value => 0,
......@@ -5179,6 +5263,7 @@ pub fn typeOf(ip: InternPool, index: Index) Index {
51795263 .generic_poison_type,
51805264 .empty_struct_type,
51815265 => .type_type,
5266
51825267 .undef => .undefined_type,
51835268 .zero, .one, .negative_one => .comptime_int_type,
51845269 .zero_usize, .one_usize => .usize_type,
......@@ -5190,8 +5275,109 @@ pub fn typeOf(ip: InternPool, index: Index) Index {
51905275 .bool_true, .bool_false => .bool_type,
51915276 .empty_struct => .empty_struct_type,
51925277 .generic_poison => .generic_poison_type,
5193 .var_args_param_type, .none => unreachable,
5194 _ => ip.indexToKey(index).typeOf(),
5278
5279 // This optimization on tags is needed so that indexToKey can call
5280 // typeOf without being recursive.
5281 _ => switch (ip.items.items(.tag)[@enumToInt(index)]) {
5282 .type_int_signed,
5283 .type_int_unsigned,
5284 .type_array_big,
5285 .type_array_small,
5286 .type_vector,
5287 .type_pointer,
5288 .type_slice,
5289 .type_optional,
5290 .type_anyframe,
5291 .type_error_union,
5292 .type_error_set,
5293 .type_inferred_error_set,
5294 .type_enum_auto,
5295 .type_enum_explicit,
5296 .type_enum_nonexhaustive,
5297 .simple_type,
5298 .type_opaque,
5299 .type_struct,
5300 .type_struct_ns,
5301 .type_struct_anon,
5302 .type_tuple_anon,
5303 .type_union_tagged,
5304 .type_union_untagged,
5305 .type_union_safety,
5306 .type_function,
5307 => .type_type,
5308
5309 .undef,
5310 .opt_null,
5311 .only_possible_value,
5312 => @intToEnum(Index, ip.items.items(.data)[@enumToInt(index)]),
5313
5314 .simple_value => unreachable, // handled via Index above
5315
5316 inline .ptr_decl,
5317 .ptr_mut_decl,
5318 .ptr_comptime_field,
5319 .ptr_int,
5320 .ptr_eu_payload,
5321 .ptr_opt_payload,
5322 .ptr_elem,
5323 .ptr_field,
5324 .ptr_slice,
5325 .opt_payload,
5326 .error_union_payload,
5327 .runtime_value,
5328 .int_small,
5329 .int_lazy_align,
5330 .int_lazy_size,
5331 .error_set_error,
5332 .error_union_error,
5333 .enum_tag,
5334 .variable,
5335 .extern_func,
5336 .func,
5337 .union_value,
5338 .bytes,
5339 .aggregate,
5340 .repeated,
5341 => |t| {
5342 const extra_index = ip.items.items(.data)[@enumToInt(index)];
5343 const field_index = std.meta.fieldIndex(t.Payload(), "ty").?;
5344 return @intToEnum(Index, ip.extra.items[extra_index + field_index]);
5345 },
5346
5347 .int_u8 => .u8_type,
5348 .int_u16 => .u16_type,
5349 .int_u32 => .u32_type,
5350 .int_i32 => .i32_type,
5351 .int_usize => .usize_type,
5352
5353 .int_comptime_int_u32,
5354 .int_comptime_int_i32,
5355 => .comptime_int_type,
5356
5357 // Note these are stored in limbs data, not extra data.
5358 .int_positive,
5359 .int_negative,
5360 => ip.limbData(Int, ip.items.items(.data)[@enumToInt(index)]).ty,
5361
5362 .enum_literal => .enum_literal_type,
5363 .float_f16 => .f16_type,
5364 .float_f32 => .f32_type,
5365 .float_f64 => .f64_type,
5366 .float_f80 => .f80_type,
5367 .float_f128 => .f128_type,
5368
5369 .float_c_longdouble_f80,
5370 .float_c_longdouble_f128,
5371 => .c_longdouble_type,
5372
5373 .float_comptime_float => .comptime_float_type,
5374
5375 .memoized_decl => unreachable,
5376 .memoized_call => unreachable,
5377 },
5378
5379 .var_args_param_type => unreachable,
5380 .none => unreachable,
51955381 };
51965382}
51975383