authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-31 01:10:50-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-31 01:20:45-07:00
logcf88cf2657d721c68055a284e8c498a18639f74c
treedc033a7f498a892bc7756cee8d4a46bbb56340c6
parent91ad96b88a88016043cb0d069aa9db47747170b6

std: make ArrayHashMap eql function accept an additional param

which is the index of the key that already exists in the hash map. This enables the use case of using `AutoArrayHashMap(void, void)` which may seem surprising at first, but is actually pretty handy! This commit includes a proof-of-concept of how I want to use it, with a new InternArena abstraction for stage2 that provides a compact way to store values (and types) in an "internment arena", thus making types stored exactly once (per arena), representable with a single u32 as a reference to a type within an InternArena, and comparable with a simple u32 integer comparison. If both types are in the same InternArena, you can check if they are equal by seeing if their index is the same. What's neat about `AutoArrayHashMap(void, void)` is that it allows us to look up the indexes by key, *without actually storing the keys*. Instead, keys are treated as ephemeral values that are constructed as needed. As a result, we have an extremely efficient encoding of types and values, represented only by three arrays, which has no pointers, and can therefore be serialized and deserialized by a single writev/readv call. The `map` field is denormalized data and can be computed from the other two fields. This is in contrast to our current Type/Value system which makes extensive use of pointers. The test at the bottom of InternArena.zig passes in this commit.

5 files changed, 357 insertions(+), 25 deletions(-)

lib/std/array_hash_map.zig+21-17
......@@ -37,8 +37,9 @@ pub const StringContext = struct {
3737 _ = self;
3838 return hashString(s);
3939 }
40 pub fn eql(self: @This(), a: []const u8, b: []const u8) bool {
40 pub fn eql(self: @This(), a: []const u8, b: []const u8, b_index: usize) bool {
4141 _ = self;
42 _ = b_index;
4243 return eqlString(a, b);
4344 }
4445};
......@@ -76,7 +77,7 @@ pub fn ArrayHashMap(
7677 comptime Context: type,
7778 comptime store_hash: bool,
7879) type {
79 comptime std.hash_map.verifyContext(Context, K, K, u32);
80 comptime std.hash_map.verifyContext(Context, K, K, u32, true);
8081 return struct {
8182 unmanaged: Unmanaged,
8283 allocator: Allocator,
......@@ -462,7 +463,7 @@ pub fn ArrayHashMapUnmanaged(
462463 comptime Context: type,
463464 comptime store_hash: bool,
464465) type {
465 comptime std.hash_map.verifyContext(Context, K, K, u32);
466 comptime std.hash_map.verifyContext(Context, K, K, u32, true);
466467 return struct {
467468 /// It is permitted to access this field directly.
468469 entries: DataList = .{},
......@@ -700,7 +701,7 @@ pub fn ArrayHashMapUnmanaged(
700701 const hashes_array = slice.items(.hash);
701702 const keys_array = slice.items(.key);
702703 for (keys_array) |*item_key, i| {
703 if (hashes_array[i] == h and checkedEql(ctx, key, item_key.*)) {
704 if (hashes_array[i] == h and checkedEql(ctx, key, item_key.*, i)) {
704705 return GetOrPutResult{
705706 .key_ptr = item_key,
706707 // workaround for #6974
......@@ -933,7 +934,7 @@ pub fn ArrayHashMapUnmanaged(
933934 const hashes_array = slice.items(.hash);
934935 const keys_array = slice.items(.key);
935936 for (keys_array) |*item_key, i| {
936 if (hashes_array[i] == h and checkedEql(ctx, key, item_key.*)) {
937 if (hashes_array[i] == h and checkedEql(ctx, key, item_key.*, i)) {
937938 return i;
938939 }
939940 }
......@@ -1245,7 +1246,7 @@ pub fn ArrayHashMapUnmanaged(
12451246 const keys_array = slice.items(.key);
12461247 for (keys_array) |*item_key, i| {
12471248 const hash_match = if (store_hash) hashes_array[i] == key_hash else true;
1248 if (hash_match and key_ctx.eql(key, item_key.*)) {
1249 if (hash_match and key_ctx.eql(key, item_key.*, i)) {
12491250 const removed_entry: KV = .{
12501251 .key = keys_array[i],
12511252 .value = slice.items(.value)[i],
......@@ -1286,7 +1287,7 @@ pub fn ArrayHashMapUnmanaged(
12861287 const keys_array = slice.items(.key);
12871288 for (keys_array) |*item_key, i| {
12881289 const hash_match = if (store_hash) hashes_array[i] == key_hash else true;
1289 if (hash_match and key_ctx.eql(key, item_key.*)) {
1290 if (hash_match and key_ctx.eql(key, item_key.*, i)) {
12901291 switch (removal_type) {
12911292 .swap => self.entries.swapRemove(i),
12921293 .ordered => self.entries.orderedRemove(i),
......@@ -1483,8 +1484,9 @@ pub fn ArrayHashMapUnmanaged(
14831484
14841485 // This pointer survives the following append because we call
14851486 // entries.ensureTotalCapacity before getOrPutInternal.
1486 const hash_match = if (store_hash) h == hashes_array[slot_data.entry_index] else true;
1487 if (hash_match and checkedEql(ctx, key, keys_array[slot_data.entry_index])) {
1487 const i = slot_data.entry_index;
1488 const hash_match = if (store_hash) h == hashes_array[i] else true;
1489 if (hash_match and checkedEql(ctx, key, keys_array[i], i)) {
14881490 return .{
14891491 .found_existing = true,
14901492 .key_ptr = &keys_array[slot_data.entry_index],
......@@ -1571,8 +1573,9 @@ pub fn ArrayHashMapUnmanaged(
15711573 if (slot_data.isEmpty() or slot_data.distance_from_start_index < distance_from_start_index)
15721574 return null;
15731575
1574 const hash_match = if (store_hash) h == hashes_array[slot_data.entry_index] else true;
1575 if (hash_match and checkedEql(ctx, key, keys_array[slot_data.entry_index]))
1576 const i = slot_data.entry_index;
1577 const hash_match = if (store_hash) h == hashes_array[i] else true;
1578 if (hash_match and checkedEql(ctx, key, keys_array[i], i))
15761579 return slot;
15771580 }
15781581 unreachable;
......@@ -1624,7 +1627,7 @@ pub fn ArrayHashMapUnmanaged(
16241627 }
16251628
16261629 inline fn checkedHash(ctx: anytype, key: anytype) u32 {
1627 comptime std.hash_map.verifyContext(@TypeOf(ctx), @TypeOf(key), K, u32);
1630 comptime std.hash_map.verifyContext(@TypeOf(ctx), @TypeOf(key), K, u32, true);
16281631 // If you get a compile error on the next line, it means that
16291632 const hash = ctx.hash(key); // your generic hash function doesn't accept your key
16301633 if (@TypeOf(hash) != u32) {
......@@ -1633,10 +1636,10 @@ pub fn ArrayHashMapUnmanaged(
16331636 }
16341637 return hash;
16351638 }
1636 inline fn checkedEql(ctx: anytype, a: anytype, b: K) bool {
1637 comptime std.hash_map.verifyContext(@TypeOf(ctx), @TypeOf(a), K, u32);
1639 inline fn checkedEql(ctx: anytype, a: anytype, b: K, b_index: usize) bool {
1640 comptime std.hash_map.verifyContext(@TypeOf(ctx), @TypeOf(a), K, u32, true);
16381641 // If you get a compile error on the next line, it means that
1639 const eql = ctx.eql(a, b); // your generic eql function doesn't accept (self, adapt key, K)
1642 const eql = ctx.eql(a, b, b_index); // your generic eql function doesn't accept (self, adapt key, K, index)
16401643 if (@TypeOf(eql) != bool) {
16411644 @compileError("Context " ++ @typeName(@TypeOf(ctx)) ++ " has a generic eql function that returns the wrong type!\n" ++
16421645 @typeName(bool) ++ " was expected, but found " ++ @typeName(@TypeOf(eql)));
......@@ -2255,9 +2258,10 @@ pub fn getAutoHashFn(comptime K: type, comptime Context: type) (fn (Context, K)
22552258 }.hash;
22562259}
22572260
2258pub fn getAutoEqlFn(comptime K: type, comptime Context: type) (fn (Context, K, K) bool) {
2261pub fn getAutoEqlFn(comptime K: type, comptime Context: type) (fn (Context, K, K, usize) bool) {
22592262 return struct {
2260 fn eql(ctx: Context, a: K, b: K) bool {
2263 fn eql(ctx: Context, a: K, b: K, b_index: usize) bool {
2264 _ = b_index;
22612265 _ = ctx;
22622266 return meta.eql(a, b);
22632267 }
lib/std/builtin.zig+3
......@@ -202,12 +202,14 @@ pub const TypeInfo = union(enum) {
202202 /// therefore must be kept in sync with the compiler implementation.
203203 pub const Int = struct {
204204 signedness: Signedness,
205 /// TODO make this u16 instead of comptime_int
205206 bits: comptime_int,
206207 };
207208
208209 /// This data structure is used by the Zig language code generation and
209210 /// therefore must be kept in sync with the compiler implementation.
210211 pub const Float = struct {
212 /// TODO make this u16 instead of comptime_int
211213 bits: comptime_int,
212214 };
213215
......@@ -217,6 +219,7 @@ pub const TypeInfo = union(enum) {
217219 size: Size,
218220 is_const: bool,
219221 is_volatile: bool,
222 /// TODO make this u16 instead of comptime_int
220223 alignment: comptime_int,
221224 address_space: AddressSpace,
222225 child: type,
lib/std/hash/auto_hash.zig+2-1
......@@ -81,7 +81,6 @@ pub fn hash(hasher: anytype, key: anytype, comptime strat: HashStrategy) void {
8181 .NoReturn,
8282 .Opaque,
8383 .Undefined,
84 .Void,
8584 .Null,
8685 .ComptimeFloat,
8786 .ComptimeInt,
......@@ -91,6 +90,8 @@ pub fn hash(hasher: anytype, key: anytype, comptime strat: HashStrategy) void {
9190 .Float,
9291 => @compileError("unable to hash type " ++ @typeName(Key)),
9392
93 .Void => return,
94
9495 // Help the optimizer see that hashing an int is easy by inlining!
9596 // TODO Check if the situation is better after #561 is resolved.
9697 .Int => {
lib/std/hash_map.zig+16-7
......@@ -131,7 +131,13 @@ pub const default_max_load_percentage = 80;
131131/// If you are passing a context to a *Adapted function, PseudoKey is the type
132132/// of the key parameter. Otherwise, when creating a HashMap or HashMapUnmanaged
133133/// type, PseudoKey = Key = K.
134pub fn verifyContext(comptime RawContext: type, comptime PseudoKey: type, comptime Key: type, comptime Hash: type) void {
134pub fn verifyContext(
135 comptime RawContext: type,
136 comptime PseudoKey: type,
137 comptime Key: type,
138 comptime Hash: type,
139 comptime is_array: bool,
140) void {
135141 comptime {
136142 var allow_const_ptr = false;
137143 var allow_mutable_ptr = false;
......@@ -166,7 +172,9 @@ pub fn verifyContext(comptime RawContext: type, comptime PseudoKey: type, compti
166172 const prefix = "\n ";
167173 const deep_prefix = prefix ++ " ";
168174 const hash_signature = "fn (self, " ++ @typeName(PseudoKey) ++ ") " ++ @typeName(Hash);
169 const eql_signature = "fn (self, " ++ @typeName(PseudoKey) ++ ", " ++ @typeName(Key) ++ ") bool";
175 const index_param = if (is_array) ", b_index: usize" else "";
176 const eql_signature = "fn (self, " ++ @typeName(PseudoKey) ++ ", " ++
177 @typeName(Key) ++ index_param ++ ") bool";
170178 const err_invalid_hash_signature = prefix ++ @typeName(Context) ++ ".hash must be " ++ hash_signature ++
171179 deep_prefix ++ "but is actually " ++ @typeName(@TypeOf(Context.hash));
172180 const err_invalid_eql_signature = prefix ++ @typeName(Context) ++ ".eql must be " ++ eql_signature ++
......@@ -255,7 +263,8 @@ pub fn verifyContext(comptime RawContext: type, comptime PseudoKey: type, compti
255263 const info = @typeInfo(@TypeOf(eql));
256264 if (info == .Fn) {
257265 const func = info.Fn;
258 if (func.args.len != 3) {
266 const args_len = if (is_array) 4 else 3;
267 if (func.args.len != args_len) {
259268 errors = errors ++ lazy.err_invalid_eql_signature;
260269 } else {
261270 var emitted_signature = false;
......@@ -360,7 +369,7 @@ pub fn HashMap(
360369 comptime Context: type,
361370 comptime max_load_percentage: u64,
362371) type {
363 comptime verifyContext(Context, K, K, u64);
372 comptime verifyContext(Context, K, K, u64, false);
364373 return struct {
365374 unmanaged: Unmanaged,
366375 allocator: Allocator,
......@@ -683,7 +692,7 @@ pub fn HashMapUnmanaged(
683692) type {
684693 if (max_load_percentage <= 0 or max_load_percentage >= 100)
685694 @compileError("max_load_percentage must be between 0 and 100.");
686 comptime verifyContext(Context, K, K, u64);
695 comptime verifyContext(Context, K, K, u64, false);
687696
688697 return struct {
689698 const Self = @This();
......@@ -1108,7 +1117,7 @@ pub fn HashMapUnmanaged(
11081117 /// from this function. To encourage that, this function is
11091118 /// marked as inline.
11101119 inline fn getIndex(self: Self, key: anytype, ctx: anytype) ?usize {
1111 comptime verifyContext(@TypeOf(ctx), @TypeOf(key), K, Hash);
1120 comptime verifyContext(@TypeOf(ctx), @TypeOf(key), K, Hash, false);
11121121
11131122 if (self.size == 0) {
11141123 return null;
......@@ -1291,7 +1300,7 @@ pub fn HashMapUnmanaged(
12911300 return result;
12921301 }
12931302 pub fn getOrPutAssumeCapacityAdapted(self: *Self, key: anytype, ctx: anytype) GetOrPutResult {
1294 comptime verifyContext(@TypeOf(ctx), @TypeOf(key), K, Hash);
1303 comptime verifyContext(@TypeOf(ctx), @TypeOf(key), K, Hash, false);
12951304
12961305 // If you get a compile error on this line, it means that your generic hash
12971306 // function is invalid for these parameters.
src/InternArena.zig created+315
......@@ -0,0 +1,315 @@
1map: std.AutoArrayHashMapUnmanaged(void, void) = .{},
2items: std.MultiArrayList(Item) = .{},
3extra: std.ArrayListUnmanaged(u32) = .{},
4
5const InternArena = @This();
6const std = @import("std");
7const Allocator = std.mem.Allocator;
8const assert = std.debug.assert;
9
10const KeyAdapter = struct {
11 intern_arena: *const InternArena,
12
13 pub fn eql(ctx: @This(), a: Key, b_void: void, b_map_index: usize) bool {
14 _ = b_void;
15 return ctx.intern_arena.indexToKey(@intToEnum(Index, b_map_index)).eql(a);
16 }
17
18 pub fn hash(ctx: @This(), a: Key) u32 {
19 _ = ctx;
20 return a.hash();
21 }
22};
23
24pub const Key = union(enum) {
25 int_type: struct {
26 signedness: std.builtin.Signedness,
27 bits: u16,
28 },
29 ptr_type: struct {
30 elem_type: Index,
31 sentinel: Index,
32 alignment: u16,
33 size: std.builtin.TypeInfo.Pointer.Size,
34 is_const: bool,
35 is_volatile: bool,
36 is_allowzero: bool,
37 address_space: std.builtin.AddressSpace,
38 },
39 array_type: struct {
40 len: u64,
41 child: Index,
42 sentinel: Index,
43 },
44 vector_type: struct {
45 len: u32,
46 child: Index,
47 },
48 optional_type: struct {
49 payload_type: Index,
50 },
51 error_union_type: struct {
52 error_set_type: Index,
53 payload_type: Index,
54 },
55 simple: Simple,
56
57 pub fn hash(key: Key) u32 {
58 var hasher = std.hash.Wyhash.init(0);
59 switch (key) {
60 .int_type => |int_type| {
61 std.hash.autoHash(&hasher, int_type);
62 },
63 .array_type => |array_type| {
64 std.hash.autoHash(&hasher, array_type);
65 },
66 else => @panic("TODO"),
67 }
68 return @truncate(u32, hasher.final());
69 }
70
71 pub fn eql(a: Key, b: Key) bool {
72 const KeyTag = std.meta.Tag(Key);
73 const a_tag: KeyTag = a;
74 const b_tag: KeyTag = b;
75 if (a_tag != b_tag) return false;
76 switch (a) {
77 .int_type => |a_info| {
78 const b_info = b.int_type;
79 return std.meta.eql(a_info, b_info);
80 },
81 .array_type => |a_info| {
82 const b_info = b.array_type;
83 return std.meta.eql(a_info, b_info);
84 },
85 else => @panic("TODO"),
86 }
87 }
88};
89
90pub const Item = struct {
91 tag: Tag,
92 /// The doc comments on the respective Tag explain how to interpret this.
93 data: u32,
94};
95
96/// Represents an index into `map`. It represents the canonical index
97/// of a `Value` within this `InternArena`. The values are typed.
98/// Two values which have the same type can be equality compared simply
99/// by checking if their indexes are equal, provided they are both in
100/// the same `InternArena`.
101pub const Index = enum(u32) {
102 none = std.math.maxInt(u32),
103 _,
104};
105
106pub const Tag = enum(u8) {
107 /// An integer type.
108 /// data is number of bits
109 type_int_signed,
110 /// An integer type.
111 /// data is number of bits
112 type_int_unsigned,
113 /// An array type.
114 /// data is payload to Array.
115 type_array,
116 /// A type or value that can be represented with only an enum tag.
117 /// data is Simple enum value
118 simple,
119 /// An unsigned integer value that can be represented by u32.
120 /// data is integer value
121 int_u32,
122 /// An unsigned integer value that can be represented by i32.
123 /// data is integer value bitcasted to u32.
124 int_i32,
125 /// A positive integer value that does not fit in 32 bits.
126 /// data is a extra index to BigInt.
127 int_big_positive,
128 /// A negative integer value that does not fit in 32 bits.
129 /// data is a extra index to BigInt.
130 int_big_negative,
131 /// A float value that can be represented by f32.
132 /// data is float value bitcasted to u32.
133 float_f32,
134 /// A float value that can be represented by f64.
135 /// data is payload index to Float64.
136 float_f64,
137 /// A float value that can be represented by f128.
138 /// data is payload index to Float128.
139 float_f128,
140};
141
142pub const Simple = enum(u32) {
143 f16,
144 f32,
145 f64,
146 f80,
147 f128,
148 usize,
149 isize,
150 c_short,
151 c_ushort,
152 c_int,
153 c_uint,
154 c_long,
155 c_ulong,
156 c_longlong,
157 c_ulonglong,
158 c_longdouble,
159 anyopaque,
160 bool,
161 void,
162 type,
163 anyerror,
164 comptime_int,
165 comptime_float,
166 noreturn,
167 @"anyframe",
168 null_type,
169 undefined_type,
170 enum_literal_type,
171 @"undefined",
172 void_value,
173 @"null",
174 bool_true,
175 bool_false,
176};
177
178pub const Array = struct {
179 len: u32,
180 child: Index,
181};
182
183pub fn deinit(ia: *InternArena, gpa: Allocator) void {
184 ia.map.deinit(gpa);
185 ia.items.deinit(gpa);
186 ia.extra.deinit(gpa);
187}
188
189pub fn indexToKey(ia: InternArena, index: Index) Key {
190 const data = ia.items.items(.data)[@enumToInt(index)];
191 return switch (ia.items.items(.tag)[@enumToInt(index)]) {
192 .type_int_signed => .{
193 .int_type = .{
194 .signedness = .signed,
195 .bits = @intCast(u16, data),
196 },
197 },
198 .type_int_unsigned => .{
199 .int_type = .{
200 .signedness = .unsigned,
201 .bits = @intCast(u16, data),
202 },
203 },
204 .type_array => {
205 const array_info = ia.extraData(Array, data);
206 return .{ .array_type = .{
207 .len = array_info.len,
208 .child = array_info.child,
209 .sentinel = .none,
210 } };
211 },
212 .simple => .{ .simple = @intToEnum(Simple, data) },
213
214 else => @panic("TODO"),
215 };
216}
217
218pub fn get(ia: *InternArena, gpa: Allocator, key: Key) Allocator.Error!Index {
219 const adapter: KeyAdapter = .{ .intern_arena = ia };
220 const gop = try ia.map.getOrPutAdapted(gpa, key, adapter);
221 if (gop.found_existing) {
222 return @intToEnum(Index, gop.index);
223 }
224 switch (key) {
225 .int_type => |int_type| {
226 const tag: Tag = switch (int_type.signedness) {
227 .signed => .type_int_signed,
228 .unsigned => .type_int_unsigned,
229 };
230 try ia.items.append(gpa, .{
231 .tag = tag,
232 .data = int_type.bits,
233 });
234 },
235 .array_type => |array_type| {
236 const len = @intCast(u32, array_type.len); // TODO have a big_array encoding
237 assert(array_type.sentinel == .none); // TODO have a sentinel_array encoding
238 try ia.items.append(gpa, .{
239 .tag = .type_array,
240 .data = try ia.addExtra(gpa, Array{
241 .len = len,
242 .child = array_type.child,
243 }),
244 });
245 },
246 else => @panic("TODO"),
247 }
248 return @intToEnum(Index, ia.items.len - 1);
249}
250
251fn addExtra(ia: *InternArena, gpa: Allocator, extra: anytype) Allocator.Error!u32 {
252 const fields = std.meta.fields(@TypeOf(extra));
253 try ia.extra.ensureUnusedCapacity(gpa, fields.len);
254 return ia.addExtraAssumeCapacity(extra);
255}
256
257fn addExtraAssumeCapacity(ia: *InternArena, extra: anytype) u32 {
258 const fields = std.meta.fields(@TypeOf(extra));
259 const result = @intCast(u32, ia.extra.items.len);
260 inline for (fields) |field| {
261 ia.extra.appendAssumeCapacity(switch (field.field_type) {
262 u32 => @field(extra, field.name),
263 Index => @enumToInt(@field(extra, field.name)),
264 i32 => @bitCast(u32, @field(extra, field.name)),
265 else => @compileError("bad field type"),
266 });
267 }
268 return result;
269}
270
271fn extraData(ia: InternArena, comptime T: type, index: usize) T {
272 const fields = std.meta.fields(T);
273 var i: usize = index;
274 var result: T = undefined;
275 inline for (fields) |field| {
276 @field(result, field.name) = switch (field.field_type) {
277 u32 => ia.extra.items[i],
278 Index => @intToEnum(Index, ia.extra.items[i]),
279 i32 => @bitCast(i32, ia.extra.items[i]),
280 else => @compileError("bad field type"),
281 };
282 i += 1;
283 }
284 return result;
285}
286
287test "basic usage" {
288 const gpa = std.testing.allocator;
289
290 var ia: InternArena = .{};
291 defer ia.deinit(gpa);
292
293 const i32_type = try ia.get(gpa, .{ .int_type = .{
294 .signedness = .signed,
295 .bits = 32,
296 } });
297 const array_i32 = try ia.get(gpa, .{ .array_type = .{
298 .len = 10,
299 .child = i32_type,
300 .sentinel = .none,
301 } });
302
303 const another_i32_type = try ia.get(gpa, .{ .int_type = .{
304 .signedness = .signed,
305 .bits = 32,
306 } });
307 try std.testing.expect(another_i32_type == i32_type);
308
309 const another_array_i32 = try ia.get(gpa, .{ .array_type = .{
310 .len = 10,
311 .child = i32_type,
312 .sentinel = .none,
313 } });
314 try std.testing.expect(another_array_i32 == array_i32);
315}