authorgravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-07-08 01:09:54+10:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-09-10 10:20:30-04:00
loga29ce78651c05029dbd72064752a099885edfd0c
tree9962dab206cea6667cba70b78611d140697ce09e
parent8fbae77770a77ccd645054e06baab45b03c8befd

std: add BloomFilter data structure

3 files changed, 258 insertions(+), 1 deletions(-)

std/bloom_filter.zig created+253
...@@ -0,0 +1,253 @@
1const builtin = @import("builtin");
2const std = @import("std.zig");
3const math = std.math;
4const debug = std.debug;
5const assert = std.debug.assert;
6const testing = std.testing;
7
8/// There is a trade off of how quickly to fill a bloom filter;
9/// the number of items is:
10/// n_items / K * ln(2)
11/// the rate of false positives is:
12/// (1-e^(-K*N/n_items))^K
13/// where N is the number of items
14pub fn BloomFilter(
15 /// Size of bloom filter in cells, must be a power of two.
16 comptime n_items: usize,
17 /// Number of cells to set per item
18 comptime K: usize,
19 /// Cell type, should be:
20 /// - `bool` for a standard bloom filter
21 /// - an unsigned integer type for a counting bloom filter
22 comptime Cell: type,
23 /// endianess of the Cell
24 comptime endian: builtin.Endian,
25 /// Hash function to use
26 comptime hash: fn (out: []u8, Ki: usize, in: []const u8) void,
27) type {
28 assert(n_items > 0);
29 assert(math.isPowerOfTwo(n_items));
30 assert(K > 0);
31 const cellEmpty = if (Cell == bool) false else Cell(0);
32 const cellMax = if (Cell == bool) true else math.maxInt(Cell);
33 const n_bytes = (n_items * comptime std.meta.bitCount(Cell)) / 8;
34 assert(n_bytes > 0);
35 const Io = std.packed_int_array.PackedIntIo(Cell, endian);
36
37 return struct {
38 const Self = @This();
39 pub const items = n_items;
40 pub const Index = math.IntFittingRange(0, n_items - 1);
41
42 data: [n_bytes]u8 = [_]u8{0} ** n_bytes,
43
44 pub fn reset(self: *Self) void {
45 std.mem.set(u8, self.data[0..], 0);
46 }
47
48 pub fn @"union"(x: Self, y: Self) Self {
49 var r = Self{ .data = undefined };
50 inline for (x.data) |v, i| {
51 r.data[i] = v | y.data[i];
52 }
53 return r;
54 }
55
56 pub fn intersection(x: Self, y: Self) Self {
57 var r = Self{ .data = undefined };
58 inline for (x.data) |v, i| {
59 r.data[i] = v & y.data[i];
60 }
61 return r;
62 }
63
64 pub fn getCell(self: Self, cell: Index) Cell {
65 return Io.get(self.data, cell, 0);
66 }
67
68 pub fn incrementCell(self: *Self, cell: Index) void {
69 if (Cell == bool or Cell == u1) {
70 // skip the 'get' operation
71 Io.set(&self.data, cell, 0, cellMax);
72 } else {
73 const old = Io.get(self.data, cell, 0);
74 if (old != cellMax) {
75 Io.set(&self.data, cell, 0, old + 1);
76 }
77 }
78 }
79
80 pub fn clearCell(self: *Self, cell: Index) void {
81 Io.set(&self.data, cell, 0, cellEmpty);
82 }
83
84 pub fn add(self: *Self, item: []const u8) void {
85 comptime var i = 0;
86 inline while (i < K) : (i += 1) {
87 var K_th_bit: packed struct { x: Index } = undefined;
88 hash(std.mem.asBytes(&K_th_bit), i, item);
89 incrementCell(self, K_th_bit.x);
90 }
91 }
92
93 pub fn contains(self: Self, item: []const u8) bool {
94 comptime var i = 0;
95 inline while (i < K) : (i += 1) {
96 var K_th_bit: packed struct { x: Index } = undefined;
97 hash(std.mem.asBytes(&K_th_bit), i, item);
98 if (getCell(self, K_th_bit.x) == cellEmpty)
99 return false;
100 }
101 return true;
102 }
103
104 pub fn resize(self: Self, comptime newsize: usize) BloomFilter(newsize, K, Cell, endian, hash) {
105 var r: BloomFilter(newsize, K, Cell, endian, hash) = undefined;
106 if (newsize < n_items) {
107 std.mem.copy(u8, r.data[0..], self.data[0..r.data.len]);
108 var copied: usize = r.data.len;
109 while (copied < self.data.len) : (copied += r.data.len) {
110 for (self.data[copied .. copied + r.data.len]) |s, i| {
111 r.data[i] |= s;
112 }
113 }
114 } else if (newsize == n_items) {
115 r = self;
116 } else if (newsize > n_items) {
117 var copied: usize = 0;
118 while (copied < r.data.len) : (copied += self.data.len) {
119 std.mem.copy(u8, r.data[copied .. copied + self.data.len], self.data);
120 }
121 }
122 return r;
123 }
124
125 /// Returns number of non-zero cells
126 pub fn popCount(self: Self) Index {
127 var n: Index = 0;
128 if (Cell == bool or Cell == u1) {
129 for (self.data) |b, i| {
130 n += @popCount(u8, b);
131 }
132 } else {
133 var i: usize = 0;
134 while (i < n_items) : (i += 1) {
135 const cell = self.getCell(@intCast(Index, i));
136 n += if (if (Cell == bool) cell else cell > 0) Index(1) else Index(0);
137 }
138 }
139 return n;
140 }
141
142 pub fn estimateItems(self: Self) f64 {
143 const m = comptime @intToFloat(f64, n_items);
144 const k = comptime @intToFloat(f64, K);
145 const X = @intToFloat(f64, self.popCount());
146 return (comptime (-m / k)) * math.log1p(X * comptime (-1 / m));
147 }
148 };
149}
150
151fn hashFunc(out: []u8, Ki: usize, in: []const u8) void {
152 var st = std.crypto.gimli.Hash.init();
153 st.update(std.mem.asBytes(&Ki));
154 st.update(in);
155 st.final(out);
156}
157
158test "std.BloomFilter" {
159 inline for ([_]type{ bool, u1, u2, u3, u4 }) |Cell| {
160 const emptyCell = if (Cell == bool) false else Cell(0);
161 const BF = BloomFilter(128 * 8, 8, Cell, builtin.endian, hashFunc);
162 var bf = BF{};
163 var i: usize = undefined;
164 // confirm that it is initialised to the empty filter
165 i = 0;
166 while (i < BF.items) : (i += 1) {
167 testing.expectEqual(emptyCell, bf.getCell(@intCast(BF.Index, i)));
168 }
169 testing.expectEqual(BF.Index(0), bf.popCount());
170 testing.expectEqual(f64(0), bf.estimateItems());
171 // fill in a few items
172 bf.incrementCell(42);
173 bf.incrementCell(255);
174 bf.incrementCell(256);
175 bf.incrementCell(257);
176 // check that they were set
177 testing.expectEqual(true, bf.getCell(42) != emptyCell);
178 testing.expectEqual(true, bf.getCell(255) != emptyCell);
179 testing.expectEqual(true, bf.getCell(256) != emptyCell);
180 testing.expectEqual(true, bf.getCell(257) != emptyCell);
181 // clear just one of them; make sure the rest are still set
182 bf.clearCell(256);
183 testing.expectEqual(true, bf.getCell(42) != emptyCell);
184 testing.expectEqual(true, bf.getCell(255) != emptyCell);
185 testing.expectEqual(false, bf.getCell(256) != emptyCell);
186 testing.expectEqual(true, bf.getCell(257) != emptyCell);
187 // reset any of the ones we've set and confirm we're back to the empty filter
188 bf.clearCell(42);
189 bf.clearCell(255);
190 bf.clearCell(257);
191 i = 0;
192 while (i < BF.items) : (i += 1) {
193 testing.expectEqual(emptyCell, bf.getCell(@intCast(BF.Index, i)));
194 }
195 testing.expectEqual(BF.Index(0), bf.popCount());
196 testing.expectEqual(f64(0), bf.estimateItems());
197
198 // Lets add a string
199 bf.add("foo");
200 testing.expectEqual(true, bf.contains("foo"));
201 {
202 // try adding same string again. make sure popcount is the same
203 const old_popcount = bf.popCount();
204 testing.expect(old_popcount > 0);
205 bf.add("foo");
206 testing.expectEqual(true, bf.contains("foo"));
207 testing.expectEqual(old_popcount, bf.popCount());
208 }
209
210 // Get back to empty filter via .reset
211 bf.reset();
212 // Double check that .reset worked
213 i = 0;
214 while (i < BF.items) : (i += 1) {
215 testing.expectEqual(emptyCell, bf.getCell(@intCast(BF.Index, i)));
216 }
217 testing.expectEqual(BF.Index(0), bf.popCount());
218 testing.expectEqual(f64(0), bf.estimateItems());
219
220 comptime var teststrings = [_][]const u8{
221 "foo",
222 "bar",
223 "a longer string",
224 "some more",
225 "the quick brown fox",
226 "unique string",
227 };
228 inline for (teststrings) |str| {
229 bf.add(str);
230 }
231 inline for (teststrings) |str| {
232 testing.expectEqual(true, bf.contains(str));
233 }
234
235 { // estimate should be close for low packing
236 const est = bf.estimateItems();
237 testing.expect(est > @intToFloat(f64, teststrings.len) - 1);
238 testing.expect(est < @intToFloat(f64, teststrings.len) + 1);
239 }
240
241 const larger_bf = bf.resize(4096);
242 inline for (teststrings) |str| {
243 testing.expectEqual(true, larger_bf.contains(str));
244 }
245 testing.expectEqual(u12(bf.popCount()) * (4096 / 1024), larger_bf.popCount());
246
247 const smaller_bf = bf.resize(64);
248 inline for (teststrings) |str| {
249 testing.expectEqual(true, smaller_bf.contains(str));
250 }
251 testing.expect(bf.popCount() <= u10(smaller_bf.popCount()) * (1024 / 64));
252 }
253}
std/meta.zig+2-1
...@@ -74,9 +74,10 @@ test "std.meta.stringToEnum" {...@@ -74,9 +74,10 @@ test "std.meta.stringToEnum" {
7474
75pub fn bitCount(comptime T: type) comptime_int {75pub fn bitCount(comptime T: type) comptime_int {
76 return switch (@typeInfo(T)) {76 return switch (@typeInfo(T)) {
77 TypeId.Bool => 1,
77 TypeId.Int => |info| info.bits,78 TypeId.Int => |info| info.bits,
78 TypeId.Float => |info| info.bits,79 TypeId.Float => |info| info.bits,
79 else => @compileError("Expected int or float type, found '" ++ @typeName(T) ++ "'"),80 else => @compileError("Expected bool, int or float type, found '" ++ @typeName(T) ++ "'"),
80 };81 };
81}82}
8283
std/std.zig+3
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1pub const AlignedArrayList = @import("array_list.zig").AlignedArrayList;1pub const AlignedArrayList = @import("array_list.zig").AlignedArrayList;
2pub const ArrayList = @import("array_list.zig").ArrayList;2pub const ArrayList = @import("array_list.zig").ArrayList;
3pub const AutoHashMap = @import("hash_map.zig").AutoHashMap;3pub const AutoHashMap = @import("hash_map.zig").AutoHashMap;
4pub const BloomFilter = @import("bloom_filter.zig").BloomFilter;
4pub const BufMap = @import("buf_map.zig").BufMap;5pub const BufMap = @import("buf_map.zig").BufMap;
5pub const BufSet = @import("buf_set.zig").BufSet;6pub const BufSet = @import("buf_set.zig").BufSet;
6pub const Buffer = @import("buffer.zig").Buffer;7pub const Buffer = @import("buffer.zig").Buffer;
...@@ -48,6 +49,7 @@ pub const mem = @import("mem.zig");...@@ -48,6 +49,7 @@ pub const mem = @import("mem.zig");
48pub const meta = @import("meta.zig");49pub const meta = @import("meta.zig");
49pub const net = @import("net.zig");50pub const net = @import("net.zig");
50pub const os = @import("os.zig");51pub const os = @import("os.zig");
52pub const packed_int_array = @import("packed_int_array.zig");
51pub const pdb = @import("pdb.zig");53pub const pdb = @import("pdb.zig");
52pub const process = @import("process.zig");54pub const process = @import("process.zig");
53pub const rand = @import("rand.zig");55pub const rand = @import("rand.zig");
...@@ -64,6 +66,7 @@ test "std" {...@@ -64,6 +66,7 @@ test "std" {
64 // run tests from these66 // run tests from these
65 _ = @import("array_list.zig");67 _ = @import("array_list.zig");
66 _ = @import("atomic.zig");68 _ = @import("atomic.zig");
69 _ = @import("bloom_filter.zig");
67 _ = @import("buf_map.zig");70 _ = @import("buf_map.zig");
68 _ = @import("buf_set.zig");71 _ = @import("buf_set.zig");
69 _ = @import("buffer.zig");72 _ = @import("buffer.zig");