authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-24 17:33:53-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-11-24 17:33:53-05:00
log8309b6188d849c5d4c27c086b7759566d9f86716
treec44f3023b78ac9b7ed126a36d894a4de7e487413
parenteea8b10463b944dae5434e748c9a5b67c2287bea
parent1a84bcefb658b45103155725067e0820dd4243a0
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3644 from daurnimator/bytefifo

Improvements to std.fifo

1 files changed, 183 insertions(+), 60 deletions(-)

lib/std/fifo.zig+183-60
...@@ -9,27 +9,77 @@ const debug = std.debug;...@@ -9,27 +9,77 @@ const debug = std.debug;
9const assert = debug.assert;9const assert = debug.assert;
10const testing = std.testing;10const testing = std.testing;
1111
12pub fn FixedSizeFifo(comptime T: type) type {12pub const LinearFifoBufferType = union(enum) {
13 /// The buffer is internal to the fifo; it is of the specified size.
14 Static: usize,
15
16 /// The buffer is passed as a slice to the initialiser.
17 Slice,
18
19 /// The buffer is managed dynamically using a `mem.Allocator`.
20 Dynamic,
21};
22
23pub fn LinearFifo(
24 comptime T: type,
25 comptime buffer_type: LinearFifoBufferType,
26) type {
27 const autoalign = false;
28
29 const powers_of_two = switch (buffer_type) {
30 .Static => std.math.isPowerOfTwo(buffer_type.Static),
31 .Slice => false, // Any size slice could be passed in
32 .Dynamic => true, // This could be configurable in future
33 };
34
13 return struct {35 return struct {
14 allocator: *Allocator,36 allocator: if (buffer_type == .Dynamic) *Allocator else void,
15 buf: []T,37 buf: if (buffer_type == .Static) [buffer_type.Static]T else []T,
16 head: usize,38 head: usize,
17 count: usize,39 count: usize,
1840
19 const Self = @This();41 const Self = @This();
2042
21 pub fn init(allocator: *Allocator) Self {43 // Type of Self argument for slice operations.
22 return Self{44 // If buffer is inline (Static) then we need to ensure we haven't
23 .allocator = allocator,45 // returned a slice into a copy on the stack
24 .buf = [_]T{},46 const SliceSelfArg = if (buffer_type == .Static) *Self else Self;
25 .head = 0,47
26 .count = 0,48 pub usingnamespace switch (buffer_type) {
27 };49 .Static => struct {
28 }50 pub fn init() Self {
51 return .{
52 .allocator = {},
53 .buf = undefined,
54 .head = 0,
55 .count = 0,
56 };
57 }
58 },
59 .Slice => struct {
60 pub fn init(buf: []T) Self {
61 return .{
62 .allocator = {},
63 .buf = buf,
64 .head = 0,
65 .count = 0,
66 };
67 }
68 },
69 .Dynamic => struct {
70 pub fn init(allocator: *Allocator) Self {
71 return .{
72 .allocator = allocator,
73 .buf = [_]T{},
74 .head = 0,
75 .count = 0,
76 };
77 }
78 },
79 };
2980
30 pub fn deinit(self: *Self) void {81 pub fn deinit(self: Self) void {
31 self.allocator.free(self.buf);82 if (buffer_type == .Dynamic) self.allocator.free(self.buf);
32 self.* = undefined;
33 }83 }
3484
35 pub fn realign(self: *Self) void {85 pub fn realign(self: *Self) void {
...@@ -59,18 +109,24 @@ pub fn FixedSizeFifo(comptime T: type) type {...@@ -59,18 +109,24 @@ pub fn FixedSizeFifo(comptime T: type) type {
59 /// Reduce allocated capacity to `size`.109 /// Reduce allocated capacity to `size`.
60 pub fn shrink(self: *Self, size: usize) void {110 pub fn shrink(self: *Self, size: usize) void {
61 assert(size >= self.count);111 assert(size >= self.count);
62 self.realign();112 if (buffer_type == .Dynamic) {
63 self.buf = self.allocator.realloc(self.buf, size) catch |e| switch (e) {113 self.realign();
64 error.OutOfMemory => return, // no problem, capacity is still correct then.114 self.buf = self.allocator.realloc(self.buf, size) catch |e| switch (e) {
65 };115 error.OutOfMemory => return, // no problem, capacity is still correct then.
116 };
117 }
66 }118 }
67119
68 /// Ensure that the buffer can fit at least `size` items120 /// Ensure that the buffer can fit at least `size` items
69 pub fn ensureCapacity(self: *Self, size: usize) error{OutOfMemory}!void {121 pub fn ensureCapacity(self: *Self, size: usize) !void {
70 if (self.buf.len >= size) return;122 if (self.buf.len >= size) return;
71 self.realign();123 if (buffer_type == .Dynamic) {
72 const new_size = math.ceilPowerOfTwo(usize, size) catch return error.OutOfMemory;124 self.realign();
73 self.buf = try self.allocator.realloc(self.buf, new_size);125 const new_size = if (powers_of_two) math.ceilPowerOfTwo(usize, size) catch return error.OutOfMemory else size;
126 self.buf = try self.allocator.realloc(self.buf, new_size);
127 } else {
128 return error.OutOfMemory;
129 }
74 }130 }
75131
76 /// Makes sure at least `size` items are unused132 /// Makes sure at least `size` items are unused
...@@ -86,29 +142,24 @@ pub fn FixedSizeFifo(comptime T: type) type {...@@ -86,29 +142,24 @@ pub fn FixedSizeFifo(comptime T: type) type {
86 }142 }
87143
88 /// Returns a writable slice from the 'read' end of the fifo144 /// Returns a writable slice from the 'read' end of the fifo
89 fn readableSliceMut(self: Self, offset: usize) []T {145 fn readableSliceMut(self: SliceSelfArg, offset: usize) []T {
90 if (offset > self.count) return [_]T{};146 if (offset > self.count) return [_]T{};
91147
92 const start = self.head + offset;148 var start = self.head + offset;
93 if (start >= self.buf.len) {149 if (start >= self.buf.len) {
94 return self.buf[start - self.buf.len ..][0 .. self.count - offset];150 start -= self.buf.len;
151 return self.buf[start..self.count - offset];
95 } else {152 } else {
96 const end: usize = self.head + self.count;153 const end = math.min(self.head + self.count, self.buf.len);
97 if (end >= self.buf.len) {154 return self.buf[start..end];
98 return self.buf[start..self.buf.len];
99 } else {
100 return self.buf[start..end];
101 }
102 }155 }
103 }156 }
104157
105 /// Returns a readable slice from `offset`158 /// Returns a readable slice from `offset`
106 pub fn readableSlice(self: Self, offset: usize) []const T {159 pub fn readableSlice(self: SliceSelfArg, offset: usize) []const T {
107 return self.readableSliceMut(offset);160 return self.readableSliceMut(offset);
108 }161 }
109162
110 const autoalign = false;
111
112 /// Discard first `count` bytes of readable data163 /// Discard first `count` bytes of readable data
113 pub fn discard(self: *Self, count: usize) void {164 pub fn discard(self: *Self, count: usize) void {
114 assert(count <= self.count);165 assert(count <= self.count);
...@@ -124,10 +175,19 @@ pub fn FixedSizeFifo(comptime T: type) type {...@@ -124,10 +175,19 @@ pub fn FixedSizeFifo(comptime T: type) type {
124 @memset(unused2.ptr, undefined, unused2.len);175 @memset(unused2.ptr, undefined, unused2.len);
125 }176 }
126 }177 }
127 self.head = (self.head + count) % self.buf.len;178 if (autoalign and self.count == count) {
128 self.count -= count;
129 if (autoalign and self.count == 0)
130 self.head = 0;179 self.head = 0;
180 self.count = 0;
181 } else {
182 var head = self.head + count;
183 if (powers_of_two) {
184 head &= self.buf.len - 1;
185 } else {
186 head %= self.buf.len;
187 }
188 self.head = head;
189 self.count -= count;
190 }
131 }191 }
132192
133 /// Read the next item from the fifo193 /// Read the next item from the fifo
...@@ -139,8 +199,8 @@ pub fn FixedSizeFifo(comptime T: type) type {...@@ -139,8 +199,8 @@ pub fn FixedSizeFifo(comptime T: type) type {
139 return c;199 return c;
140 }200 }
141201
142 /// Read data from the fifo into `dst`, returns slice of bytes copied (subslice of `dst`)202 /// Read data from the fifo into `dst`, returns number of bytes copied.
143 pub fn read(self: *Self, dst: []T) []T {203 pub fn read(self: *Self, dst: []T) usize {
144 var dst_left = dst;204 var dst_left = dst;
145205
146 while (dst_left.len > 0) {206 while (dst_left.len > 0) {
...@@ -152,7 +212,7 @@ pub fn FixedSizeFifo(comptime T: type) type {...@@ -152,7 +212,7 @@ pub fn FixedSizeFifo(comptime T: type) type {
152 dst_left = dst_left[n..];212 dst_left = dst_left[n..];
153 }213 }
154214
155 return dst[0 .. dst.len - dst_left.len];215 return dst.len - dst_left.len;
156 }216 }
157217
158 /// Returns number of bytes available in fifo218 /// Returns number of bytes available in fifo
...@@ -162,7 +222,7 @@ pub fn FixedSizeFifo(comptime T: type) type {...@@ -162,7 +222,7 @@ pub fn FixedSizeFifo(comptime T: type) type {
162222
163 /// Returns the first section of writable buffer223 /// Returns the first section of writable buffer
164 /// Note that this may be of length 0224 /// Note that this may be of length 0
165 pub fn writableSlice(self: Self, offset: usize) []T {225 pub fn writableSlice(self: SliceSelfArg, offset: usize) []T {
166 if (offset > self.buf.len) return [_]T{};226 if (offset > self.buf.len) return [_]T{};
167227
168 const tail = self.head + offset + self.count;228 const tail = self.head + offset + self.count;
...@@ -193,7 +253,8 @@ pub fn FixedSizeFifo(comptime T: type) type {...@@ -193,7 +253,8 @@ pub fn FixedSizeFifo(comptime T: type) type {
193 self.count += count;253 self.count += count;
194 }254 }
195255
196 /// Appends the data in `src` to the fifo. You must256 /// Appends the data in `src` to the fifo.
257 /// You must have ensured there is enough space.
197 pub fn writeAssumeCapacity(self: *Self, src: []const T) void {258 pub fn writeAssumeCapacity(self: *Self, src: []const T) void {
198 assert(self.writableLength() >= src.len);259 assert(self.writableLength() >= src.len);
199260
...@@ -208,6 +269,20 @@ pub fn FixedSizeFifo(comptime T: type) type {...@@ -208,6 +269,20 @@ pub fn FixedSizeFifo(comptime T: type) type {
208 }269 }
209 }270 }
210271
272 /// Write a single item to the fifo
273 pub fn writeItem(self: *Self, item: T) !void {
274 try self.ensureUnusedCapacity(1);
275
276 var tail = self.head + self.count;
277 if (powers_of_two) {
278 tail &= self.buf.len - 1;
279 } else {
280 tail %= self.buf.len;
281 }
282 self.buf[tail] = byte;
283 self.update(1);
284 }
285
211 /// Appends the data in `src` to the fifo.286 /// Appends the data in `src` to the fifo.
212 /// Allocates more memory as necessary287 /// Allocates more memory as necessary
213 pub fn write(self: *Self, src: []const T) !void {288 pub fn write(self: *Self, src: []const T) !void {
...@@ -216,16 +291,27 @@ pub fn FixedSizeFifo(comptime T: type) type {...@@ -216,16 +291,27 @@ pub fn FixedSizeFifo(comptime T: type) type {
216 return self.writeAssumeCapacity(src);291 return self.writeAssumeCapacity(src);
217 }292 }
218293
219 pub fn print(self: *Self, comptime format: []const u8, args: ...) !void {294 pub usingnamespace if (T == u8)
220 return std.fmt.format(self, error{OutOfMemory}, Self.write, format, args);295 struct {
221 }296 pub fn print(self: *Self, comptime format: []const u8, args: ...) !void {
297 return std.fmt.format(self, error{OutOfMemory}, Self.write, format, args);
298 }
299 }
300 else
301 struct {};
222302
223 /// Make `count` bytes available before the current read location303 /// Make `count` items available before the current read location
224 fn rewind(self: *Self, size: usize) void {304 fn rewind(self: *Self, count: usize) void {
225 assert(self.writableLength() >= size);305 assert(self.writableLength() >= count);
226306
227 self.head = (self.head + (self.buf.len - size)) % self.buf.len;307 var head = self.head + (self.buf.len - count);
228 self.count += size;308 if (powers_of_two) {
309 head &= self.buf.len - 1;
310 } else {
311 head %= self.buf.len;
312 }
313 self.head = head;
314 self.count += count;
229 }315 }
230316
231 /// Place data back into the read stream317 /// Place data back into the read stream
...@@ -235,9 +321,13 @@ pub fn FixedSizeFifo(comptime T: type) type {...@@ -235,9 +321,13 @@ pub fn FixedSizeFifo(comptime T: type) type {
235 self.rewind(src.len);321 self.rewind(src.len);
236322
237 const slice = self.readableSliceMut(0);323 const slice = self.readableSliceMut(0);
238 mem.copy(T, slice, src[0..slice.len]);324 if (src.len < slice.len) {
239 const slice2 = self.readableSliceMut(slice.len);325 mem.copy(T, slice, src);
240 mem.copy(T, slice2, src[slice.len..]);326 } else {
327 mem.copy(T, slice, src[0..slice.len]);
328 const slice2 = self.readableSliceMut(slice.len);
329 mem.copy(T, slice2, src[slice.len..]);
330 }
241 }331 }
242332
243 /// Peek at the item at `offset`333 /// Peek at the item at `offset`
...@@ -245,15 +335,19 @@ pub fn FixedSizeFifo(comptime T: type) type {...@@ -245,15 +335,19 @@ pub fn FixedSizeFifo(comptime T: type) type {
245 if (offset >= self.count)335 if (offset >= self.count)
246 return error.EndOfStream;336 return error.EndOfStream;
247337
248 return self.buf[(self.head + offset) % self.buf.len];338 var index = self.head + offset;
339 if (powers_of_two) {
340 index &= self.buf.len - 1;
341 } else {
342 index %= self.buf.len;
343 }
344 return self.buf[index];
249 }345 }
250 };346 };
251}347}
252348
253const ByteFifo = FixedSizeFifo(u8);349test "LinearFifo(u8, .Dynamic)" {
254350 var fifo = LinearFifo(u8, .Dynamic).init(debug.global_allocator);
255test "ByteFifo" {
256 var fifo = ByteFifo.init(debug.global_allocator);
257 defer fifo.deinit();351 defer fifo.deinit();
258352
259 try fifo.write("HELLO");353 try fifo.write("HELLO");
...@@ -304,7 +398,10 @@ test "ByteFifo" {...@@ -304,7 +398,10 @@ test "ByteFifo" {
304 {398 {
305 try fifo.unget("prependedstring");399 try fifo.unget("prependedstring");
306 var result: [30]u8 = undefined;400 var result: [30]u8 = undefined;
307 testing.expectEqualSlices(u8, "prependedstringabcdefghij", fifo.read(&result));401 testing.expectEqualSlices(u8, "prependedstringabcdefghij", result[0..fifo.read(&result)]);
402 try fifo.unget("b");
403 try fifo.unget("a");
404 testing.expectEqualSlices(u8, "ab", result[0..fifo.read(&result)]);
308 }405 }
309406
310 fifo.shrink(0);407 fifo.shrink(0);
...@@ -312,7 +409,33 @@ test "ByteFifo" {...@@ -312,7 +409,33 @@ test "ByteFifo" {
312 {409 {
313 try fifo.print("{}, {}!", "Hello", "World");410 try fifo.print("{}, {}!", "Hello", "World");
314 var result: [30]u8 = undefined;411 var result: [30]u8 = undefined;
315 testing.expectEqualSlices(u8, "Hello, World!", fifo.read(&result));412 testing.expectEqualSlices(u8, "Hello, World!", result[0..fifo.read(&result)]);
316 testing.expectEqual(@as(usize, 0), fifo.readableLength());413 testing.expectEqual(@as(usize, 0), fifo.readableLength());
317 }414 }
318}415}
416
417test "LinearFifo" {
418 inline for ([_]type{ u1, u8, u16, u64 }) |T| {
419 inline for ([_]LinearFifoBufferType{ LinearFifoBufferType{ .Static = 32 }, .Slice, .Dynamic }) |bt| {
420 const FifoType = LinearFifo(T, bt);
421 var buf: if (bt == .Slice) [32]T else void = undefined;
422 var fifo = switch (bt) {
423 .Static => FifoType.init(),
424 .Slice => FifoType.init(buf[0..]),
425 .Dynamic => FifoType.init(debug.global_allocator),
426 };
427 defer fifo.deinit();
428
429 try fifo.write([_]T{ 0, 1, 1, 0, 1 });
430 testing.expectEqual(@as(usize, 5), fifo.readableLength());
431
432 {
433 testing.expectEqual(@as(T, 0), try fifo.readItem());
434 testing.expectEqual(@as(T, 1), try fifo.readItem());
435 testing.expectEqual(@as(T, 1), try fifo.readItem());
436 testing.expectEqual(@as(T, 0), try fifo.readItem());
437 testing.expectEqual(@as(T, 1), try fifo.readItem());
438 }
439 }
440 }
441}