authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-02 21:10:36+01:00
committergravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-04 09:53:01+01:00
log8a963fd66e22820119bac1b0a568f0cf3af4fa8e
tree3bce31a144e7f100438d8945c53747ea48e01bf1
parent90c1a2c41aafb1e35e075fb7d0bdcf04c00db913

flate: 32 bit BitReader

Extend BitReader to accept size of internal buffer. It can be u64 (only option until now) or u32.

2 files changed, 84 insertions(+), 52 deletions(-)

lib/std/compress/flate/bit_reader.zig+83-51
...@@ -2,8 +2,16 @@ const std = @import("std");...@@ -2,8 +2,16 @@ const std = @import("std");
2const assert = std.debug.assert;2const assert = std.debug.assert;
3const testing = std.testing;3const testing = std.testing;
44
5pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {5pub fn bitReader(comptime T: type, reader: anytype) BitReader(T, @TypeOf(reader)) {
6 return BitReader(@TypeOf(reader)).init(reader);6 return BitReader(T, @TypeOf(reader)).init(reader);
7}
8
9pub fn BitReader64(comptime ReaderType: type) type {
10 return BitReader(u64, ReaderType);
11}
12
13pub fn BitReader32(comptime ReaderType: type) type {
14 return BitReader(u32, ReaderType);
7}15}
816
9/// Bit reader used during inflate (decompression). Has internal buffer of 6417/// Bit reader used during inflate (decompression). Has internal buffer of 64
...@@ -15,12 +23,16 @@ pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {...@@ -15,12 +23,16 @@ pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {
15/// fill buffer from forward_reader by calling fill in advance and readF with23/// fill buffer from forward_reader by calling fill in advance and readF with
16/// buffered flag set.24/// buffered flag set.
17///25///
18pub fn BitReader(comptime ReaderType: type) type {26pub fn BitReader(T: type, comptime ReaderType: type) type {
27 assert(T == u32 or T == u64);
28 const t_bytes: usize = @sizeOf(T);
29 const Tshift = if (T == u64) u6 else u5;
30
19 return struct {31 return struct {
20 // Underlying reader used for filling internal bits buffer32 // Underlying reader used for filling internal bits buffer
21 forward_reader: ReaderType = undefined,33 forward_reader: ReaderType = undefined,
22 // Internal buffer of 64 bits34 // Internal buffer of 64 bits
23 bits: u64 = 0,35 bits: T = 0,
24 // Number of bits in the buffer36 // Number of bits in the buffer
25 nbits: u32 = 0,37 nbits: u32 = 0,
2638
...@@ -51,14 +63,14 @@ pub fn BitReader(comptime ReaderType: type) type {...@@ -51,14 +63,14 @@ pub fn BitReader(comptime ReaderType: type) type {
5163
52 // Number of empty bytes in bits, round nbits to whole bytes.64 // Number of empty bytes in bits, round nbits to whole bytes.
53 const empty_bytes =65 const empty_bytes =
54 @as(u8, if (self.nbits & 0x7 == 0) 8 else 7) - // 8 for 8, 16, 24..., 7 otherwise66 @as(u8, if (self.nbits & 0x7 == 0) t_bytes else t_bytes - 1) - // 8 for 8, 16, 24..., 7 otherwise
55 (self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 867 (self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
5668
57 var buf: [8]u8 = [_]u8{0} ** 8;69 var buf: [t_bytes]u8 = [_]u8{0} ** t_bytes;
58 const bytes_read = self.forward_reader.readAll(buf[0..empty_bytes]) catch 0;70 const bytes_read = self.forward_reader.readAll(buf[0..empty_bytes]) catch 0;
59 if (bytes_read > 0) {71 if (bytes_read > 0) {
60 const u: u64 = std.mem.readInt(u64, buf[0..8], .little);72 const u: T = std.mem.readInt(T, buf[0..t_bytes], .little);
61 self.bits |= u << @as(u6, @intCast(self.nbits));73 self.bits |= u << @as(Tshift, @intCast(self.nbits));
62 self.nbits += 8 * @as(u8, @intCast(bytes_read));74 self.nbits += 8 * @as(u8, @intCast(bytes_read));
63 return;75 return;
64 }76 }
...@@ -99,7 +111,17 @@ pub fn BitReader(comptime ReaderType: type) type {...@@ -99,7 +111,17 @@ pub fn BitReader(comptime ReaderType: type) type {
99111
100 /// Read with flags provided.112 /// Read with flags provided.
101 pub fn readF(self: *Self, comptime U: type, comptime how: u3) !U {113 pub fn readF(self: *Self, comptime U: type, comptime how: u3) !U {
102 const n: u6 = @bitSizeOf(U);114 if (U == T) {
115 assert(how == 0);
116 assert(self.alignBits() == 0);
117 try self.fill(@bitSizeOf(T));
118 assert(self.nbits == @bitSizeOf(T));
119 const v = self.bits;
120 self.nbits = 0;
121 self.bits = 0;
122 return v;
123 }
124 const n: Tshift = @bitSizeOf(U);
103 switch (how) {125 switch (how) {
104 0 => { // `normal` read126 0 => { // `normal` read
105 try self.fill(n); // ensure that there are n bits in the buffer127 try self.fill(n); // ensure that there are n bits in the buffer
...@@ -157,7 +179,7 @@ pub fn BitReader(comptime ReaderType: type) type {...@@ -157,7 +179,7 @@ pub fn BitReader(comptime ReaderType: type) type {
157 }179 }
158180
159 /// Advance buffer for n bits.181 /// Advance buffer for n bits.
160 pub fn shift(self: *Self, n: u6) !void {182 pub fn shift(self: *Self, n: Tshift) !void {
161 if (n > self.nbits) return error.EndOfStream;183 if (n > self.nbits) return error.EndOfStream;
162 self.bits >>= n;184 self.bits >>= n;
163 self.nbits -= n;185 self.nbits -= n;
...@@ -218,10 +240,10 @@ pub fn BitReader(comptime ReaderType: type) type {...@@ -218,10 +240,10 @@ pub fn BitReader(comptime ReaderType: type) type {
218 };240 };
219}241}
220242
221test "BitReader" {243test "readF" {
222 var fbs = std.io.fixedBufferStream(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });244 var fbs = std.io.fixedBufferStream(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
223 var br = bitReader(fbs.reader());245 var br = bitReader(u64, fbs.reader());
224 const F = BitReader(@TypeOf(fbs.reader())).flag;246 const F = BitReader64(@TypeOf(fbs.reader())).flag;
225247
226 try testing.expectEqual(@as(u8, 48), br.nbits);248 try testing.expectEqual(@as(u8, 48), br.nbits);
227 try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);249 try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
...@@ -254,36 +276,38 @@ test "BitReader" {...@@ -254,36 +276,38 @@ test "BitReader" {
254}276}
255277
256test "read block type 1 data" {278test "read block type 1 data" {
257 const data = [_]u8{279 inline for ([_]type{ u64, u32 }) |T| {
258 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1280 const data = [_]u8{
259 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,281 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
260 0x0c, 0x01, 0x02, 0x03, //282 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
261 0xaa, 0xbb, 0xcc, 0xdd,283 0x0c, 0x01, 0x02, 0x03, //
262 };284 0xaa, 0xbb, 0xcc, 0xdd,
263 var fbs = std.io.fixedBufferStream(&data);285 };
264 var br = bitReader(fbs.reader());286 var fbs = std.io.fixedBufferStream(&data);
265 const F = BitReader(@TypeOf(fbs.reader())).flag;287 var br = bitReader(T, fbs.reader());
288 const F = BitReader(T, @TypeOf(fbs.reader())).flag;
266289
267 try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal290 try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
268 try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type291 try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
269292
270 for ("Hello world\n") |c| {293 for ("Hello world\n") |c| {
271 try testing.expectEqual(@as(u8, c), try br.readF(u8, F.reverse) - 0x30);294 try testing.expectEqual(@as(u8, c), try br.readF(u8, F.reverse) - 0x30);
295 }
296 try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
297 br.alignToByte();
298 try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
299 try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
300 try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
272 }301 }
273 try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
274 br.alignToByte();
275 try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
276 try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
277 try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
278}302}
279303
280test "init" {304test "shift/fill" {
281 const data = [_]u8{305 const data = [_]u8{
282 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,306 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
283 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,307 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
284 };308 };
285 var fbs = std.io.fixedBufferStream(&data);309 var fbs = std.io.fixedBufferStream(&data);
286 var br = bitReader(fbs.reader());310 var br = bitReader(u64, fbs.reader());
287311
288 try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);312 try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
289 try br.shift(8);313 try br.shift(8);
...@@ -303,31 +327,39 @@ test "init" {...@@ -303,31 +327,39 @@ test "init" {
303}327}
304328
305test "readAll" {329test "readAll" {
306 const data = [_]u8{330 inline for ([_]type{ u64, u32 }) |T| {
307 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,331 const data = [_]u8{
308 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,332 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
309 };333 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
310 var fbs = std.io.fixedBufferStream(&data);334 };
311 var br = bitReader(fbs.reader());335 var fbs = std.io.fixedBufferStream(&data);
336 var br = bitReader(T, fbs.reader());
312337
313 try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);338 switch (T) {
339 u64 => try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits),
340 u32 => try testing.expectEqual(@as(u32, 0x04_03_02_01), br.bits),
341 else => unreachable,
342 }
314343
315 var out: [16]u8 = undefined;344 var out: [16]u8 = undefined;
316 try br.readAll(out[0..]);345 try br.readAll(out[0..]);
317 try testing.expect(br.nbits == 0);346 try testing.expect(br.nbits == 0);
318 try testing.expect(br.bits == 0);347 try testing.expect(br.bits == 0);
319348
320 try testing.expectEqualSlices(u8, data[0..16], &out);349 try testing.expectEqualSlices(u8, data[0..16], &out);
350 }
321}351}
322352
323test "readFixedCode" {353test "readFixedCode" {
324 const fixed_codes = @import("huffman_encoder.zig").fixed_codes;354 inline for ([_]type{ u64, u32 }) |T| {
355 const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
325356
326 var fbs = std.io.fixedBufferStream(&fixed_codes);357 var fbs = std.io.fixedBufferStream(&fixed_codes);
327 var rdr = bitReader(fbs.reader());358 var rdr = bitReader(T, fbs.reader());
328359
329 for (0..286) |c| {360 for (0..286) |c| {
330 try testing.expectEqual(c, try rdr.readFixedCode());361 try testing.expectEqual(c, try rdr.readFixedCode());
362 }
363 try testing.expect(rdr.nbits == 0);
331 }364 }
332 try testing.expect(rdr.nbits == 0);
333}365}
lib/std/compress/flate/inflate.zig+1-1
...@@ -3,7 +3,7 @@ const assert = std.debug.assert;...@@ -3,7 +3,7 @@ const assert = std.debug.assert;
3const testing = std.testing;3const testing = std.testing;
44
5const hfd = @import("huffman_decoder.zig");5const hfd = @import("huffman_decoder.zig");
6const BitReader = @import("bit_reader.zig").BitReader;6const BitReader = @import("bit_reader.zig").BitReader64;
7const CircularBuffer = @import("CircularBuffer.zig");7const CircularBuffer = @import("CircularBuffer.zig");
8const Container = @import("container.zig").Container;8const Container = @import("container.zig").Container;
9const Token = @import("Token.zig");9const Token = @import("Token.zig");