authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-02 23:23:23+01:00
committergravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-04 09:53:01+01:00
logf2508abfa6461b4d28d6a8b1d72026810ff30a21
tree721b09e4c2ea229691d875f5aae51bd2750a422b
parent711281602a0efdd3b194c0419e8bccc2bb399167

flate: use 4 bytes lookahead for zlib

That ensures no bytes are left in the BitReader buffer after we reach end of the stream.

5 files changed, 26 insertions(+), 9 deletions(-)

lib/std/compress/flate.zig+1-1
...@@ -13,7 +13,7 @@ pub fn decompress(reader: anytype, writer: anytype) !void {...@@ -13,7 +13,7 @@ pub fn decompress(reader: anytype, writer: anytype) !void {
1313
14/// Decompressor type14/// Decompressor type
15pub fn Decompressor(comptime ReaderType: type) type {15pub fn Decompressor(comptime ReaderType: type) type {
16 return inflate.Inflate(.raw, ReaderType);16 return inflate.Decompressor(.raw, ReaderType);
17}17}
1818
19/// Create Decompressor which will read compressed data from reader.19/// Create Decompressor which will read compressed data from reader.
lib/std/compress/flate/container.zig+1
...@@ -154,6 +154,7 @@ pub const Container = enum {...@@ -154,6 +154,7 @@ pub const Container = enum {
154 pub fn parseFooter(comptime wrap: Container, hasher: *Hasher(wrap), reader: anytype) !void {154 pub fn parseFooter(comptime wrap: Container, hasher: *Hasher(wrap), reader: anytype) !void {
155 switch (wrap) {155 switch (wrap) {
156 .gzip => {156 .gzip => {
157 try reader.fill(0);
157 if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum;158 if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum;
158 if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize;159 if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize;
159 },160 },
lib/std/compress/flate/inflate.zig+22-6
...@@ -3,7 +3,7 @@ const assert = std.debug.assert;...@@ -3,7 +3,7 @@ const assert = std.debug.assert;
3const testing = std.testing;3const testing = std.testing;
44
5const hfd = @import("huffman_decoder.zig");5const hfd = @import("huffman_decoder.zig");
6const BitReader = @import("bit_reader.zig").BitReader64;6const BitReader = @import("bit_reader.zig").BitReader;
7const CircularBuffer = @import("CircularBuffer.zig");7const CircularBuffer = @import("CircularBuffer.zig");
8const Container = @import("container.zig").Container;8const Container = @import("container.zig").Container;
9const Token = @import("Token.zig");9const Token = @import("Token.zig");
...@@ -17,8 +17,16 @@ pub fn decompress(comptime container: Container, reader: anytype, writer: anytyp...@@ -17,8 +17,16 @@ pub fn decompress(comptime container: Container, reader: anytype, writer: anytyp
17}17}
1818
19/// Inflate decompressor for the reader type.19/// Inflate decompressor for the reader type.
20pub fn decompressor(comptime container: Container, reader: anytype) Inflate(container, @TypeOf(reader)) {20pub fn decompressor(comptime container: Container, reader: anytype) Decompressor(container, @TypeOf(reader)) {
21 return Inflate(container, @TypeOf(reader)).init(reader);21 return Decompressor(container, @TypeOf(reader)).init(reader);
22}
23
24pub fn Decompressor(comptime container: Container, comptime ReaderType: type) type {
25 // zlib has 4 bytes footer, lookahead of 4 bytes ensures that we will not overshoot.
26 // gzip has 8 bytes footer so we will not overshoot even with 8 bytes of lookahead.
27 // For raw deflate there is always possibility of overshot so we use 8 bytes lookahead.
28 const lookahead: type = if (container == .zlib) u32 else u64;
29 return Inflate(container, lookahead, ReaderType);
22}30}
2331
24/// Inflate decompresses deflate bit stream. Reads compressed data from reader32/// Inflate decompresses deflate bit stream. Reads compressed data from reader
...@@ -40,9 +48,12 @@ pub fn decompressor(comptime container: Container, reader: anytype) Inflate(cont...@@ -40,9 +48,12 @@ pub fn decompressor(comptime container: Container, reader: anytype) Inflate(cont
40/// * 64K for history (CircularBuffer)48/// * 64K for history (CircularBuffer)
41/// * ~10K huffman decoders (Literal and DistanceDecoder)49/// * ~10K huffman decoders (Literal and DistanceDecoder)
42///50///
43pub fn Inflate(comptime container: Container, comptime ReaderType: type) type {51pub fn Inflate(comptime container: Container, comptime LookaheadType: type, comptime ReaderType: type) type {
52 assert(LookaheadType == u32 or LookaheadType == u64);
53 const BitReaderType = BitReader(LookaheadType, ReaderType);
54
44 return struct {55 return struct {
45 const BitReaderType = BitReader(ReaderType);56 //const BitReaderType = BitReader(ReaderType);
46 const F = BitReaderType.flag;57 const F = BitReaderType.flag;
4758
48 bits: BitReaderType = .{},59 bits: BitReaderType = .{},
...@@ -219,9 +230,14 @@ pub fn Inflate(comptime container: Container, comptime ReaderType: type) type {...@@ -219,9 +230,14 @@ pub fn Inflate(comptime container: Container, comptime ReaderType: type) type {
219 switch (sym.kind) {230 switch (sym.kind) {
220 .literal => self.hist.write(sym.symbol),231 .literal => self.hist.write(sym.symbol),
221 .match => { // Decode match backreference <length, distance>232 .match => { // Decode match backreference <length, distance>
222 try self.bits.fill(5 + 15 + 13); // so we can use buffered reads233 // fill so we can use buffered reads
234 if (LookaheadType == u32)
235 try self.bits.fill(5 + 15)
236 else
237 try self.bits.fill(5 + 15 + 13);
223 const length = try self.decodeLength(sym.symbol);238 const length = try self.decodeLength(sym.symbol);
224 const dsm = try self.decodeSymbol(&self.dst_dec);239 const dsm = try self.decodeSymbol(&self.dst_dec);
240 if (LookaheadType == u32) try self.bits.fill(13);
225 const distance = try self.decodeDistance(dsm.symbol);241 const distance = try self.decodeDistance(dsm.symbol);
226 try self.hist.writeMatch(length, distance);242 try self.hist.writeMatch(length, distance);
227 },243 },
lib/std/compress/gzip.zig+1-1
...@@ -8,7 +8,7 @@ pub fn decompress(reader: anytype, writer: anytype) !void {...@@ -8,7 +8,7 @@ pub fn decompress(reader: anytype, writer: anytype) !void {
88
9/// Decompressor type9/// Decompressor type
10pub fn Decompressor(comptime ReaderType: type) type {10pub fn Decompressor(comptime ReaderType: type) type {
11 return inflate.Inflate(.gzip, ReaderType);11 return inflate.Decompressor(.gzip, ReaderType);
12}12}
1313
14/// Create Decompressor which will read compressed data from reader.14/// Create Decompressor which will read compressed data from reader.
lib/std/compress/zlib.zig+1-1
...@@ -8,7 +8,7 @@ pub fn decompress(reader: anytype, writer: anytype) !void {...@@ -8,7 +8,7 @@ pub fn decompress(reader: anytype, writer: anytype) !void {
88
9/// Decompressor type9/// Decompressor type
10pub fn Decompressor(comptime ReaderType: type) type {10pub fn Decompressor(comptime ReaderType: type) type {
11 return inflate.Inflate(.zlib, ReaderType);11 return inflate.Decompressor(.zlib, ReaderType);
12}12}
1313
14/// Create Decompressor which will read compressed data from reader.14/// Create Decompressor which will read compressed data from reader.