Commit 3e8cb153ea44ad1f403fbf6d23a67c839df57925 | zig

author	Igor Anić <igor.anic@gmail.com> 2024-02-17 18:49:49+01:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-02-17 15:31:13-08:00
log	3e8cb153ea44ad1f403fbf6d23a67c839df57925
tree	0a196079d351a3a64b3d19818064801a8323a732
parent	79e99c401ca32031d29aff77e43758ca0b67d5d9

fix flate regression

Until now literal and distance code lengths where treated as two
different arrays. But according to rfc they can overlap:

  The code length repeat codes can cross from HLIT + 257 to the
  HDIST + 1 code lengths.  In other words, all code lengths form
  a single sequence of HLIT + HDIST + 258 values.

Now code lengths are decoded in single array which is then split
to literal and distance part.

3 files changed, 42 insertions(+), 20 deletions(-)

lib
- std
  - compress
    flate
    inflate.zig
    testdata
    fuzz
    bug_18966.expect
    bug_18966.input

lib/std/compress/flate/inflate.zig+25-20

 @@ -157,30 +157,23 @@ pub fn Inflate(comptime container: Container, comptime ReaderType: type) type {
             var cl_dec: hfd.CodegenDecoder = .{};
             try cl_dec.generate(&cl_lens);
-            // literal code lengths
+            // decoded code lengths
-            var lit_lens = [_]u4{0} ** (286);
+            var dec_lens = [_]u4{0} ** (286 + 30);
             var pos: usize = 0;
-            while (pos < hlit) {
+            while (pos < hlit + hdist) {
                 const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
                 try self.bits.shift(sym.code_bits);
-                pos += try self.dynamicCodeLength(sym.symbol, &lit_lens, pos);
+                pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);
             }
-            if (pos > hlit)
+            if (pos > hlit + hdist) {
                 return error.InvalidDynamicBlockHeader;
-            // distance code lenths
-            var dst_lens = [_]u4{0} ** (30);
-            pos = 0;
-            while (pos < hdist) {
-                const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
-                try self.bits.shift(sym.code_bits);
-                pos += try self.dynamicCodeLength(sym.symbol, &dst_lens, pos);
             }
-            if (pos > hdist)
-                return error.InvalidDynamicBlockHeader;
-            try self.lit_dec.generate(&lit_lens);
+            // literal code lengts to literal decoder
-            try self.dst_dec.generate(&dst_lens);
+            try self.lit_dec.generate(dec_lens[0..hlit]);
+            // distance code lengths to distance decoder
+            try self.dst_dec.generate(dec_lens[hlit .. hlit + hdist]);
         }
         // Decode code length symbol to code length. Writes decoded length into
 @@ -496,7 +489,7 @@ test "flate.Inflate fuzzing tests" {
         .{ .input = "puff14", .err = error.EndOfStream },
         .{ .input = "puff15", .err = error.IncompleteHuffmanTree },
         .{ .input = "puff16", .err = error.InvalidDynamicBlockHeader },
-        .{ .input = "puff17", .err = error.InvalidDynamicBlockHeader }, // 25
+        .{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25
         .{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader },
         .{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader },
         .{ .input = "fuzz3", .err = error.InvalidMatch },
 @@ -506,8 +499,8 @@ test "flate.Inflate fuzzing tests" {
         .{ .input = "puff20", .err = error.OversubscribedHuffmanTree },
         .{ .input = "puff21", .err = error.OversubscribedHuffmanTree },
         .{ .input = "puff22", .err = error.OversubscribedHuffmanTree },
-        .{ .input = "puff23", .err = error.InvalidDynamicBlockHeader }, // 35
+        .{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35
-        .{ .input = "puff24", .err = error.InvalidDynamicBlockHeader },
+        .{ .input = "puff24", .err = error.IncompleteHuffmanTree },
         .{ .input = "puff25", .err = error.OversubscribedHuffmanTree },
         .{ .input = "puff26", .err = error.InvalidDynamicBlockHeader },
         .{ .input = "puff27", .err = error.InvalidDynamicBlockHeader },
 @@ -527,3 +520,15 @@ test "flate.Inflate fuzzing tests" {
         }
     }
 }
+test "flate bug 18966" {
+    const input = @embedFile("testdata/fuzz/bug_18966.input");
+    const expect = @embedFile("testdata/fuzz/bug_18966.expect");
+    var in = std.io.fixedBufferStream(input);
+    var out = std.ArrayList(u8).init(testing.allocator);
+    defer out.deinit();
+    try decompress(.gzip, in.reader(), out.writer());
+    try testing.expectEqualStrings(expect, out.items);
+}

lib/std/compress/flate/testdata/fuzz/bug_18966.expect created+17

...	@@ -0,0 +1,17 @@
1	conditions,correction,exchange,id,participant_timestamp,price,sequence_number,sip_timestamp,size,tape,trf_id,trf_timestamp
2	,,1,,0,9.92,1929969624145677,1063224024145000000,200,1,,0
3	,,1,,0,9.86,1929967211195252,1063221611195000000,100,1,,0
4	,,1,,0,9.86,1929967094672373,1063221494672000000,200,1,,0
5	,,1,,0,9.85,1929967050174259,1063221450174000000,100,1,,0
6	,,1,,0,9.85,1929967013672246,1063221413672000000,200,1,,0
7	,,1,,0,9.85,1929966984671861,1063221384671000000,200,1,,0
8	,,1,,0,9.9,1929966976177115,1063221376177000000,200,1,,0
9	,,1,,0,9.85,1929966961186410,1063221361186000000,1000,1,,0
10	,,1,,0,9.85,1929957130426170,1063211530426000000,100,1,,0
11	,,1,,0,9.85,1929956938455804,1063211338455000000,100,1,,0
12	,,1,,0,9.85,1929956786452149,1063211186452000000,300,1,,0
13	,,1,,0,9.85,1929956658456522,1063211058456000000,500,1,,0
14	,,1,,0,9.85,1929956650952287,1063211050952000000,300,1,,0
15	,,1,,0,9.85,1929956528452578,1063210928452000000,1500,1,,0
16	,,1,,0,9.85,1929950990948195,1063205390948000000,100,1,,0
17	,,1,,0,9.85,1929947489432428,1063201889432000000,100,1,,0

lib/std/compress/flate/testdata/fuzz/bug_18966.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/bug_18966.input differ