authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-26 20:33:45-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-26 21:00:58-07:00
log722e066173f5235b1dd1b341acb5cb1e79002ed7
treed35a48a883217774d239199995d8282c10a8bc2b
parentd87eb7d4e4f2ea606a18640fcc019b60cc435cdd

std.compress.xz.Decompress: some tests passing

4 files changed, 224 insertions(+), 164 deletions(-)

lib/std/compress/lzma.zig+58-41
...@@ -12,11 +12,19 @@ pub const RangeDecoder = struct {...@@ -12,11 +12,19 @@ pub const RangeDecoder = struct {
12 code: u32,12 code: u32,
1313
14 pub fn init(reader: *Reader) !RangeDecoder {14 pub fn init(reader: *Reader) !RangeDecoder {
15 var counter: u64 = 0;
16 return initCounting(reader, &counter);
17 }
18
19 pub fn initCounting(reader: *Reader, n_read: *u64) !RangeDecoder {
15 const reserved = try reader.takeByte();20 const reserved = try reader.takeByte();
21 n_read.* += 1;
16 if (reserved != 0) return error.InvalidRangeCode;22 if (reserved != 0) return error.InvalidRangeCode;
23 const code = try reader.takeInt(u32, .big);
24 n_read.* += 4;
17 return .{25 return .{
18 .range = 0xFFFF_FFFF,26 .range = 0xFFFF_FFFF,
19 .code = try reader.takeInt(u32, .big),27 .code = code,
20 };28 };
21 }29 }
2230
...@@ -24,47 +32,47 @@ pub const RangeDecoder = struct {...@@ -24,47 +32,47 @@ pub const RangeDecoder = struct {
24 return self.code == 0;32 return self.code == 0;
25 }33 }
2634
27 fn normalize(self: *RangeDecoder, reader: *Reader) !void {35 fn normalize(self: *RangeDecoder, reader: *Reader, n_read: *u64) !void {
28 if (self.range < 0x0100_0000) {36 if (self.range < 0x0100_0000) {
29 self.range <<= 8;37 self.range <<= 8;
30 self.code = (self.code << 8) ^ @as(u32, try reader.takeByte());38 self.code = (self.code << 8) ^ @as(u32, try reader.takeByte());
39 n_read.* += 1;
31 }40 }
32 }41 }
3342
34 fn getBit(self: *RangeDecoder, reader: *Reader) !bool {43 fn getBit(self: *RangeDecoder, reader: *Reader, n_read: *u64) !bool {
35 self.range >>= 1;44 self.range >>= 1;
3645
37 const bit = self.code >= self.range;46 const bit = self.code >= self.range;
38 if (bit)47 if (bit) self.code -= self.range;
39 self.code -= self.range;
4048
41 try self.normalize(reader);49 try self.normalize(reader, n_read);
42 return bit;50 return bit;
43 }51 }
4452
45 pub fn get(self: *RangeDecoder, reader: *Reader, count: usize) !u32 {53 pub fn get(self: *RangeDecoder, reader: *Reader, count: usize, n_read: *u64) !u32 {
46 var result: u32 = 0;54 var result: u32 = 0;
47 var i: usize = 0;55 for (0..count) |_| {
48 while (i < count) : (i += 1)56 result = (result << 1) ^ @intFromBool(try self.getBit(reader, n_read));
49 result = (result << 1) ^ @intFromBool(try self.getBit(reader));57 }
50 return result;58 return result;
51 }59 }
5260
53 pub fn decodeBit(self: *RangeDecoder, reader: *Reader, prob: *u16) !bool {61 pub fn decodeBit(self: *RangeDecoder, reader: *Reader, prob: *u16, n_read: *u64) !bool {
54 const bound = (self.range >> 11) * prob.*;62 const bound = (self.range >> 11) * prob.*;
5563
56 if (self.code < bound) {64 if (self.code < bound) {
57 prob.* += (0x800 - prob.*) >> 5;65 prob.* += (0x800 - prob.*) >> 5;
58 self.range = bound;66 self.range = bound;
5967
60 try self.normalize(reader);68 try self.normalize(reader, n_read);
61 return false;69 return false;
62 } else {70 } else {
63 prob.* -= prob.* >> 5;71 prob.* -= prob.* >> 5;
64 self.code -= bound;72 self.code -= bound;
65 self.range -= bound;73 self.range -= bound;
6674
67 try self.normalize(reader);75 try self.normalize(reader, n_read);
68 return true;76 return true;
69 }77 }
70 }78 }
...@@ -74,11 +82,12 @@ pub const RangeDecoder = struct {...@@ -74,11 +82,12 @@ pub const RangeDecoder = struct {
74 reader: *Reader,82 reader: *Reader,
75 num_bits: u5,83 num_bits: u5,
76 probs: []u16,84 probs: []u16,
85 n_read: *u64,
77 ) !u32 {86 ) !u32 {
78 var tmp: u32 = 1;87 var tmp: u32 = 1;
79 var i: @TypeOf(num_bits) = 0;88 var i: @TypeOf(num_bits) = 0;
80 while (i < num_bits) : (i += 1) {89 while (i < num_bits) : (i += 1) {
81 const bit = try self.decodeBit(reader, &probs[tmp]);90 const bit = try self.decodeBit(reader, &probs[tmp], n_read);
82 tmp = (tmp << 1) ^ @intFromBool(bit);91 tmp = (tmp << 1) ^ @intFromBool(bit);
83 }92 }
84 return tmp - (@as(u32, 1) << num_bits);93 return tmp - (@as(u32, 1) << num_bits);
...@@ -90,12 +99,13 @@ pub const RangeDecoder = struct {...@@ -90,12 +99,13 @@ pub const RangeDecoder = struct {
90 num_bits: u5,99 num_bits: u5,
91 probs: []u16,100 probs: []u16,
92 offset: usize,101 offset: usize,
102 n_read: *u64,
93 ) !u32 {103 ) !u32 {
94 var result: u32 = 0;104 var result: u32 = 0;
95 var tmp: usize = 1;105 var tmp: usize = 1;
96 var i: @TypeOf(num_bits) = 0;106 var i: @TypeOf(num_bits) = 0;
97 while (i < num_bits) : (i += 1) {107 while (i < num_bits) : (i += 1) {
98 const bit = @intFromBool(try self.decodeBit(reader, &probs[offset + tmp]));108 const bit = @intFromBool(try self.decodeBit(reader, &probs[offset + tmp], n_read));
99 tmp = (tmp << 1) ^ bit;109 tmp = (tmp << 1) ^ bit;
100 result ^= @as(u32, bit) << i;110 result ^= @as(u32, bit) << i;
101 }111 }
...@@ -177,13 +187,14 @@ pub const Decode = struct {...@@ -177,13 +187,14 @@ pub const Decode = struct {
177 /// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.187 /// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.
178 buffer: anytype,188 buffer: anytype,
179 decoder: *RangeDecoder,189 decoder: *RangeDecoder,
190 n_read: *u64,
180 ) !ProcessingStatus {191 ) !ProcessingStatus {
181 const gpa = allocating.allocator;192 const gpa = allocating.allocator;
182 const writer = &allocating.writer;193 const writer = &allocating.writer;
183 const pos_state = buffer.len & ((@as(usize, 1) << self.properties.pb) - 1);194 const pos_state = buffer.len & ((@as(usize, 1) << self.properties.pb) - 1);
184195
185 if (!try decoder.decodeBit(reader, &self.is_match[(self.state << 4) + pos_state])) {196 if (!try decoder.decodeBit(reader, &self.is_match[(self.state << 4) + pos_state], n_read)) {
186 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder);197 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, n_read);
187198
188 try buffer.appendLiteral(gpa, byte, writer);199 try buffer.appendLiteral(gpa, byte, writer);
189200
...@@ -197,18 +208,18 @@ pub const Decode = struct {...@@ -197,18 +208,18 @@ pub const Decode = struct {
197 }208 }
198209
199 var len: usize = undefined;210 var len: usize = undefined;
200 if (try decoder.decodeBit(reader, &self.is_rep[self.state])) {211 if (try decoder.decodeBit(reader, &self.is_rep[self.state], n_read)) {
201 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state])) {212 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], n_read)) {
202 if (!try decoder.decodeBit(reader, &self.is_rep_0long[(self.state << 4) + pos_state])) {213 if (!try decoder.decodeBit(reader, &self.is_rep_0long[(self.state << 4) + pos_state], n_read)) {
203 self.state = if (self.state < 7) 9 else 11;214 self.state = if (self.state < 7) 9 else 11;
204 const dist = self.rep[0] + 1;215 const dist = self.rep[0] + 1;
205 try buffer.appendLz(gpa, 1, dist, writer);216 try buffer.appendLz(gpa, 1, dist, writer);
206 return .more;217 return .more;
207 }218 }
208 } else {219 } else {
209 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state]))220 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], n_read))
210 1221 1
211 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state]))222 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], n_read))
212 2223 2
213 else224 else
214 3;225 3;
...@@ -220,7 +231,7 @@ pub const Decode = struct {...@@ -220,7 +231,7 @@ pub const Decode = struct {
220 self.rep[0] = dist;231 self.rep[0] = dist;
221 }232 }
222233
223 len = try self.rep_len_decoder.decode(reader, decoder, pos_state);234 len = try self.rep_len_decoder.decode(reader, decoder, pos_state, n_read);
224235
225 self.state = if (self.state < 7) 8 else 11;236 self.state = if (self.state < 7) 8 else 11;
226 } else {237 } else {
...@@ -228,11 +239,11 @@ pub const Decode = struct {...@@ -228,11 +239,11 @@ pub const Decode = struct {
228 self.rep[2] = self.rep[1];239 self.rep[2] = self.rep[1];
229 self.rep[1] = self.rep[0];240 self.rep[1] = self.rep[0];
230241
231 len = try self.len_decoder.decode(reader, decoder, pos_state);242 len = try self.len_decoder.decode(reader, decoder, pos_state, n_read);
232243
233 self.state = if (self.state < 7) 7 else 10;244 self.state = if (self.state < 7) 7 else 10;
234245
235 const rep_0 = try self.decodeDistance(reader, decoder, len);246 const rep_0 = try self.decodeDistance(reader, decoder, len, n_read);
236247
237 self.rep[0] = rep_0;248 self.rep[0] = rep_0;
238 if (self.rep[0] == 0xFFFF_FFFF) {249 if (self.rep[0] == 0xFFFF_FFFF) {
...@@ -257,6 +268,7 @@ pub const Decode = struct {...@@ -257,6 +268,7 @@ pub const Decode = struct {
257 /// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.268 /// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.
258 buffer: anytype,269 buffer: anytype,
259 decoder: *RangeDecoder,270 decoder: *RangeDecoder,
271 n_read: *u64,
260 ) !u8 {272 ) !u8 {
261 const def_prev_byte = 0;273 const def_prev_byte = 0;
262 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));274 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
...@@ -275,6 +287,7 @@ pub const Decode = struct {...@@ -275,6 +287,7 @@ pub const Decode = struct {
275 const bit = @intFromBool(try decoder.decodeBit(287 const bit = @intFromBool(try decoder.decodeBit(
276 reader,288 reader,
277 &probs[((@as(usize, 1) + match_bit) << 8) + result],289 &probs[((@as(usize, 1) + match_bit) << 8) + result],
290 n_read,
278 ));291 ));
279 result = (result << 1) ^ bit;292 result = (result << 1) ^ bit;
280 if (match_bit != bit) {293 if (match_bit != bit) {
...@@ -284,10 +297,10 @@ pub const Decode = struct {...@@ -284,10 +297,10 @@ pub const Decode = struct {
284 }297 }
285298
286 while (result < 0x100) {299 while (result < 0x100) {
287 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result]));300 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result], n_read));
288 }301 }
289302
290 return @as(u8, @truncate(result - 0x100));303 return @truncate(result - 0x100);
291 }304 }
292305
293 fn decodeDistance(306 fn decodeDistance(
...@@ -295,12 +308,12 @@ pub const Decode = struct {...@@ -295,12 +308,12 @@ pub const Decode = struct {
295 reader: *Reader,308 reader: *Reader,
296 decoder: *RangeDecoder,309 decoder: *RangeDecoder,
297 length: usize,310 length: usize,
311 n_read: *u64,
298 ) !usize {312 ) !usize {
299 const len_state = if (length > 3) 3 else length;313 const len_state = if (length > 3) 3 else length;
300314
301 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder));315 const pos_slot: usize = try self.pos_slot_decoder[len_state].parse(reader, decoder, n_read);
302 if (pos_slot < 4)316 if (pos_slot < 4) return pos_slot;
303 return pos_slot;
304317
305 const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));318 const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
306 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;319 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
...@@ -311,10 +324,11 @@ pub const Decode = struct {...@@ -311,10 +324,11 @@ pub const Decode = struct {
311 num_direct_bits,324 num_direct_bits,
312 &self.pos_decoders,325 &self.pos_decoders,
313 result - pos_slot,326 result - pos_slot,
327 n_read,
314 );328 );
315 } else {329 } else {
316 result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4;330 result += @as(usize, try decoder.get(reader, num_direct_bits - 4, n_read)) << 4;
317 result += try self.align_decoder.parseReverse(reader, decoder);331 result += try self.align_decoder.parseReverse(reader, decoder, n_read);
318 }332 }
319333
320 return result;334 return result;
...@@ -435,16 +449,17 @@ pub const Decode = struct {...@@ -435,16 +449,17 @@ pub const Decode = struct {
435 return struct {449 return struct {
436 probs: [1 << num_bits]u16 = @splat(0x400),450 probs: [1 << num_bits]u16 = @splat(0x400),
437451
438 pub fn parse(self: *@This(), reader: *Reader, decoder: *RangeDecoder) !u32 {452 pub fn parse(self: *@This(), reader: *Reader, decoder: *RangeDecoder, n_read: *u64) !u32 {
439 return decoder.parseBitTree(reader, num_bits, &self.probs);453 return decoder.parseBitTree(reader, num_bits, &self.probs, n_read);
440 }454 }
441455
442 pub fn parseReverse(456 pub fn parseReverse(
443 self: *@This(),457 self: *@This(),
444 reader: *Reader,458 reader: *Reader,
445 decoder: *RangeDecoder,459 decoder: *RangeDecoder,
460 n_read: *u64,
446 ) !u32 {461 ) !u32 {
447 return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0);462 return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, n_read);
448 }463 }
449464
450 pub fn reset(self: *@This()) void {465 pub fn reset(self: *@This()) void {
...@@ -465,13 +480,14 @@ pub const Decode = struct {...@@ -465,13 +480,14 @@ pub const Decode = struct {
465 reader: *Reader,480 reader: *Reader,
466 decoder: *RangeDecoder,481 decoder: *RangeDecoder,
467 pos_state: usize,482 pos_state: usize,
483 n_read: *u64,
468 ) !usize {484 ) !usize {
469 if (!try decoder.decodeBit(reader, &self.choice)) {485 if (!try decoder.decodeBit(reader, &self.choice, n_read)) {
470 return @as(usize, try self.low_coder[pos_state].parse(reader, decoder));486 return @as(usize, try self.low_coder[pos_state].parse(reader, decoder, n_read));
471 } else if (!try decoder.decodeBit(reader, &self.choice2)) {487 } else if (!try decoder.decodeBit(reader, &self.choice2, n_read)) {
472 return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder)) + 8;488 return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder, n_read)) + 8;
473 } else {489 } else {
474 return @as(usize, try self.high_coder.parse(reader, decoder)) + 16;490 return @as(usize, try self.high_coder.parse(reader, decoder, n_read)) + 16;
475 }491 }
476 }492 }
477493
...@@ -701,7 +717,8 @@ pub const Decompress = struct {...@@ -701,7 +717,8 @@ pub const Decompress = struct {
701 } else if (d.range_decoder.isFinished()) {717 } else if (d.range_decoder.isFinished()) {
702 break :process_next;718 break :process_next;
703 }719 }
704 switch (d.decode.process(d.input, &allocating, &d.buffer, &d.range_decoder) catch |err| switch (err) {720 var n_read: u64 = 0;
721 switch (d.decode.process(d.input, &allocating, &d.buffer, &d.range_decoder, &n_read) catch |err| switch (err) {
705 error.WriteFailed => {722 error.WriteFailed => {
706 d.err = error.OutOfMemory;723 d.err = error.OutOfMemory;
707 return error.ReadFailed;724 return error.ReadFailed;
lib/std/compress/lzma2.zig+25-17
...@@ -116,24 +116,29 @@ pub const Decode = struct {...@@ -116,24 +116,29 @@ pub const Decode = struct {
116 self.* = undefined;116 self.* = undefined;
117 }117 }
118118
119 pub fn decompress(d: *Decode, reader: *Reader, allocating: *Writer.Allocating) !void {119 /// Returns how many compressed bytes were consumed.
120 pub fn decompress(d: *Decode, reader: *Reader, allocating: *Writer.Allocating) !u64 {
120 const gpa = allocating.allocator;121 const gpa = allocating.allocator;
121122
122 var accum = AccumBuffer.init(std.math.maxInt(usize));123 var accum = AccumBuffer.init(std.math.maxInt(usize));
123 defer accum.deinit(gpa);124 defer accum.deinit(gpa);
124125
126 var n_read: u64 = 0;
127
125 while (true) {128 while (true) {
126 const status = try reader.takeByte();129 const status = try reader.takeByte();
130 n_read += 1;
127131
128 switch (status) {132 switch (status) {
129 0 => break,133 0 => break,
130 1 => try parseUncompressed(reader, allocating, &accum, true),134 1 => n_read += try parseUncompressed(reader, allocating, &accum, true),
131 2 => try parseUncompressed(reader, allocating, &accum, false),135 2 => n_read += try parseUncompressed(reader, allocating, &accum, false),
132 else => try d.parseLzma(reader, allocating, &accum, status),136 else => n_read += try d.parseLzma(reader, allocating, &accum, status),
133 }137 }
134 }138 }
135139
136 try accum.finish(&allocating.writer);140 try accum.finish(&allocating.writer);
141 return n_read;
137 }142 }
138143
139 fn parseLzma(144 fn parseLzma(
...@@ -142,7 +147,7 @@ pub const Decode = struct {...@@ -142,7 +147,7 @@ pub const Decode = struct {
142 allocating: *Writer.Allocating,147 allocating: *Writer.Allocating,
143 accum: *AccumBuffer,148 accum: *AccumBuffer,
144 status: u8,149 status: u8,
145 ) !void {150 ) !u64 {
146 if (status & 0x80 == 0) return error.CorruptInput;151 if (status & 0x80 == 0) return error.CorruptInput;
147152
148 const Reset = struct {153 const Reset = struct {
...@@ -175,15 +180,19 @@ pub const Decode = struct {...@@ -175,15 +180,19 @@ pub const Decode = struct {
175 else => unreachable,180 else => unreachable,
176 };181 };
177182
183 var n_read: u64 = 0;
184
178 const unpacked_size = blk: {185 const unpacked_size = blk: {
179 var tmp: u64 = status & 0x1F;186 var tmp: u64 = status & 0x1F;
180 tmp <<= 16;187 tmp <<= 16;
181 tmp |= try reader.takeInt(u16, .big);188 tmp |= try reader.takeInt(u16, .big);
189 n_read += 2;
182 break :blk tmp + 1;190 break :blk tmp + 1;
183 };191 };
184192
185 const packed_size = blk: {193 const packed_size = blk: {
186 const tmp: u17 = try reader.takeInt(u16, .big);194 const tmp: u17 = try reader.takeInt(u16, .big);
195 n_read += 2;
187 break :blk tmp + 1;196 break :blk tmp + 1;
188 };197 };
189198
...@@ -196,6 +205,7 @@ pub const Decode = struct {...@@ -196,6 +205,7 @@ pub const Decode = struct {
196205
197 if (reset.props) {206 if (reset.props) {
198 var props = try reader.takeByte();207 var props = try reader.takeByte();
208 n_read += 1;
199 if (props >= 225) {209 if (props >= 225) {
200 return error.CorruptInput;210 return error.CorruptInput;
201 }211 }
...@@ -216,23 +226,21 @@ pub const Decode = struct {...@@ -216,23 +226,21 @@ pub const Decode = struct {
216 try ld.resetState(allocating.allocator, new_props);226 try ld.resetState(allocating.allocator, new_props);
217 }227 }
218228
219 var range_decoder = try lzma.RangeDecoder.init(reader);229 const start_count = n_read;
230 var range_decoder = try lzma.RangeDecoder.initCounting(reader, &n_read);
220231
221 while (true) {232 while (true) {
222 if (accum.len >= unpacked_size) break;233 if (accum.len >= unpacked_size) break;
223 if (range_decoder.isFinished()) break;234 if (range_decoder.isFinished()) break;
224 switch (try ld.process(reader, allocating, accum, &range_decoder)) {235 switch (try ld.process(reader, allocating, accum, &range_decoder, &n_read)) {
225 .more => continue,236 .more => continue,
226 .finished => break,237 .finished => break,
227 }238 }
228 }239 }
229 if (accum.len != unpacked_size) return error.DecompressedSizeMismatch;240 if (accum.len != unpacked_size) return error.DecompressedSizeMismatch;
241 if (n_read - start_count != packed_size) return error.CompressedSizeMismatch;
230242
231 // TODO restore this error243 return n_read;
232 //if (counter.bytes_read != packed_size) {
233 // return error.CorruptInput;
234 //}
235 _ = packed_size;
236 }244 }
237245
238 fn parseUncompressed(246 fn parseUncompressed(
...@@ -240,18 +248,17 @@ pub const Decode = struct {...@@ -240,18 +248,17 @@ pub const Decode = struct {
240 allocating: *Writer.Allocating,248 allocating: *Writer.Allocating,
241 accum: *AccumBuffer,249 accum: *AccumBuffer,
242 reset_dict: bool,250 reset_dict: bool,
243 ) !void {251 ) !usize {
244 const unpacked_size = @as(u17, try reader.takeInt(u16, .big)) + 1;252 const unpacked_size = @as(u17, try reader.takeInt(u16, .big)) + 1;
245253
246 if (reset_dict) try accum.reset(&allocating.writer);254 if (reset_dict) try accum.reset(&allocating.writer);
247255
248 const gpa = allocating.allocator;256 const gpa = allocating.allocator;
249257
250 var i = unpacked_size;258 for (0..unpacked_size) |_| {
251 while (i != 0) {
252 try accum.appendByte(gpa, try reader.takeByte());259 try accum.appendByte(gpa, try reader.takeByte());
253 i -= 1;
254 }260 }
261 return 2 + unpacked_size;
255 }262 }
256};263};
257264
...@@ -268,6 +275,7 @@ test "decompress hello world stream" {...@@ -268,6 +275,7 @@ test "decompress hello world stream" {
268 var result: std.Io.Writer.Allocating = .init(gpa);275 var result: std.Io.Writer.Allocating = .init(gpa);
269 defer result.deinit();276 defer result.deinit();
270277
271 try decode.decompress(&stream, &result);278 const n_read = try decode.decompress(&stream, &result);
279 try std.testing.expectEqual(compressed.len, n_read);
272 try std.testing.expectEqualStrings(expected, result.written());280 try std.testing.expectEqualStrings(expected, result.written());
273}281}
lib/std/compress/xz/Decompress.zig+72-69
...@@ -8,6 +8,7 @@ const Sha256 = std.crypto.hash.sha2.Sha256;...@@ -8,6 +8,7 @@ const Sha256 = std.crypto.hash.sha2.Sha256;
8const lzma2 = std.compress.lzma2;8const lzma2 = std.compress.lzma2;
9const Writer = std.Io.Writer;9const Writer = std.Io.Writer;
10const Reader = std.Io.Reader;10const Reader = std.Io.Reader;
11const assert = std.debug.assert;
1112
12/// Underlying compressed data stream to pull bytes from.13/// Underlying compressed data stream to pull bytes from.
13input: *Reader,14input: *Reader,
...@@ -28,6 +29,7 @@ pub const Error = error{...@@ -28,6 +29,7 @@ pub const Error = error{
28 Overflow,29 Overflow,
29 InvalidRangeCode,30 InvalidRangeCode,
30 DecompressedSizeMismatch,31 DecompressedSizeMismatch,
32 CompressedSizeMismatch,
31};33};
3234
33pub const Check = enum(u4) {35pub const Check = enum(u4) {
...@@ -62,10 +64,10 @@ pub fn init(...@@ -62,10 +64,10 @@ pub fn init(
62 if (!std.mem.eql(u8, magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))64 if (!std.mem.eql(u8, magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
63 return error.NotXzStream;65 return error.NotXzStream;
6466
65 const actual_hash = Crc32.hash(try input.peek(@sizeOf(StreamFlags)));67 const computed_checksum = Crc32.hash(try input.peek(@sizeOf(StreamFlags)));
66 const stream_flags = input.takeStruct(StreamFlags, .little) catch unreachable;68 const stream_flags = input.takeStruct(StreamFlags, .little) catch unreachable;
67 const stored_hash = try input.takeInt(u32, .little);69 const stored_hash = try input.takeInt(u32, .little);
68 if (actual_hash != stored_hash) return error.WrongChecksum;70 if (computed_checksum != stored_hash) return error.WrongChecksum;
6971
70 return .{72 return .{
71 .input = input,73 .input = input,
...@@ -129,6 +131,7 @@ fn readIndirect(r: *Reader) Reader.Error!usize {...@@ -129,6 +131,7 @@ fn readIndirect(r: *Reader) Reader.Error!usize {
129 r.end = allocating.writer.end;131 r.end = allocating.writer.end;
130 }132 }
131133
134 if (d.err != null) return error.ReadFailed;
132 if (d.block_count == std.math.maxInt(usize)) return error.EndOfStream;135 if (d.block_count == std.math.maxInt(usize)) return error.EndOfStream;
133136
134 readBlock(input, &allocating) catch |err| switch (err) {137 readBlock(input, &allocating) catch |err| switch (err) {
...@@ -137,7 +140,10 @@ fn readIndirect(r: *Reader) Reader.Error!usize {...@@ -137,7 +140,10 @@ fn readIndirect(r: *Reader) Reader.Error!usize {
137 return error.ReadFailed;140 return error.ReadFailed;
138 },141 },
139 error.SuccessfulEndOfStream => {142 error.SuccessfulEndOfStream => {
140 finish(d);143 finish(d) catch |finish_err| {
144 d.err = finish_err;
145 return error.ReadFailed;
146 };
141 d.block_count = std.math.maxInt(usize);147 d.block_count = std.math.maxInt(usize);
142 return error.EndOfStream;148 return error.EndOfStream;
143 },149 },
...@@ -184,7 +190,7 @@ fn readBlock(input: *Reader, allocating: *Writer.Allocating) !void {...@@ -184,7 +190,7 @@ fn readBlock(input: *Reader, allocating: *Writer.Allocating) !void {
184 var packed_size: ?u64 = null;190 var packed_size: ?u64 = null;
185 var unpacked_size: ?u64 = null;191 var unpacked_size: ?u64 = null;
186192
187 {193 const header_size = h: {
188 // Read the block header via peeking so that we can hash the whole thing too.194 // Read the block header via peeking so that we can hash the whole thing too.
189 const first_byte: usize = try input.peekByte();195 const first_byte: usize = try input.peekByte();
190 if (first_byte == 0) return error.SuccessfulEndOfStream;196 if (first_byte == 0) return error.SuccessfulEndOfStream;
...@@ -223,95 +229,92 @@ fn readBlock(input: *Reader, allocating: *Writer.Allocating) !void {...@@ -223,95 +229,92 @@ fn readBlock(input: *Reader, allocating: *Writer.Allocating) !void {
223229
224 const actual_header_size = input.seek - header_seek_start;230 const actual_header_size = input.seek - header_seek_start;
225 if (actual_header_size > declared_header_size) return error.CorruptInput;231 if (actual_header_size > declared_header_size) return error.CorruptInput;
226 var remaining_bytes = declared_header_size - actual_header_size;232 const remaining_bytes = declared_header_size - actual_header_size;
227 while (remaining_bytes != 0) {233 for (0..remaining_bytes) |_| {
228 if (try input.takeByte() != 0) return error.CorruptInput;234 if (try input.takeByte() != 0) return error.CorruptInput;
229 remaining_bytes -= 1;
230 }235 }
231236
232 const header_slice = input.buffer[header_seek_start..][0..declared_header_size];237 const header_slice = input.buffer[header_seek_start..][0..declared_header_size];
233 const actual_hash = Crc32.hash(header_slice);238 const computed_checksum = Crc32.hash(header_slice);
234 const declared_hash = try input.takeInt(u32, .little);239 const declared_checksum = try input.takeInt(u32, .little);
235 if (actual_hash != declared_hash) return error.WrongChecksum;240 if (computed_checksum != declared_checksum) return error.WrongChecksum;
236 }241 break :h declared_header_size;
242 };
237243
238 // Compressed Data244 // Compressed Data
239245
240 var lzma2_decode = try lzma2.Decode.init(allocating.allocator);246 var lzma2_decode = try lzma2.Decode.init(allocating.allocator);
247 defer lzma2_decode.deinit(allocating.allocator);
241 const before_size = allocating.writer.end;248 const before_size = allocating.writer.end;
242 try lzma2_decode.decompress(input, allocating);249 const packed_bytes_read = try lzma2_decode.decompress(input, allocating);
243 const unpacked_bytes = allocating.writer.end - before_size;250 const unpacked_bytes = allocating.writer.end - before_size;
244251
245 // TODO restore this check252 if (packed_size) |s| {
246 //if (packed_size) |s| {253 if (s != packed_bytes_read) return error.CorruptInput;
247 // if (s != packed_counter.bytes_read)254 }
248 // return error.CorruptInput;
249 //}
250255
251 if (unpacked_size) |s| {256 if (unpacked_size) |s| {
252 if (s != unpacked_bytes) return error.CorruptInput;257 if (s != unpacked_bytes) return error.CorruptInput;
253 }258 }
254259
255 // Block Padding260 // Block Padding
256 if (true) @panic("TODO account for block padding");261 const block_counter = header_size + packed_bytes_read;
257 //while (block_counter.bytes_read % 4 != 0) {262 const padding = (4 - (block_counter % 4)) % 4;
258 // if (try block_reader.takeByte() != 0)263 for (0..padding) |_| {
259 // return error.CorruptInput;264 if (try input.takeByte() != 0) return error.CorruptInput;
260 //}265 }
261
262}266}
263267
264fn finish(d: *Decompress) void {268fn finish(d: *Decompress) !void {
265 _ = d;269 const input = d.input;
266 @panic("TODO");270 const index_size = blk: {
267 //const input = d.input;271 // Assume that we already peeked a zero in readBlock().
268 //const index_size = blk: {272 assert(input.buffered()[0] == 0);
269 // const record_count = try input.takeLeb128(u64);273 var input_counter: u64 = 1;
270 // if (record_count != d.block_decode.block_count)274 var checksum: Crc32 = .init();
271 // return error.CorruptInput;275 checksum.update(&.{0});
272276 input.toss(1);
273 // var i: usize = 0;
274 // while (i < record_count) : (i += 1) {
275 // // TODO: validate records
276 // _ = try std.leb.readUleb128(u64, counting_reader);
277 // _ = try std.leb.readUleb128(u64, counting_reader);
278 // }
279
280 // while (counter.bytes_read % 4 != 0) {
281 // if (try counting_reader.takeByte() != 0)
282 // return error.CorruptInput;
283 // }
284
285 // const hash_a = hasher.hasher.final();
286 // const hash_b = try counting_reader.takeInt(u32, .little);
287 // if (hash_a != hash_b)
288 // return error.WrongChecksum;
289
290 // break :blk counter.bytes_read;
291 //};
292
293 //const hash_a = try d.in_reader.takeInt(u32, .little);
294277
295 //const hash_b = blk: {278 const record_count = try countLeb128(input, u64, &input_counter, &checksum);
296 // var hasher = hashedReader(d.in_reader, Crc32.init());279 if (record_count != d.block_count)
297 // const hashed_reader = hasher.reader();280 return error.CorruptInput;
298281
299 // const backward_size = (@as(u64, try hashed_reader.takeInt(u32, .little)) + 1) * 4;282 for (0..record_count) |_| {
300 // if (backward_size != index_size)283 // TODO: validate records
301 // return error.CorruptInput;284 _ = try countLeb128(input, u64, &input_counter, &checksum);
285 _ = try countLeb128(input, u64, &input_counter, &checksum);
286 }
302287
303 // var check: Check = undefined;288 const padding_len = (4 - (input_counter % 4)) % 4;
304 // try readStreamFlags(hashed_reader, &check);289 const padding = try input.take(padding_len);
290 for (padding) |byte| {
291 if (byte != 0) return error.CorruptInput;
292 }
293 checksum.update(padding);
305294
306 // break :blk hasher.hasher.final();295 const declared_checksum = try input.takeInt(u32, .little);
307 //};296 const computed_checksum = checksum.final();
297 if (computed_checksum != declared_checksum) return error.WrongChecksum;
308298
309 //if (hash_a != hash_b)299 break :blk input_counter + padding.len + 4;
310 // return error.WrongChecksum;300 };
311301
312 //const magic = try d.in_reader.takeBytesNoEof(2);302 const declared_checksum = try input.takeInt(u32, .little);
313 //if (!std.mem.eql(u8, &magic, &.{ 'Y', 'Z' }))303 const computed_checksum = Crc32.hash(try input.peek(4 + @sizeOf(StreamFlags)));
314 // return error.CorruptInput;304 if (declared_checksum != computed_checksum) return error.WrongChecksum;
305 const backward_size = (@as(u64, try input.takeInt(u32, .little)) + 1) * 4;
306 if (backward_size != index_size) return error.CorruptInput;
307 input.toss(@sizeOf(StreamFlags));
308 if (!std.mem.eql(u8, try input.takeArray(2), &.{ 'Y', 'Z' }))
309 return error.CorruptInput;
310}
315311
316 //return 0;312fn countLeb128(reader: *Reader, comptime T: type, counter: *u64, hasher: *Crc32) !T {
313 try reader.fill(8);
314 const start = reader.seek;
315 const result = try reader.takeLeb128(T);
316 const read_slice = reader.buffer[start..reader.seek];
317 hasher.update(read_slice);
318 counter.* += read_slice.len;
319 return result;
317}320}
lib/std/compress/xz/test.zig+69-37
...@@ -22,47 +22,79 @@ fn testReader(data: []const u8, comptime expected: []const u8) !void {...@@ -22,47 +22,79 @@ fn testReader(data: []const u8, comptime expected: []const u8) !void {
22 try testing.expectEqualSlices(u8, expected, result);22 try testing.expectEqualSlices(u8, expected, result);
23}23}
2424
25test "compressed data" {25test "fixture good-0-empty.xz" {
26 try testReader(@embedFile("testdata/good-0-empty.xz"), "");26 try testReader(@embedFile("testdata/good-0-empty.xz"), "");
27}
2728
28 inline for ([_][]const u8{29const hello_world_text =
29 "good-1-check-none.xz",30 \\Hello
30 "good-1-check-crc32.xz",31 \\World!
31 "good-1-check-crc64.xz",32 \\
32 "good-1-check-sha256.xz",33;
33 "good-2-lzma2.xz",
34 "good-1-block_header-1.xz",
35 "good-1-block_header-2.xz",
36 "good-1-block_header-3.xz",
37 }) |filename| {
38 try testReader(@embedFile("testdata/" ++ filename),
39 \\Hello
40 \\World!
41 \\
42 );
43 }
4434
45 inline for ([_][]const u8{35test "fixture good-1-check-none.xz" {
46 "good-1-lzma2-1.xz",36 try testReader(@embedFile("testdata/good-1-check-none.xz"), hello_world_text);
47 "good-1-lzma2-2.xz",37}
48 "good-1-lzma2-3.xz",38
49 "good-1-lzma2-4.xz",39test "fixture good-1-check-crc32.xz" {
50 }) |filename| {40 try testReader(@embedFile("testdata/good-1-check-crc32.xz"), hello_world_text);
51 try testReader(@embedFile("testdata/" ++ filename),41}
52 \\Lorem ipsum dolor sit amet, consectetur adipisicing 42
53 \\elit, sed do eiusmod tempor incididunt ut 43test "fixture good-1-check-crc64.xz" {
54 \\labore et dolore magna aliqua. Ut enim 44 try testReader(@embedFile("testdata/good-1-check-crc64.xz"), hello_world_text);
55 \\ad minim veniam, quis nostrud exercitation ullamco 45}
56 \\laboris nisi ut aliquip ex ea commodo 46
57 \\consequat. Duis aute irure dolor in reprehenderit 47test "fixture good-1-check-sha256.xz" {
58 \\in voluptate velit esse cillum dolore eu 48 try testReader(@embedFile("testdata/good-1-check-sha256.xz"), hello_world_text);
59 \\fugiat nulla pariatur. Excepteur sint occaecat cupidatat 49}
60 \\non proident, sunt in culpa qui officia 50
61 \\deserunt mollit anim id est laborum. 51test "fixture good-2-lzma2.xz" {
62 \\52 try testReader(@embedFile("testdata/good-2-lzma2.xz"), hello_world_text);
63 );53}
64 }54
55test "fixture good-1-block_header-1.xz" {
56 try testReader(@embedFile("testdata/good-1-block_header-1.xz"), hello_world_text);
57}
58
59test "fixture good-1-block_header-2.xz" {
60 try testReader(@embedFile("testdata/good-1-block_header-2.xz"), hello_world_text);
61}
62
63test "fixture good-1-block_header-3.xz" {
64 try testReader(@embedFile("testdata/good-1-block_header-3.xz"), hello_world_text);
65}
66
67const lorem_ipsum_text =
68 \\Lorem ipsum dolor sit amet, consectetur adipisicing
69 \\elit, sed do eiusmod tempor incididunt ut
70 \\labore et dolore magna aliqua. Ut enim
71 \\ad minim veniam, quis nostrud exercitation ullamco
72 \\laboris nisi ut aliquip ex ea commodo
73 \\consequat. Duis aute irure dolor in reprehenderit
74 \\in voluptate velit esse cillum dolore eu
75 \\fugiat nulla pariatur. Excepteur sint occaecat cupidatat
76 \\non proident, sunt in culpa qui officia
77 \\deserunt mollit anim id est laborum.
78 \\
79;
80
81test "fixture good-1-lzma2-1.xz" {
82 try testReader(@embedFile("testdata/good-1-lzma2-1.xz"), lorem_ipsum_text);
83}
84
85test "fixture good-1-lzma2-2.xz" {
86 try testReader(@embedFile("testdata/good-1-lzma2-2.xz"), lorem_ipsum_text);
87}
88
89test "fixture good-1-lzma2-3.xz" {
90 try testReader(@embedFile("testdata/good-1-lzma2-3.xz"), lorem_ipsum_text);
91}
92
93test "fixture good-1-lzma2-4.xz" {
94 try testReader(@embedFile("testdata/good-1-lzma2-4.xz"), lorem_ipsum_text);
95}
6596
97test "fixture good-1-lzma2-5.xz" {
66 try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");98 try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");
67}99}
68100