authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-26 20:33:45-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-26 21:00:58-07:00
log722e066173f5235b1dd1b341acb5cb1e79002ed7
treed35a48a883217774d239199995d8282c10a8bc2b
parentd87eb7d4e4f2ea606a18640fcc019b60cc435cdd

std.compress.xz.Decompress: some tests passing

4 files changed, 224 insertions(+), 164 deletions(-)

lib/std/compress/lzma.zig+58-41
......@@ -12,11 +12,19 @@ pub const RangeDecoder = struct {
1212 code: u32,
1313
1414 pub fn init(reader: *Reader) !RangeDecoder {
15 var counter: u64 = 0;
16 return initCounting(reader, &counter);
17 }
18
19 pub fn initCounting(reader: *Reader, n_read: *u64) !RangeDecoder {
1520 const reserved = try reader.takeByte();
21 n_read.* += 1;
1622 if (reserved != 0) return error.InvalidRangeCode;
23 const code = try reader.takeInt(u32, .big);
24 n_read.* += 4;
1725 return .{
1826 .range = 0xFFFF_FFFF,
19 .code = try reader.takeInt(u32, .big),
27 .code = code,
2028 };
2129 }
2230
......@@ -24,47 +32,47 @@ pub const RangeDecoder = struct {
2432 return self.code == 0;
2533 }
2634
27 fn normalize(self: *RangeDecoder, reader: *Reader) !void {
35 fn normalize(self: *RangeDecoder, reader: *Reader, n_read: *u64) !void {
2836 if (self.range < 0x0100_0000) {
2937 self.range <<= 8;
3038 self.code = (self.code << 8) ^ @as(u32, try reader.takeByte());
39 n_read.* += 1;
3140 }
3241 }
3342
34 fn getBit(self: *RangeDecoder, reader: *Reader) !bool {
43 fn getBit(self: *RangeDecoder, reader: *Reader, n_read: *u64) !bool {
3544 self.range >>= 1;
3645
3746 const bit = self.code >= self.range;
38 if (bit)
39 self.code -= self.range;
47 if (bit) self.code -= self.range;
4048
41 try self.normalize(reader);
49 try self.normalize(reader, n_read);
4250 return bit;
4351 }
4452
45 pub fn get(self: *RangeDecoder, reader: *Reader, count: usize) !u32 {
53 pub fn get(self: *RangeDecoder, reader: *Reader, count: usize, n_read: *u64) !u32 {
4654 var result: u32 = 0;
47 var i: usize = 0;
48 while (i < count) : (i += 1)
49 result = (result << 1) ^ @intFromBool(try self.getBit(reader));
55 for (0..count) |_| {
56 result = (result << 1) ^ @intFromBool(try self.getBit(reader, n_read));
57 }
5058 return result;
5159 }
5260
53 pub fn decodeBit(self: *RangeDecoder, reader: *Reader, prob: *u16) !bool {
61 pub fn decodeBit(self: *RangeDecoder, reader: *Reader, prob: *u16, n_read: *u64) !bool {
5462 const bound = (self.range >> 11) * prob.*;
5563
5664 if (self.code < bound) {
5765 prob.* += (0x800 - prob.*) >> 5;
5866 self.range = bound;
5967
60 try self.normalize(reader);
68 try self.normalize(reader, n_read);
6169 return false;
6270 } else {
6371 prob.* -= prob.* >> 5;
6472 self.code -= bound;
6573 self.range -= bound;
6674
67 try self.normalize(reader);
75 try self.normalize(reader, n_read);
6876 return true;
6977 }
7078 }
......@@ -74,11 +82,12 @@ pub const RangeDecoder = struct {
7482 reader: *Reader,
7583 num_bits: u5,
7684 probs: []u16,
85 n_read: *u64,
7786 ) !u32 {
7887 var tmp: u32 = 1;
7988 var i: @TypeOf(num_bits) = 0;
8089 while (i < num_bits) : (i += 1) {
81 const bit = try self.decodeBit(reader, &probs[tmp]);
90 const bit = try self.decodeBit(reader, &probs[tmp], n_read);
8291 tmp = (tmp << 1) ^ @intFromBool(bit);
8392 }
8493 return tmp - (@as(u32, 1) << num_bits);
......@@ -90,12 +99,13 @@ pub const RangeDecoder = struct {
9099 num_bits: u5,
91100 probs: []u16,
92101 offset: usize,
102 n_read: *u64,
93103 ) !u32 {
94104 var result: u32 = 0;
95105 var tmp: usize = 1;
96106 var i: @TypeOf(num_bits) = 0;
97107 while (i < num_bits) : (i += 1) {
98 const bit = @intFromBool(try self.decodeBit(reader, &probs[offset + tmp]));
108 const bit = @intFromBool(try self.decodeBit(reader, &probs[offset + tmp], n_read));
99109 tmp = (tmp << 1) ^ bit;
100110 result ^= @as(u32, bit) << i;
101111 }
......@@ -177,13 +187,14 @@ pub const Decode = struct {
177187 /// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.
178188 buffer: anytype,
179189 decoder: *RangeDecoder,
190 n_read: *u64,
180191 ) !ProcessingStatus {
181192 const gpa = allocating.allocator;
182193 const writer = &allocating.writer;
183194 const pos_state = buffer.len & ((@as(usize, 1) << self.properties.pb) - 1);
184195
185 if (!try decoder.decodeBit(reader, &self.is_match[(self.state << 4) + pos_state])) {
186 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder);
196 if (!try decoder.decodeBit(reader, &self.is_match[(self.state << 4) + pos_state], n_read)) {
197 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, n_read);
187198
188199 try buffer.appendLiteral(gpa, byte, writer);
189200
......@@ -197,18 +208,18 @@ pub const Decode = struct {
197208 }
198209
199210 var len: usize = undefined;
200 if (try decoder.decodeBit(reader, &self.is_rep[self.state])) {
201 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state])) {
202 if (!try decoder.decodeBit(reader, &self.is_rep_0long[(self.state << 4) + pos_state])) {
211 if (try decoder.decodeBit(reader, &self.is_rep[self.state], n_read)) {
212 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], n_read)) {
213 if (!try decoder.decodeBit(reader, &self.is_rep_0long[(self.state << 4) + pos_state], n_read)) {
203214 self.state = if (self.state < 7) 9 else 11;
204215 const dist = self.rep[0] + 1;
205216 try buffer.appendLz(gpa, 1, dist, writer);
206217 return .more;
207218 }
208219 } else {
209 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state]))
220 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], n_read))
210221 1
211 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state]))
222 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], n_read))
212223 2
213224 else
214225 3;
......@@ -220,7 +231,7 @@ pub const Decode = struct {
220231 self.rep[0] = dist;
221232 }
222233
223 len = try self.rep_len_decoder.decode(reader, decoder, pos_state);
234 len = try self.rep_len_decoder.decode(reader, decoder, pos_state, n_read);
224235
225236 self.state = if (self.state < 7) 8 else 11;
226237 } else {
......@@ -228,11 +239,11 @@ pub const Decode = struct {
228239 self.rep[2] = self.rep[1];
229240 self.rep[1] = self.rep[0];
230241
231 len = try self.len_decoder.decode(reader, decoder, pos_state);
242 len = try self.len_decoder.decode(reader, decoder, pos_state, n_read);
232243
233244 self.state = if (self.state < 7) 7 else 10;
234245
235 const rep_0 = try self.decodeDistance(reader, decoder, len);
246 const rep_0 = try self.decodeDistance(reader, decoder, len, n_read);
236247
237248 self.rep[0] = rep_0;
238249 if (self.rep[0] == 0xFFFF_FFFF) {
......@@ -257,6 +268,7 @@ pub const Decode = struct {
257268 /// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.
258269 buffer: anytype,
259270 decoder: *RangeDecoder,
271 n_read: *u64,
260272 ) !u8 {
261273 const def_prev_byte = 0;
262274 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
......@@ -275,6 +287,7 @@ pub const Decode = struct {
275287 const bit = @intFromBool(try decoder.decodeBit(
276288 reader,
277289 &probs[((@as(usize, 1) + match_bit) << 8) + result],
290 n_read,
278291 ));
279292 result = (result << 1) ^ bit;
280293 if (match_bit != bit) {
......@@ -284,10 +297,10 @@ pub const Decode = struct {
284297 }
285298
286299 while (result < 0x100) {
287 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result]));
300 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result], n_read));
288301 }
289302
290 return @as(u8, @truncate(result - 0x100));
303 return @truncate(result - 0x100);
291304 }
292305
293306 fn decodeDistance(
......@@ -295,12 +308,12 @@ pub const Decode = struct {
295308 reader: *Reader,
296309 decoder: *RangeDecoder,
297310 length: usize,
311 n_read: *u64,
298312 ) !usize {
299313 const len_state = if (length > 3) 3 else length;
300314
301 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder));
302 if (pos_slot < 4)
303 return pos_slot;
315 const pos_slot: usize = try self.pos_slot_decoder[len_state].parse(reader, decoder, n_read);
316 if (pos_slot < 4) return pos_slot;
304317
305318 const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
306319 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
......@@ -311,10 +324,11 @@ pub const Decode = struct {
311324 num_direct_bits,
312325 &self.pos_decoders,
313326 result - pos_slot,
327 n_read,
314328 );
315329 } else {
316 result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4;
317 result += try self.align_decoder.parseReverse(reader, decoder);
330 result += @as(usize, try decoder.get(reader, num_direct_bits - 4, n_read)) << 4;
331 result += try self.align_decoder.parseReverse(reader, decoder, n_read);
318332 }
319333
320334 return result;
......@@ -435,16 +449,17 @@ pub const Decode = struct {
435449 return struct {
436450 probs: [1 << num_bits]u16 = @splat(0x400),
437451
438 pub fn parse(self: *@This(), reader: *Reader, decoder: *RangeDecoder) !u32 {
439 return decoder.parseBitTree(reader, num_bits, &self.probs);
452 pub fn parse(self: *@This(), reader: *Reader, decoder: *RangeDecoder, n_read: *u64) !u32 {
453 return decoder.parseBitTree(reader, num_bits, &self.probs, n_read);
440454 }
441455
442456 pub fn parseReverse(
443457 self: *@This(),
444458 reader: *Reader,
445459 decoder: *RangeDecoder,
460 n_read: *u64,
446461 ) !u32 {
447 return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0);
462 return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, n_read);
448463 }
449464
450465 pub fn reset(self: *@This()) void {
......@@ -465,13 +480,14 @@ pub const Decode = struct {
465480 reader: *Reader,
466481 decoder: *RangeDecoder,
467482 pos_state: usize,
483 n_read: *u64,
468484 ) !usize {
469 if (!try decoder.decodeBit(reader, &self.choice)) {
470 return @as(usize, try self.low_coder[pos_state].parse(reader, decoder));
471 } else if (!try decoder.decodeBit(reader, &self.choice2)) {
472 return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder)) + 8;
485 if (!try decoder.decodeBit(reader, &self.choice, n_read)) {
486 return @as(usize, try self.low_coder[pos_state].parse(reader, decoder, n_read));
487 } else if (!try decoder.decodeBit(reader, &self.choice2, n_read)) {
488 return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder, n_read)) + 8;
473489 } else {
474 return @as(usize, try self.high_coder.parse(reader, decoder)) + 16;
490 return @as(usize, try self.high_coder.parse(reader, decoder, n_read)) + 16;
475491 }
476492 }
477493
......@@ -701,7 +717,8 @@ pub const Decompress = struct {
701717 } else if (d.range_decoder.isFinished()) {
702718 break :process_next;
703719 }
704 switch (d.decode.process(d.input, &allocating, &d.buffer, &d.range_decoder) catch |err| switch (err) {
720 var n_read: u64 = 0;
721 switch (d.decode.process(d.input, &allocating, &d.buffer, &d.range_decoder, &n_read) catch |err| switch (err) {
705722 error.WriteFailed => {
706723 d.err = error.OutOfMemory;
707724 return error.ReadFailed;
lib/std/compress/lzma2.zig+25-17
......@@ -116,24 +116,29 @@ pub const Decode = struct {
116116 self.* = undefined;
117117 }
118118
119 pub fn decompress(d: *Decode, reader: *Reader, allocating: *Writer.Allocating) !void {
119 /// Returns how many compressed bytes were consumed.
120 pub fn decompress(d: *Decode, reader: *Reader, allocating: *Writer.Allocating) !u64 {
120121 const gpa = allocating.allocator;
121122
122123 var accum = AccumBuffer.init(std.math.maxInt(usize));
123124 defer accum.deinit(gpa);
124125
126 var n_read: u64 = 0;
127
125128 while (true) {
126129 const status = try reader.takeByte();
130 n_read += 1;
127131
128132 switch (status) {
129133 0 => break,
130 1 => try parseUncompressed(reader, allocating, &accum, true),
131 2 => try parseUncompressed(reader, allocating, &accum, false),
132 else => try d.parseLzma(reader, allocating, &accum, status),
134 1 => n_read += try parseUncompressed(reader, allocating, &accum, true),
135 2 => n_read += try parseUncompressed(reader, allocating, &accum, false),
136 else => n_read += try d.parseLzma(reader, allocating, &accum, status),
133137 }
134138 }
135139
136140 try accum.finish(&allocating.writer);
141 return n_read;
137142 }
138143
139144 fn parseLzma(
......@@ -142,7 +147,7 @@ pub const Decode = struct {
142147 allocating: *Writer.Allocating,
143148 accum: *AccumBuffer,
144149 status: u8,
145 ) !void {
150 ) !u64 {
146151 if (status & 0x80 == 0) return error.CorruptInput;
147152
148153 const Reset = struct {
......@@ -175,15 +180,19 @@ pub const Decode = struct {
175180 else => unreachable,
176181 };
177182
183 var n_read: u64 = 0;
184
178185 const unpacked_size = blk: {
179186 var tmp: u64 = status & 0x1F;
180187 tmp <<= 16;
181188 tmp |= try reader.takeInt(u16, .big);
189 n_read += 2;
182190 break :blk tmp + 1;
183191 };
184192
185193 const packed_size = blk: {
186194 const tmp: u17 = try reader.takeInt(u16, .big);
195 n_read += 2;
187196 break :blk tmp + 1;
188197 };
189198
......@@ -196,6 +205,7 @@ pub const Decode = struct {
196205
197206 if (reset.props) {
198207 var props = try reader.takeByte();
208 n_read += 1;
199209 if (props >= 225) {
200210 return error.CorruptInput;
201211 }
......@@ -216,23 +226,21 @@ pub const Decode = struct {
216226 try ld.resetState(allocating.allocator, new_props);
217227 }
218228
219 var range_decoder = try lzma.RangeDecoder.init(reader);
229 const start_count = n_read;
230 var range_decoder = try lzma.RangeDecoder.initCounting(reader, &n_read);
220231
221232 while (true) {
222233 if (accum.len >= unpacked_size) break;
223234 if (range_decoder.isFinished()) break;
224 switch (try ld.process(reader, allocating, accum, &range_decoder)) {
235 switch (try ld.process(reader, allocating, accum, &range_decoder, &n_read)) {
225236 .more => continue,
226237 .finished => break,
227238 }
228239 }
229240 if (accum.len != unpacked_size) return error.DecompressedSizeMismatch;
241 if (n_read - start_count != packed_size) return error.CompressedSizeMismatch;
230242
231 // TODO restore this error
232 //if (counter.bytes_read != packed_size) {
233 // return error.CorruptInput;
234 //}
235 _ = packed_size;
243 return n_read;
236244 }
237245
238246 fn parseUncompressed(
......@@ -240,18 +248,17 @@ pub const Decode = struct {
240248 allocating: *Writer.Allocating,
241249 accum: *AccumBuffer,
242250 reset_dict: bool,
243 ) !void {
251 ) !usize {
244252 const unpacked_size = @as(u17, try reader.takeInt(u16, .big)) + 1;
245253
246254 if (reset_dict) try accum.reset(&allocating.writer);
247255
248256 const gpa = allocating.allocator;
249257
250 var i = unpacked_size;
251 while (i != 0) {
258 for (0..unpacked_size) |_| {
252259 try accum.appendByte(gpa, try reader.takeByte());
253 i -= 1;
254260 }
261 return 2 + unpacked_size;
255262 }
256263};
257264
......@@ -268,6 +275,7 @@ test "decompress hello world stream" {
268275 var result: std.Io.Writer.Allocating = .init(gpa);
269276 defer result.deinit();
270277
271 try decode.decompress(&stream, &result);
278 const n_read = try decode.decompress(&stream, &result);
279 try std.testing.expectEqual(compressed.len, n_read);
272280 try std.testing.expectEqualStrings(expected, result.written());
273281}
lib/std/compress/xz/Decompress.zig+72-69
......@@ -8,6 +8,7 @@ const Sha256 = std.crypto.hash.sha2.Sha256;
88const lzma2 = std.compress.lzma2;
99const Writer = std.Io.Writer;
1010const Reader = std.Io.Reader;
11const assert = std.debug.assert;
1112
1213/// Underlying compressed data stream to pull bytes from.
1314input: *Reader,
......@@ -28,6 +29,7 @@ pub const Error = error{
2829 Overflow,
2930 InvalidRangeCode,
3031 DecompressedSizeMismatch,
32 CompressedSizeMismatch,
3133};
3234
3335pub const Check = enum(u4) {
......@@ -62,10 +64,10 @@ pub fn init(
6264 if (!std.mem.eql(u8, magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
6365 return error.NotXzStream;
6466
65 const actual_hash = Crc32.hash(try input.peek(@sizeOf(StreamFlags)));
67 const computed_checksum = Crc32.hash(try input.peek(@sizeOf(StreamFlags)));
6668 const stream_flags = input.takeStruct(StreamFlags, .little) catch unreachable;
6769 const stored_hash = try input.takeInt(u32, .little);
68 if (actual_hash != stored_hash) return error.WrongChecksum;
70 if (computed_checksum != stored_hash) return error.WrongChecksum;
6971
7072 return .{
7173 .input = input,
......@@ -129,6 +131,7 @@ fn readIndirect(r: *Reader) Reader.Error!usize {
129131 r.end = allocating.writer.end;
130132 }
131133
134 if (d.err != null) return error.ReadFailed;
132135 if (d.block_count == std.math.maxInt(usize)) return error.EndOfStream;
133136
134137 readBlock(input, &allocating) catch |err| switch (err) {
......@@ -137,7 +140,10 @@ fn readIndirect(r: *Reader) Reader.Error!usize {
137140 return error.ReadFailed;
138141 },
139142 error.SuccessfulEndOfStream => {
140 finish(d);
143 finish(d) catch |finish_err| {
144 d.err = finish_err;
145 return error.ReadFailed;
146 };
141147 d.block_count = std.math.maxInt(usize);
142148 return error.EndOfStream;
143149 },
......@@ -184,7 +190,7 @@ fn readBlock(input: *Reader, allocating: *Writer.Allocating) !void {
184190 var packed_size: ?u64 = null;
185191 var unpacked_size: ?u64 = null;
186192
187 {
193 const header_size = h: {
188194 // Read the block header via peeking so that we can hash the whole thing too.
189195 const first_byte: usize = try input.peekByte();
190196 if (first_byte == 0) return error.SuccessfulEndOfStream;
......@@ -223,95 +229,92 @@ fn readBlock(input: *Reader, allocating: *Writer.Allocating) !void {
223229
224230 const actual_header_size = input.seek - header_seek_start;
225231 if (actual_header_size > declared_header_size) return error.CorruptInput;
226 var remaining_bytes = declared_header_size - actual_header_size;
227 while (remaining_bytes != 0) {
232 const remaining_bytes = declared_header_size - actual_header_size;
233 for (0..remaining_bytes) |_| {
228234 if (try input.takeByte() != 0) return error.CorruptInput;
229 remaining_bytes -= 1;
230235 }
231236
232237 const header_slice = input.buffer[header_seek_start..][0..declared_header_size];
233 const actual_hash = Crc32.hash(header_slice);
234 const declared_hash = try input.takeInt(u32, .little);
235 if (actual_hash != declared_hash) return error.WrongChecksum;
236 }
238 const computed_checksum = Crc32.hash(header_slice);
239 const declared_checksum = try input.takeInt(u32, .little);
240 if (computed_checksum != declared_checksum) return error.WrongChecksum;
241 break :h declared_header_size;
242 };
237243
238244 // Compressed Data
239245
240246 var lzma2_decode = try lzma2.Decode.init(allocating.allocator);
247 defer lzma2_decode.deinit(allocating.allocator);
241248 const before_size = allocating.writer.end;
242 try lzma2_decode.decompress(input, allocating);
249 const packed_bytes_read = try lzma2_decode.decompress(input, allocating);
243250 const unpacked_bytes = allocating.writer.end - before_size;
244251
245 // TODO restore this check
246 //if (packed_size) |s| {
247 // if (s != packed_counter.bytes_read)
248 // return error.CorruptInput;
249 //}
252 if (packed_size) |s| {
253 if (s != packed_bytes_read) return error.CorruptInput;
254 }
250255
251256 if (unpacked_size) |s| {
252257 if (s != unpacked_bytes) return error.CorruptInput;
253258 }
254259
255260 // Block Padding
256 if (true) @panic("TODO account for block padding");
257 //while (block_counter.bytes_read % 4 != 0) {
258 // if (try block_reader.takeByte() != 0)
259 // return error.CorruptInput;
260 //}
261
261 const block_counter = header_size + packed_bytes_read;
262 const padding = (4 - (block_counter % 4)) % 4;
263 for (0..padding) |_| {
264 if (try input.takeByte() != 0) return error.CorruptInput;
265 }
262266}
263267
264fn finish(d: *Decompress) void {
265 _ = d;
266 @panic("TODO");
267 //const input = d.input;
268 //const index_size = blk: {
269 // const record_count = try input.takeLeb128(u64);
270 // if (record_count != d.block_decode.block_count)
271 // return error.CorruptInput;
272
273 // var i: usize = 0;
274 // while (i < record_count) : (i += 1) {
275 // // TODO: validate records
276 // _ = try std.leb.readUleb128(u64, counting_reader);
277 // _ = try std.leb.readUleb128(u64, counting_reader);
278 // }
279
280 // while (counter.bytes_read % 4 != 0) {
281 // if (try counting_reader.takeByte() != 0)
282 // return error.CorruptInput;
283 // }
284
285 // const hash_a = hasher.hasher.final();
286 // const hash_b = try counting_reader.takeInt(u32, .little);
287 // if (hash_a != hash_b)
288 // return error.WrongChecksum;
289
290 // break :blk counter.bytes_read;
291 //};
292
293 //const hash_a = try d.in_reader.takeInt(u32, .little);
268fn finish(d: *Decompress) !void {
269 const input = d.input;
270 const index_size = blk: {
271 // Assume that we already peeked a zero in readBlock().
272 assert(input.buffered()[0] == 0);
273 var input_counter: u64 = 1;
274 var checksum: Crc32 = .init();
275 checksum.update(&.{0});
276 input.toss(1);
294277
295 //const hash_b = blk: {
296 // var hasher = hashedReader(d.in_reader, Crc32.init());
297 // const hashed_reader = hasher.reader();
278 const record_count = try countLeb128(input, u64, &input_counter, &checksum);
279 if (record_count != d.block_count)
280 return error.CorruptInput;
298281
299 // const backward_size = (@as(u64, try hashed_reader.takeInt(u32, .little)) + 1) * 4;
300 // if (backward_size != index_size)
301 // return error.CorruptInput;
282 for (0..record_count) |_| {
283 // TODO: validate records
284 _ = try countLeb128(input, u64, &input_counter, &checksum);
285 _ = try countLeb128(input, u64, &input_counter, &checksum);
286 }
302287
303 // var check: Check = undefined;
304 // try readStreamFlags(hashed_reader, &check);
288 const padding_len = (4 - (input_counter % 4)) % 4;
289 const padding = try input.take(padding_len);
290 for (padding) |byte| {
291 if (byte != 0) return error.CorruptInput;
292 }
293 checksum.update(padding);
305294
306 // break :blk hasher.hasher.final();
307 //};
295 const declared_checksum = try input.takeInt(u32, .little);
296 const computed_checksum = checksum.final();
297 if (computed_checksum != declared_checksum) return error.WrongChecksum;
308298
309 //if (hash_a != hash_b)
310 // return error.WrongChecksum;
299 break :blk input_counter + padding.len + 4;
300 };
311301
312 //const magic = try d.in_reader.takeBytesNoEof(2);
313 //if (!std.mem.eql(u8, &magic, &.{ 'Y', 'Z' }))
314 // return error.CorruptInput;
302 const declared_checksum = try input.takeInt(u32, .little);
303 const computed_checksum = Crc32.hash(try input.peek(4 + @sizeOf(StreamFlags)));
304 if (declared_checksum != computed_checksum) return error.WrongChecksum;
305 const backward_size = (@as(u64, try input.takeInt(u32, .little)) + 1) * 4;
306 if (backward_size != index_size) return error.CorruptInput;
307 input.toss(@sizeOf(StreamFlags));
308 if (!std.mem.eql(u8, try input.takeArray(2), &.{ 'Y', 'Z' }))
309 return error.CorruptInput;
310}
315311
316 //return 0;
312fn countLeb128(reader: *Reader, comptime T: type, counter: *u64, hasher: *Crc32) !T {
313 try reader.fill(8);
314 const start = reader.seek;
315 const result = try reader.takeLeb128(T);
316 const read_slice = reader.buffer[start..reader.seek];
317 hasher.update(read_slice);
318 counter.* += read_slice.len;
319 return result;
317320}
lib/std/compress/xz/test.zig+69-37
......@@ -22,47 +22,79 @@ fn testReader(data: []const u8, comptime expected: []const u8) !void {
2222 try testing.expectEqualSlices(u8, expected, result);
2323}
2424
25test "compressed data" {
25test "fixture good-0-empty.xz" {
2626 try testReader(@embedFile("testdata/good-0-empty.xz"), "");
27}
2728
28 inline for ([_][]const u8{
29 "good-1-check-none.xz",
30 "good-1-check-crc32.xz",
31 "good-1-check-crc64.xz",
32 "good-1-check-sha256.xz",
33 "good-2-lzma2.xz",
34 "good-1-block_header-1.xz",
35 "good-1-block_header-2.xz",
36 "good-1-block_header-3.xz",
37 }) |filename| {
38 try testReader(@embedFile("testdata/" ++ filename),
39 \\Hello
40 \\World!
41 \\
42 );
43 }
29const hello_world_text =
30 \\Hello
31 \\World!
32 \\
33;
4434
45 inline for ([_][]const u8{
46 "good-1-lzma2-1.xz",
47 "good-1-lzma2-2.xz",
48 "good-1-lzma2-3.xz",
49 "good-1-lzma2-4.xz",
50 }) |filename| {
51 try testReader(@embedFile("testdata/" ++ filename),
52 \\Lorem ipsum dolor sit amet, consectetur adipisicing
53 \\elit, sed do eiusmod tempor incididunt ut
54 \\labore et dolore magna aliqua. Ut enim
55 \\ad minim veniam, quis nostrud exercitation ullamco
56 \\laboris nisi ut aliquip ex ea commodo
57 \\consequat. Duis aute irure dolor in reprehenderit
58 \\in voluptate velit esse cillum dolore eu
59 \\fugiat nulla pariatur. Excepteur sint occaecat cupidatat
60 \\non proident, sunt in culpa qui officia
61 \\deserunt mollit anim id est laborum.
62 \\
63 );
64 }
35test "fixture good-1-check-none.xz" {
36 try testReader(@embedFile("testdata/good-1-check-none.xz"), hello_world_text);
37}
38
39test "fixture good-1-check-crc32.xz" {
40 try testReader(@embedFile("testdata/good-1-check-crc32.xz"), hello_world_text);
41}
42
43test "fixture good-1-check-crc64.xz" {
44 try testReader(@embedFile("testdata/good-1-check-crc64.xz"), hello_world_text);
45}
46
47test "fixture good-1-check-sha256.xz" {
48 try testReader(@embedFile("testdata/good-1-check-sha256.xz"), hello_world_text);
49}
50
51test "fixture good-2-lzma2.xz" {
52 try testReader(@embedFile("testdata/good-2-lzma2.xz"), hello_world_text);
53}
54
55test "fixture good-1-block_header-1.xz" {
56 try testReader(@embedFile("testdata/good-1-block_header-1.xz"), hello_world_text);
57}
58
59test "fixture good-1-block_header-2.xz" {
60 try testReader(@embedFile("testdata/good-1-block_header-2.xz"), hello_world_text);
61}
62
63test "fixture good-1-block_header-3.xz" {
64 try testReader(@embedFile("testdata/good-1-block_header-3.xz"), hello_world_text);
65}
66
67const lorem_ipsum_text =
68 \\Lorem ipsum dolor sit amet, consectetur adipisicing
69 \\elit, sed do eiusmod tempor incididunt ut
70 \\labore et dolore magna aliqua. Ut enim
71 \\ad minim veniam, quis nostrud exercitation ullamco
72 \\laboris nisi ut aliquip ex ea commodo
73 \\consequat. Duis aute irure dolor in reprehenderit
74 \\in voluptate velit esse cillum dolore eu
75 \\fugiat nulla pariatur. Excepteur sint occaecat cupidatat
76 \\non proident, sunt in culpa qui officia
77 \\deserunt mollit anim id est laborum.
78 \\
79;
80
81test "fixture good-1-lzma2-1.xz" {
82 try testReader(@embedFile("testdata/good-1-lzma2-1.xz"), lorem_ipsum_text);
83}
84
85test "fixture good-1-lzma2-2.xz" {
86 try testReader(@embedFile("testdata/good-1-lzma2-2.xz"), lorem_ipsum_text);
87}
88
89test "fixture good-1-lzma2-3.xz" {
90 try testReader(@embedFile("testdata/good-1-lzma2-3.xz"), lorem_ipsum_text);
91}
92
93test "fixture good-1-lzma2-4.xz" {
94 try testReader(@embedFile("testdata/good-1-lzma2-4.xz"), lorem_ipsum_text);
95}
6596
97test "fixture good-1-lzma2-5.xz" {
6698 try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");
6799}
68100