authorgravatar for fncontroloption@noreply.codeberg.orgFnControlOption <fncontroloption@noreply.codeberg.org> 2023-01-23 11:46:40-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-24 15:24:04-07:00
log06ce15e8f719756cc12d928cfdae12be99a9e4c2
tree117b722abbf4439392c53a5cf6ef055ec57f101d
parentdfcedfdca0b1e26d8a735327fe8e42d7887b21b2

Add an xz decoder to the standard library

32 files changed, 1230 insertions(+), 0 deletions(-)

build.zig+2
...@@ -122,6 +122,8 @@ pub fn build(b: *Builder) !void {...@@ -122,6 +122,8 @@ pub fn build(b: *Builder) !void {
122 "compress-gettysburg.txt",122 "compress-gettysburg.txt",
123 "compress-pi.txt",123 "compress-pi.txt",
124 "rfc1951.txt",124 "rfc1951.txt",
125 // exclude files from lib/std/compress/xz/testdata
126 ".xz",
125 // exclude files from lib/std/tz/127 // exclude files from lib/std/tz/
126 ".tzif",128 ".tzif",
127 // others129 // others
lib/std/compress/xz.zig created+5
...@@ -0,0 +1,5 @@
1pub usingnamespace @import("xz/stream.zig");
2
3test {
4 _ = @import("xz/stream.zig");
5}
lib/std/compress/xz/block.zig created+319
...@@ -0,0 +1,319 @@
1const std = @import("std");
2const check = @import("check.zig");
3const lzma = @import("lzma.zig");
4const multibyte = @import("multibyte.zig");
5const Allocator = std.mem.Allocator;
6const Crc32 = std.hash.Crc32;
7const Crc64 = std.hash.crc.Crc64Xz;
8const Sha256 = std.crypto.hash.sha2.Sha256;
9
10const DecodeError = error{
11 CorruptInput,
12 EndOfStream,
13 EndOfStreamWithNoError,
14 WrongChecksum,
15 Unsupported,
16 Overflow,
17};
18
19pub fn decoder(allocator: Allocator, reader: anytype, check_kind: check.Kind) !Decoder(@TypeOf(reader)) {
20 return Decoder(@TypeOf(reader)).init(allocator, reader, check_kind);
21}
22
23pub fn Decoder(comptime ReaderType: type) type {
24 return struct {
25 const Self = @This();
26 pub const Error =
27 ReaderType.Error ||
28 DecodeError ||
29 Allocator.Error;
30 pub const Reader = std.io.Reader(*Self, Error, read);
31
32 allocator: Allocator,
33 inner_reader: ReaderType,
34 check_kind: check.Kind,
35 err: ?Error,
36 accum: lzma.LzAccumBuffer,
37 lzma_state: lzma.DecoderState,
38 block_count: usize,
39
40 fn init(allocator: Allocator, in_reader: ReaderType, check_kind: check.Kind) !Self {
41 return Self{
42 .allocator = allocator,
43 .inner_reader = in_reader,
44 .check_kind = check_kind,
45 .err = null,
46 .accum = .{},
47 .lzma_state = try lzma.DecoderState.init(allocator),
48 .block_count = 0,
49 };
50 }
51
52 pub fn deinit(self: *Self) void {
53 self.accum.deinit(self.allocator);
54 self.lzma_state.deinit(self.allocator);
55 }
56
57 pub fn reader(self: *Self) Reader {
58 return .{ .context = self };
59 }
60
61 pub fn read(self: *Self, output: []u8) Error!usize {
62 while (true) {
63 if (self.accum.to_read.items.len > 0) {
64 const n = self.accum.read(output);
65 if (self.accum.to_read.items.len == 0 and self.err != null) {
66 if (self.err.? == DecodeError.EndOfStreamWithNoError) {
67 return n;
68 }
69 return self.err.?;
70 }
71 return n;
72 }
73 if (self.err != null) {
74 if (self.err.? == DecodeError.EndOfStreamWithNoError) {
75 return 0;
76 }
77 return self.err.?;
78 }
79 self.readBlock() catch |e| {
80 self.err = e;
81 if (self.accum.to_read.items.len == 0) {
82 try self.accum.reset(self.allocator);
83 }
84 };
85 }
86 }
87
88 fn readBlock(self: *Self) Error!void {
89 const unpacked_pos = self.accum.to_read.items.len;
90
91 var block_counter = std.io.countingReader(self.inner_reader);
92 const block_reader = block_counter.reader();
93
94 var packed_size: ?u64 = null;
95 var unpacked_size: ?u64 = null;
96
97 // Block Header
98 {
99 var header_hasher = std.compress.hashedReader(block_reader, Crc32.init());
100 const header_reader = header_hasher.reader();
101
102 const header_size = try header_reader.readByte() * 4;
103 if (header_size == 0)
104 return error.EndOfStreamWithNoError;
105
106 const Flags = packed struct(u8) {
107 last_filter_index: u2,
108 reserved: u4,
109 has_packed_size: bool,
110 has_unpacked_size: bool,
111 };
112
113 const flags = try header_reader.readStruct(Flags);
114 const filter_count = @as(u3, flags.last_filter_index) + 1;
115 if (filter_count > 1)
116 return error.Unsupported;
117
118 if (flags.has_packed_size)
119 packed_size = try multibyte.readInt(header_reader);
120
121 if (flags.has_unpacked_size)
122 unpacked_size = try multibyte.readInt(header_reader);
123
124 const FilterId = enum(u64) {
125 lzma2 = 0x21,
126 _,
127 };
128
129 const filter_id = @intToEnum(
130 FilterId,
131 try multibyte.readInt(header_reader),
132 );
133
134 if (@enumToInt(filter_id) >= 0x4000_0000_0000_0000)
135 return error.CorruptInput;
136
137 if (filter_id != .lzma2)
138 return error.Unsupported;
139
140 const properties_size = try multibyte.readInt(header_reader);
141 if (properties_size != 1)
142 return error.CorruptInput;
143
144 // TODO: use filter properties
145 _ = try header_reader.readByte();
146
147 while (block_counter.bytes_read != header_size) {
148 if (try header_reader.readByte() != 0)
149 return error.CorruptInput;
150 }
151
152 const hash_a = header_hasher.hasher.final();
153 const hash_b = try header_reader.readIntLittle(u32);
154 if (hash_a != hash_b)
155 return error.WrongChecksum;
156 }
157
158 // Compressed Data
159 var packed_counter = std.io.countingReader(block_reader);
160 const packed_reader = packed_counter.reader();
161 while (try self.readLzma2Chunk(packed_reader)) {}
162
163 if (packed_size) |s| {
164 if (s != packed_counter.bytes_read)
165 return error.CorruptInput;
166 }
167
168 const unpacked_bytes = self.accum.to_read.items[unpacked_pos..];
169 if (unpacked_size) |s| {
170 if (s != unpacked_bytes.len)
171 return error.CorruptInput;
172 }
173
174 // Block Padding
175 while (block_counter.bytes_read % 4 != 0) {
176 if (try block_reader.readByte() != 0)
177 return error.CorruptInput;
178 }
179
180 // Check
181 switch (self.check_kind) {
182 .none => {},
183 .crc32 => {
184 const hash_a = Crc32.hash(unpacked_bytes);
185 const hash_b = try self.inner_reader.readIntLittle(u32);
186 if (hash_a != hash_b)
187 return error.WrongChecksum;
188 },
189 .crc64 => {
190 const hash_a = Crc64.hash(unpacked_bytes);
191 const hash_b = try self.inner_reader.readIntLittle(u64);
192 if (hash_a != hash_b)
193 return error.WrongChecksum;
194 },
195 .sha256 => {
196 var hash_a: [Sha256.digest_length]u8 = undefined;
197 Sha256.hash(unpacked_bytes, &hash_a, .{});
198
199 var hash_b: [Sha256.digest_length]u8 = undefined;
200 try self.inner_reader.readNoEof(&hash_b);
201
202 if (!std.mem.eql(u8, &hash_a, &hash_b))
203 return error.WrongChecksum;
204 },
205 else => return error.Unsupported,
206 }
207
208 self.block_count += 1;
209 }
210
211 fn readLzma2Chunk(self: *Self, packed_reader: anytype) Error!bool {
212 const status = try packed_reader.readByte();
213 switch (status) {
214 0 => {
215 try self.accum.reset(self.allocator);
216 return false;
217 },
218 1, 2 => {
219 if (status == 1)
220 try self.accum.reset(self.allocator);
221
222 const size = try packed_reader.readIntBig(u16) + 1;
223 try self.accum.ensureUnusedCapacity(self.allocator, size);
224
225 var i: usize = 0;
226 while (i < size) : (i += 1)
227 self.accum.appendAssumeCapacity(try packed_reader.readByte());
228
229 return true;
230 },
231 else => {
232 if (status & 0x80 == 0)
233 return error.CorruptInput;
234
235 const Reset = struct {
236 dict: bool,
237 state: bool,
238 props: bool,
239 };
240
241 const reset = switch ((status >> 5) & 0x3) {
242 0 => Reset{
243 .dict = false,
244 .state = false,
245 .props = false,
246 },
247 1 => Reset{
248 .dict = false,
249 .state = true,
250 .props = false,
251 },
252 2 => Reset{
253 .dict = false,
254 .state = true,
255 .props = true,
256 },
257 3 => Reset{
258 .dict = true,
259 .state = true,
260 .props = true,
261 },
262 else => unreachable,
263 };
264
265 const unpacked_size = blk: {
266 var tmp: u64 = status & 0x1F;
267 tmp <<= 16;
268 tmp |= try packed_reader.readIntBig(u16);
269 break :blk tmp + 1;
270 };
271
272 const packed_size = blk: {
273 var tmp: u64 = try packed_reader.readIntBig(u16);
274 break :blk tmp + 1;
275 };
276
277 if (reset.dict)
278 try self.accum.reset(self.allocator);
279
280 if (reset.state) {
281 var new_props = self.lzma_state.lzma_props;
282
283 if (reset.props) {
284 var props = try packed_reader.readByte();
285 if (props >= 225)
286 return error.CorruptInput;
287
288 const lc = @intCast(u4, props % 9);
289 props /= 9;
290 const lp = @intCast(u3, props % 5);
291 props /= 5;
292 const pb = @intCast(u3, props);
293
294 if (lc + lp > 4)
295 return error.CorruptInput;
296
297 new_props = .{ .lc = lc, .lp = lp, .pb = pb };
298 }
299
300 try self.lzma_state.reset_state(self.allocator, new_props);
301 }
302
303 self.lzma_state.unpacked_size = unpacked_size + self.accum.len();
304
305 const buffer = try self.allocator.alloc(u8, packed_size);
306 defer self.allocator.free(buffer);
307
308 for (buffer) |*b|
309 b.* = try packed_reader.readByte();
310
311 var rangecoder = try lzma.RangeDecoder.init(buffer);
312 try self.lzma_state.process(self.allocator, &self.accum, &rangecoder);
313
314 return true;
315 },
316 }
317 }
318 };
319}
lib/std/compress/xz/check.zig created+7
...@@ -0,0 +1,7 @@
1pub const Kind = enum(u4) {
2 none = 0x00,
3 crc32 = 0x01,
4 crc64 = 0x04,
5 sha256 = 0x0A,
6 _,
7};
lib/std/compress/xz/lzma.zig created+658
...@@ -0,0 +1,658 @@
1// Ported from https://github.com/gendx/lzma-rs
2
3const std = @import("std");
4const assert = std.debug.assert;
5const Allocator = std.mem.Allocator;
6const ArrayListUnmanaged = std.ArrayListUnmanaged;
7
8const LzmaProperties = struct {
9 lc: u4,
10 lp: u3,
11 pb: u3,
12
13 fn validate(self: LzmaProperties) void {
14 assert(self.lc <= 8);
15 assert(self.lp <= 4);
16 assert(self.pb <= 4);
17 }
18};
19
20pub const DecoderState = struct {
21 lzma_props: LzmaProperties,
22 unpacked_size: ?u64,
23 literal_probs: Vec2D(u16),
24 pos_slot_decoder: [4]BitTree,
25 align_decoder: BitTree,
26 pos_decoders: [115]u16,
27 is_match: [192]u16,
28 is_rep: [12]u16,
29 is_rep_g0: [12]u16,
30 is_rep_g1: [12]u16,
31 is_rep_g2: [12]u16,
32 is_rep_0long: [192]u16,
33 state: usize,
34 rep: [4]usize,
35 len_decoder: LenDecoder,
36 rep_len_decoder: LenDecoder,
37
38 pub fn init(allocator: Allocator) !DecoderState {
39 return .{
40 .lzma_props = LzmaProperties{ .lc = 0, .lp = 0, .pb = 0 },
41 .unpacked_size = null,
42 .literal_probs = try Vec2D(u16).init(allocator, 0x400, 1, 0x300),
43 .pos_slot_decoder = .{
44 try BitTree.init(allocator, 6),
45 try BitTree.init(allocator, 6),
46 try BitTree.init(allocator, 6),
47 try BitTree.init(allocator, 6),
48 },
49 .align_decoder = try BitTree.init(allocator, 4),
50 .pos_decoders = .{0x400} ** 115,
51 .is_match = .{0x400} ** 192,
52 .is_rep = .{0x400} ** 12,
53 .is_rep_g0 = .{0x400} ** 12,
54 .is_rep_g1 = .{0x400} ** 12,
55 .is_rep_g2 = .{0x400} ** 12,
56 .is_rep_0long = .{0x400} ** 192,
57 .state = 0,
58 .rep = .{0} ** 4,
59 .len_decoder = try LenDecoder.init(allocator),
60 .rep_len_decoder = try LenDecoder.init(allocator),
61 };
62 }
63
64 pub fn deinit(self: *DecoderState, allocator: Allocator) void {
65 self.literal_probs.deinit(allocator);
66 for (self.pos_slot_decoder) |*t| t.deinit(allocator);
67 self.align_decoder.deinit(allocator);
68 self.len_decoder.deinit(allocator);
69 self.rep_len_decoder.deinit(allocator);
70 }
71
72 pub fn reset_state(self: *DecoderState, allocator: Allocator, new_props: LzmaProperties) !void {
73 new_props.validate();
74 if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
75 self.literal_probs.fill(0x400);
76 } else {
77 self.literal_probs.deinit(allocator);
78 self.literal_probs = try Vec2D(u16).init(allocator, 0x400, @as(usize, 1) << (new_props.lc + new_props.lp), 0x300);
79 }
80
81 self.lzma_props = new_props;
82 for (self.pos_slot_decoder) |*t| t.reset();
83 self.align_decoder.reset();
84 self.pos_decoders = .{0x400} ** 115;
85 self.is_match = .{0x400} ** 192;
86 self.is_rep = .{0x400} ** 12;
87 self.is_rep_g0 = .{0x400} ** 12;
88 self.is_rep_g1 = .{0x400} ** 12;
89 self.is_rep_g2 = .{0x400} ** 12;
90 self.is_rep_0long = .{0x400} ** 192;
91 self.state = 0;
92 self.rep = .{0} ** 4;
93 self.len_decoder.reset();
94 self.rep_len_decoder.reset();
95 }
96
97 fn processNextInner(
98 self: *DecoderState,
99 allocator: Allocator,
100 output: *LzAccumBuffer,
101 rangecoder: *RangeDecoder,
102 update: bool,
103 ) !ProcessingStatus {
104 const pos_state = output.len() & ((@as(usize, 1) << self.lzma_props.pb) - 1);
105
106 if (!try rangecoder.decodeBit(
107 &self.is_match[(self.state << 4) + pos_state],
108 update,
109 )) {
110 const byte: u8 = try self.decodeLiteral(output, rangecoder, update);
111
112 if (update) {
113 try output.appendLiteral(allocator, byte);
114
115 self.state = if (self.state < 4)
116 0
117 else if (self.state < 10)
118 self.state - 3
119 else
120 self.state - 6;
121 }
122 return .continue_;
123 }
124
125 var len: usize = undefined;
126 if (try rangecoder.decodeBit(&self.is_rep[self.state], update)) {
127 if (!try rangecoder.decodeBit(&self.is_rep_g0[self.state], update)) {
128 if (!try rangecoder.decodeBit(
129 &self.is_rep_0long[(self.state << 4) + pos_state],
130 update,
131 )) {
132 if (update) {
133 self.state = if (self.state < 7) 9 else 11;
134 const dist = self.rep[0] + 1;
135 try output.appendLz(allocator, 1, dist);
136 }
137 return .continue_;
138 }
139 } else {
140 const idx: usize = if (!try rangecoder.decodeBit(&self.is_rep_g1[self.state], update))
141 1
142 else if (!try rangecoder.decodeBit(&self.is_rep_g2[self.state], update))
143 2
144 else
145 3;
146 if (update) {
147 const dist = self.rep[idx];
148 var i = idx;
149 while (i > 0) : (i -= 1) {
150 self.rep[i] = self.rep[i - 1];
151 }
152 self.rep[0] = dist;
153 }
154 }
155
156 len = try self.rep_len_decoder.decode(rangecoder, pos_state, update);
157
158 if (update) {
159 self.state = if (self.state < 7) 8 else 11;
160 }
161 } else {
162 if (update) {
163 self.rep[3] = self.rep[2];
164 self.rep[2] = self.rep[1];
165 self.rep[1] = self.rep[0];
166 }
167
168 len = try self.len_decoder.decode(rangecoder, pos_state, update);
169
170 if (update) {
171 self.state = if (self.state < 7) 7 else 10;
172 }
173
174 const rep_0 = try self.decodeDistance(rangecoder, len, update);
175
176 if (update) {
177 self.rep[0] = rep_0;
178 if (self.rep[0] == 0xFFFF_FFFF) {
179 if (rangecoder.isFinished()) {
180 return .finished;
181 }
182 return error.CorruptInput;
183 }
184 }
185 }
186
187 if (update) {
188 len += 2;
189
190 const dist = self.rep[0] + 1;
191 try output.appendLz(allocator, len, dist);
192 }
193
194 return .continue_;
195 }
196
197 fn processNext(
198 self: *DecoderState,
199 allocator: Allocator,
200 output: *LzAccumBuffer,
201 rangecoder: *RangeDecoder,
202 ) !ProcessingStatus {
203 return self.processNextInner(allocator, output, rangecoder, true);
204 }
205
206 pub fn process(
207 self: *DecoderState,
208 allocator: Allocator,
209 output: *LzAccumBuffer,
210 rangecoder: *RangeDecoder,
211 ) !void {
212 while (true) {
213 if (self.unpacked_size) |unpacked_size| {
214 if (output.len() >= unpacked_size) {
215 break;
216 }
217 } else if (rangecoder.isFinished()) {
218 break;
219 }
220
221 if (try self.processNext(allocator, output, rangecoder) == .finished) {
222 break;
223 }
224 }
225
226 if (self.unpacked_size) |len| {
227 if (len != output.len()) {
228 return error.CorruptInput;
229 }
230 }
231 }
232
233 fn decodeLiteral(
234 self: *DecoderState,
235 output: *LzAccumBuffer,
236 rangecoder: *RangeDecoder,
237 update: bool,
238 ) !u8 {
239 const def_prev_byte = 0;
240 const prev_byte = @as(usize, output.lastOr(def_prev_byte));
241
242 var result: usize = 1;
243 const lit_state = ((output.len() & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
244 (prev_byte >> (8 - self.lzma_props.lc));
245 const probs = try self.literal_probs.get(lit_state);
246
247 if (self.state >= 7) {
248 var match_byte = @as(usize, try output.lastN(self.rep[0] + 1));
249
250 while (result < 0x100) {
251 const match_bit = (match_byte >> 7) & 1;
252 match_byte <<= 1;
253 const bit = @boolToInt(try rangecoder.decodeBit(
254 &probs[((@as(usize, 1) + match_bit) << 8) + result],
255 update,
256 ));
257 result = (result << 1) ^ bit;
258 if (match_bit != bit) {
259 break;
260 }
261 }
262 }
263
264 while (result < 0x100) {
265 result = (result << 1) ^ @boolToInt(try rangecoder.decodeBit(&probs[result], update));
266 }
267
268 return @truncate(u8, result - 0x100);
269 }
270
271 fn decodeDistance(
272 self: *DecoderState,
273 rangecoder: *RangeDecoder,
274 length: usize,
275 update: bool,
276 ) !usize {
277 const len_state = if (length > 3) 3 else length;
278
279 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(rangecoder, update));
280 if (pos_slot < 4)
281 return pos_slot;
282
283 const num_direct_bits = @intCast(u5, (pos_slot >> 1) - 1);
284 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
285
286 if (pos_slot < 14) {
287 result += try rangecoder.parseReverseBitTree(
288 num_direct_bits,
289 &self.pos_decoders,
290 result - pos_slot,
291 update,
292 );
293 } else {
294 result += @as(usize, try rangecoder.get(num_direct_bits - 4)) << 4;
295 result += try self.align_decoder.parseReverse(rangecoder, update);
296 }
297
298 return result;
299 }
300};
301
302const ProcessingStatus = enum {
303 continue_,
304 finished,
305};
306
307pub const LzAccumBuffer = struct {
308 to_read: ArrayListUnmanaged(u8) = .{},
309 buf: ArrayListUnmanaged(u8) = .{},
310
311 pub fn deinit(self: *LzAccumBuffer, allocator: Allocator) void {
312 self.to_read.deinit(allocator);
313 self.buf.deinit(allocator);
314 }
315
316 pub fn read(self: *LzAccumBuffer, output: []u8) usize {
317 const input = self.to_read.items;
318 const n = std.math.min(input.len, output.len);
319 std.mem.copy(u8, output[0..n], input[0..n]);
320 std.mem.copy(u8, input, input[n..]);
321 self.to_read.shrinkRetainingCapacity(input.len - n);
322 return n;
323 }
324
325 pub fn ensureUnusedCapacity(
326 self: *LzAccumBuffer,
327 allocator: Allocator,
328 additional_count: usize,
329 ) !void {
330 try self.buf.ensureUnusedCapacity(allocator, additional_count);
331 }
332
333 pub fn appendAssumeCapacity(self: *LzAccumBuffer, byte: u8) void {
334 self.buf.appendAssumeCapacity(byte);
335 }
336
337 pub fn reset(self: *LzAccumBuffer, allocator: Allocator) !void {
338 try self.to_read.appendSlice(allocator, self.buf.items);
339 self.buf.clearRetainingCapacity();
340 }
341
342 pub fn len(self: *const LzAccumBuffer) usize {
343 return self.buf.items.len;
344 }
345
346 pub fn lastOr(self: *const LzAccumBuffer, lit: u8) u8 {
347 const buf_len = self.buf.items.len;
348 return if (buf_len == 0)
349 lit
350 else
351 self.buf.items[buf_len - 1];
352 }
353
354 pub fn lastN(self: *const LzAccumBuffer, dist: usize) !u8 {
355 const buf_len = self.buf.items.len;
356 if (dist > buf_len) {
357 return error.CorruptInput;
358 }
359
360 return self.buf.items[buf_len - dist];
361 }
362
363 pub fn appendLiteral(self: *LzAccumBuffer, allocator: Allocator, lit: u8) !void {
364 try self.buf.append(allocator, lit);
365 }
366
367 pub fn appendLz(self: *LzAccumBuffer, allocator: Allocator, length: usize, dist: usize) !void {
368 const buf_len = self.buf.items.len;
369 if (dist > buf_len) {
370 return error.CorruptInput;
371 }
372
373 var offset = buf_len - dist;
374 var i: usize = 0;
375 while (i < length) : (i += 1) {
376 const x = self.buf.items[offset];
377 try self.buf.append(allocator, x);
378 offset += 1;
379 }
380 }
381};
382
383pub const RangeDecoder = struct {
384 stream: std.io.FixedBufferStream([]const u8),
385 range: u32,
386 code: u32,
387
388 pub fn init(buffer: []const u8) !RangeDecoder {
389 var dec = RangeDecoder{
390 .stream = std.io.fixedBufferStream(buffer),
391 .range = 0xFFFF_FFFF,
392 .code = 0,
393 };
394 const reader = dec.stream.reader();
395 _ = try reader.readByte();
396 dec.code = try reader.readIntBig(u32);
397 return dec;
398 }
399
400 pub fn fromParts(
401 buffer: []const u8,
402 range: u32,
403 code: u32,
404 ) RangeDecoder {
405 return .{
406 .stream = std.io.fixedBufferStream(buffer),
407 .range = range,
408 .code = code,
409 };
410 }
411
412 pub fn set(self: *RangeDecoder, range: u32, code: u32) void {
413 self.range = range;
414 self.code = code;
415 }
416
417 pub fn readInto(self: *RangeDecoder, dest: []u8) !usize {
418 return self.stream.read(dest);
419 }
420
421 pub inline fn isFinished(self: *const RangeDecoder) bool {
422 return self.code == 0 and self.isEof();
423 }
424
425 pub inline fn isEof(self: *const RangeDecoder) bool {
426 return self.stream.pos == self.stream.buffer.len;
427 }
428
429 inline fn normalize(self: *RangeDecoder) !void {
430 if (self.range < 0x0100_0000) {
431 self.range <<= 8;
432 self.code = (self.code << 8) ^ @as(u32, try self.stream.reader().readByte());
433 }
434 }
435
436 inline fn getBit(self: *RangeDecoder) !bool {
437 self.range >>= 1;
438
439 const bit = self.code >= self.range;
440 if (bit)
441 self.code -= self.range;
442
443 try self.normalize();
444 return bit;
445 }
446
447 fn get(self: *RangeDecoder, count: usize) !u32 {
448 var result: u32 = 0;
449 var i: usize = 0;
450 while (i < count) : (i += 1)
451 result = (result << 1) ^ @boolToInt(try self.getBit());
452 return result;
453 }
454
455 pub inline fn decodeBit(self: *RangeDecoder, prob: *u16, update: bool) !bool {
456 const bound = (self.range >> 11) * prob.*;
457
458 if (self.code < bound) {
459 if (update)
460 prob.* += (0x800 - prob.*) >> 5;
461 self.range = bound;
462
463 try self.normalize();
464 return false;
465 } else {
466 if (update)
467 prob.* -= prob.* >> 5;
468 self.code -= bound;
469 self.range -= bound;
470
471 try self.normalize();
472 return true;
473 }
474 }
475
476 fn parseBitTree(
477 self: *RangeDecoder,
478 num_bits: u5,
479 probs: []u16,
480 update: bool,
481 ) !u32 {
482 var tmp: u32 = 1;
483 var i: u5 = 0;
484 while (i < num_bits) : (i += 1) {
485 const bit = try self.decodeBit(&probs[tmp], update);
486 tmp = (tmp << 1) ^ @boolToInt(bit);
487 }
488 return tmp - (@as(u32, 1) << num_bits);
489 }
490
491 pub fn parseReverseBitTree(
492 self: *RangeDecoder,
493 num_bits: u5,
494 probs: []u16,
495 offset: usize,
496 update: bool,
497 ) !u32 {
498 var result: u32 = 0;
499 var tmp: usize = 1;
500 var i: u5 = 0;
501 while (i < num_bits) : (i += 1) {
502 const bit = @boolToInt(try self.decodeBit(&probs[offset + tmp], update));
503 tmp = (tmp << 1) ^ bit;
504 result ^= @as(u32, bit) << i;
505 }
506 return result;
507 }
508};
509
510fn Vec2D(comptime T: type) type {
511 return struct {
512 data: []T,
513 cols: usize,
514
515 const Self = @This();
516
517 pub fn init(allocator: Allocator, data: T, rows: usize, cols: usize) !Self {
518 const len = try std.math.mul(usize, rows, cols);
519 var vec2d = Self{
520 .data = try allocator.alloc(T, len),
521 .cols = cols,
522 };
523 vec2d.fill(data);
524 return vec2d;
525 }
526
527 pub fn deinit(self: *Self, allocator: Allocator) void {
528 allocator.free(self.data);
529 }
530
531 pub fn fill(self: *Self, value: T) void {
532 std.mem.set(T, self.data, value);
533 }
534
535 pub fn get(self: *Self, row: usize) ![]T {
536 const start_row = try std.math.mul(usize, row, self.cols);
537 return self.data[start_row .. start_row + self.cols];
538 }
539 };
540}
541
542const BitTree = struct {
543 num_bits: u5,
544 probs: ArrayListUnmanaged(u16),
545
546 pub fn init(allocator: Allocator, num_bits: u5) !BitTree {
547 var probs_len = @as(usize, 1) << num_bits;
548 var probs = try ArrayListUnmanaged(u16).initCapacity(allocator, probs_len);
549 while (probs_len > 0) : (probs_len -= 1)
550 probs.appendAssumeCapacity(0x400);
551 return .{ .num_bits = num_bits, .probs = probs };
552 }
553
554 pub fn deinit(self: *BitTree, allocator: Allocator) void {
555 self.probs.deinit(allocator);
556 }
557
558 pub fn parse(
559 self: *BitTree,
560 rangecoder: *RangeDecoder,
561 update: bool,
562 ) !u32 {
563 return rangecoder.parseBitTree(self.num_bits, self.probs.items, update);
564 }
565
566 pub fn parseReverse(
567 self: *BitTree,
568 rangecoder: *RangeDecoder,
569 update: bool,
570 ) !u32 {
571 return rangecoder.parseReverseBitTree(self.num_bits, self.probs.items, 0, update);
572 }
573
574 pub fn reset(self: *BitTree) void {
575 std.mem.set(u16, self.probs.items, 0x400);
576 }
577};
578
579const LenDecoder = struct {
580 choice: u16,
581 choice2: u16,
582 low_coder: [16]BitTree,
583 mid_coder: [16]BitTree,
584 high_coder: BitTree,
585
586 pub fn init(allocator: Allocator) !LenDecoder {
587 return .{
588 .choice = 0x400,
589 .choice2 = 0x400,
590 .low_coder = .{
591 try BitTree.init(allocator, 3),
592 try BitTree.init(allocator, 3),
593 try BitTree.init(allocator, 3),
594 try BitTree.init(allocator, 3),
595 try BitTree.init(allocator, 3),
596 try BitTree.init(allocator, 3),
597 try BitTree.init(allocator, 3),
598 try BitTree.init(allocator, 3),
599 try BitTree.init(allocator, 3),
600 try BitTree.init(allocator, 3),
601 try BitTree.init(allocator, 3),
602 try BitTree.init(allocator, 3),
603 try BitTree.init(allocator, 3),
604 try BitTree.init(allocator, 3),
605 try BitTree.init(allocator, 3),
606 try BitTree.init(allocator, 3),
607 },
608 .mid_coder = .{
609 try BitTree.init(allocator, 3),
610 try BitTree.init(allocator, 3),
611 try BitTree.init(allocator, 3),
612 try BitTree.init(allocator, 3),
613 try BitTree.init(allocator, 3),
614 try BitTree.init(allocator, 3),
615 try BitTree.init(allocator, 3),
616 try BitTree.init(allocator, 3),
617 try BitTree.init(allocator, 3),
618 try BitTree.init(allocator, 3),
619 try BitTree.init(allocator, 3),
620 try BitTree.init(allocator, 3),
621 try BitTree.init(allocator, 3),
622 try BitTree.init(allocator, 3),
623 try BitTree.init(allocator, 3),
624 try BitTree.init(allocator, 3),
625 },
626 .high_coder = try BitTree.init(allocator, 8),
627 };
628 }
629
630 pub fn deinit(self: *LenDecoder, allocator: Allocator) void {
631 for (self.low_coder) |*t| t.deinit(allocator);
632 for (self.mid_coder) |*t| t.deinit(allocator);
633 self.high_coder.deinit(allocator);
634 }
635
636 pub fn decode(
637 self: *LenDecoder,
638 rangecoder: *RangeDecoder,
639 pos_state: usize,
640 update: bool,
641 ) !usize {
642 if (!try rangecoder.decodeBit(&self.choice, update)) {
643 return @as(usize, try self.low_coder[pos_state].parse(rangecoder, update));
644 } else if (!try rangecoder.decodeBit(&self.choice2, update)) {
645 return @as(usize, try self.mid_coder[pos_state].parse(rangecoder, update)) + 8;
646 } else {
647 return @as(usize, try self.high_coder.parse(rangecoder, update)) + 16;
648 }
649 }
650
651 pub fn reset(self: *LenDecoder) void {
652 self.choice = 0x400;
653 self.choice2 = 0x400;
654 for (self.low_coder) |*t| t.reset();
655 for (self.mid_coder) |*t| t.reset();
656 self.high_coder.reset();
657 }
658};
lib/std/compress/xz/multibyte.zig created+23
...@@ -0,0 +1,23 @@
1const Multibyte = packed struct(u8) {
2 value: u7,
3 more: bool,
4};
5
6pub fn readInt(reader: anytype) !u64 {
7 const max_size = 9;
8
9 var chunk = try reader.readStruct(Multibyte);
10 var num: u64 = chunk.value;
11 var i: u6 = 0;
12
13 while (chunk.more) {
14 chunk = try reader.readStruct(Multibyte);
15 i += 1;
16 if (i >= max_size or @bitCast(u8, chunk) == 0x00)
17 return error.CorruptInput;
18
19 num |= @as(u64, chunk.value) << (i * 7);
20 }
21
22 return num;
23}
lib/std/compress/xz/stream.zig created+136
...@@ -0,0 +1,136 @@
1const std = @import("std");
2const block = @import("block.zig");
3const check = @import("check.zig");
4const multibyte = @import("multibyte.zig");
5const Allocator = std.mem.Allocator;
6const Crc32 = std.hash.Crc32;
7
8test {
9 _ = @import("stream_test.zig");
10}
11
12const Flags = packed struct(u16) {
13 reserved1: u8,
14 check_kind: check.Kind,
15 reserved2: u4,
16};
17
18pub fn stream(allocator: Allocator, reader: anytype) !Stream(@TypeOf(reader)) {
19 return Stream(@TypeOf(reader)).init(allocator, reader);
20}
21
22pub fn Stream(comptime ReaderType: type) type {
23 return struct {
24 const Self = @This();
25
26 pub const Error = ReaderType.Error || block.Decoder(ReaderType).Error;
27 pub const Reader = std.io.Reader(*Self, Error, read);
28
29 allocator: Allocator,
30 block_decoder: block.Decoder(ReaderType),
31 in_reader: ReaderType,
32
33 fn init(allocator: Allocator, source: ReaderType) !Self {
34 const Header = extern struct {
35 magic: [6]u8,
36 flags: Flags,
37 crc32: u32,
38 };
39
40 const header = try source.readStruct(Header);
41
42 if (!std.mem.eql(u8, &header.magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
43 return error.BadHeader;
44
45 if (header.flags.reserved1 != 0 or header.flags.reserved2 != 0)
46 return error.BadHeader;
47
48 const hash = Crc32.hash(std.mem.asBytes(&header.flags));
49 if (hash != header.crc32)
50 return error.WrongChecksum;
51
52 return Self{
53 .allocator = allocator,
54 .block_decoder = try block.decoder(allocator, source, header.flags.check_kind),
55 .in_reader = source,
56 };
57 }
58
59 pub fn deinit(self: *Self) void {
60 self.block_decoder.deinit();
61 }
62
63 pub fn reader(self: *Self) Reader {
64 return .{ .context = self };
65 }
66
67 pub fn read(self: *Self, buffer: []u8) Error!usize {
68 if (buffer.len == 0)
69 return 0;
70
71 const r = try self.block_decoder.read(buffer);
72 if (r != 0)
73 return r;
74
75 const index_size = blk: {
76 var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
77 hasher.hasher.update(&[1]u8{0x00});
78
79 var counter = std.io.countingReader(hasher.reader());
80 counter.bytes_read += 1;
81
82 const counting_reader = counter.reader();
83
84 const record_count = try multibyte.readInt(counting_reader);
85 if (record_count != self.block_decoder.block_count)
86 return error.CorruptInput;
87
88 var i: usize = 0;
89 while (i < record_count) : (i += 1) {
90 // TODO: validate records
91 _ = try multibyte.readInt(counting_reader);
92 _ = try multibyte.readInt(counting_reader);
93 }
94
95 while (counter.bytes_read % 4 != 0) {
96 if (try counting_reader.readByte() != 0)
97 return error.CorruptInput;
98 }
99
100 const hash_a = hasher.hasher.final();
101 const hash_b = try counting_reader.readIntLittle(u32);
102 if (hash_a != hash_b)
103 return error.WrongChecksum;
104
105 break :blk counter.bytes_read;
106 };
107
108 const Footer = extern struct {
109 crc32: u32,
110 backward_size: u32,
111 flags: Flags,
112 magic: [2]u8,
113 };
114
115 const footer = try self.in_reader.readStruct(Footer);
116 const backward_size = (footer.backward_size + 1) * 4;
117 if (backward_size != index_size)
118 return error.CorruptInput;
119
120 if (footer.flags.reserved1 != 0 or footer.flags.reserved2 != 0)
121 return error.CorruptInput;
122
123 var hasher = Crc32.init();
124 hasher.update(std.mem.asBytes(&footer.backward_size));
125 hasher.update(std.mem.asBytes(&footer.flags));
126 const hash = hasher.final();
127 if (hash != footer.crc32)
128 return error.WrongChecksum;
129
130 if (!std.mem.eql(u8, &footer.magic, &.{ 'Y', 'Z' }))
131 return error.CorruptInput;
132
133 return 0;
134 }
135 };
136}
lib/std/compress/xz/stream_test.zig created+80
...@@ -0,0 +1,80 @@
1const std = @import("std");
2const testing = std.testing;
3const stream = @import("stream.zig").stream;
4
5fn decompress(data: []const u8) ![]u8 {
6 var in_stream = std.io.fixedBufferStream(data);
7
8 var xz_stream = try stream(testing.allocator, in_stream.reader());
9 defer xz_stream.deinit();
10
11 return xz_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));
12}
13
14fn testReader(data: []const u8, comptime expected: []const u8) !void {
15 const buf = try decompress(data);
16 defer testing.allocator.free(buf);
17
18 try testing.expectEqualSlices(u8, expected, buf);
19}
20
21test "compressed data" {
22 try testReader(@embedFile("testdata/good-0-empty.xz"), "");
23
24 inline for ([_][]const u8{
25 "good-1-check-none.xz",
26 "good-1-check-crc32.xz",
27 "good-1-check-crc64.xz",
28 "good-1-check-sha256.xz",
29 "good-2-lzma2.xz",
30 "good-1-block_header-1.xz",
31 "good-1-block_header-2.xz",
32 "good-1-block_header-3.xz",
33 }) |filename| {
34 try testReader(@embedFile("testdata/" ++ filename),
35 \\Hello
36 \\World!
37 \\
38 );
39 }
40
41 inline for ([_][]const u8{
42 "good-1-lzma2-1.xz",
43 "good-1-lzma2-2.xz",
44 "good-1-lzma2-3.xz",
45 "good-1-lzma2-4.xz",
46 }) |filename| {
47 try testReader(@embedFile("testdata/" ++ filename),
48 \\Lorem ipsum dolor sit amet, consectetur adipisicing
49 \\elit, sed do eiusmod tempor incididunt ut
50 \\labore et dolore magna aliqua. Ut enim
51 \\ad minim veniam, quis nostrud exercitation ullamco
52 \\laboris nisi ut aliquip ex ea commodo
53 \\consequat. Duis aute irure dolor in reprehenderit
54 \\in voluptate velit esse cillum dolore eu
55 \\fugiat nulla pariatur. Excepteur sint occaecat cupidatat
56 \\non proident, sunt in culpa qui officia
57 \\deserunt mollit anim id est laborum.
58 \\
59 );
60 }
61
62 try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");
63}
64
65test "unsupported" {
66 inline for ([_][]const u8{
67 "good-1-delta-lzma2.tiff.xz",
68 "good-1-x86-lzma2.xz",
69 "good-1-sparc-lzma2.xz",
70 "good-1-arm64-lzma2-1.xz",
71 "good-1-arm64-lzma2-2.xz",
72 "good-1-3delta-lzma2.xz",
73 "good-1-empty-bcj-lzma2.xz",
74 }) |filename| {
75 try testing.expectError(
76 error.Unsupported,
77 decompress(@embedFile("testdata/" ++ filename)),
78 );
79 }
80}
lib/std/compress/xz/testdata/good-0-empty.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-0-empty.xz differ
lib/std/compress/xz/testdata/good-0cat-empty.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-0cat-empty.xz differ
lib/std/compress/xz/testdata/good-0catpad-empty.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-0catpad-empty.xz differ
lib/std/compress/xz/testdata/good-0pad-empty.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-0pad-empty.xz differ
lib/std/compress/xz/testdata/good-1-3delta-lzma2.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-3delta-lzma2.xz differ
lib/std/compress/xz/testdata/good-1-arm64-lzma2-1.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-arm64-lzma2-1.xz differ
lib/std/compress/xz/testdata/good-1-arm64-lzma2-2.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-arm64-lzma2-2.xz differ
lib/std/compress/xz/testdata/good-1-block_header-1.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-block_header-1.xz differ
lib/std/compress/xz/testdata/good-1-block_header-2.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-block_header-2.xz differ
lib/std/compress/xz/testdata/good-1-block_header-3.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-block_header-3.xz differ
lib/std/compress/xz/testdata/good-1-check-crc32.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-check-crc32.xz differ
lib/std/compress/xz/testdata/good-1-check-crc64.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-check-crc64.xz differ
lib/std/compress/xz/testdata/good-1-check-none.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-check-none.xz differ
lib/std/compress/xz/testdata/good-1-check-sha256.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-check-sha256.xz differ
lib/std/compress/xz/testdata/good-1-delta-lzma2.tiff.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-delta-lzma2.tiff.xz differ
lib/std/compress/xz/testdata/good-1-empty-bcj-lzma2.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-empty-bcj-lzma2.xz differ
lib/std/compress/xz/testdata/good-1-lzma2-1.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-lzma2-1.xz differ
lib/std/compress/xz/testdata/good-1-lzma2-2.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-lzma2-2.xz differ
lib/std/compress/xz/testdata/good-1-lzma2-3.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-lzma2-3.xz differ
lib/std/compress/xz/testdata/good-1-lzma2-4.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-lzma2-4.xz differ
lib/std/compress/xz/testdata/good-1-lzma2-5.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-lzma2-5.xz differ
lib/std/compress/xz/testdata/good-1-sparc-lzma2.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-sparc-lzma2.xz differ
lib/std/compress/xz/testdata/good-1-x86-lzma2.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-x86-lzma2.xz differ
lib/std/compress/xz/testdata/good-2-lzma2.xz created
Binary files /dev/null and b/lib/std/compress/xz/testdata/good-2-lzma2.xz differ