authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-09 10:13:25-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-02-09 10:13:25-05:00
logd24ebf1d12cf66665b52136a2807f97ff021d78d
tree0f1ca92fd5befc1f4c1fea4a663acd6a8177b1d6
parent2d017f379f6dfa5e35944044eaf34347371b8d33
parent43c76e0c8e742d17bda32bc358ff4125b84e6b26
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14518 from FnControlOption/lzma

19 files changed, 1339 insertions(+), 787 deletions(-)

build.zig+2
...@@ -128,6 +128,8 @@ pub fn build(b: *std.Build) !void {...@@ -128,6 +128,8 @@ pub fn build(b: *std.Build) !void {
128 "compress-gettysburg.txt",128 "compress-gettysburg.txt",
129 "compress-pi.txt",129 "compress-pi.txt",
130 "rfc1951.txt",130 "rfc1951.txt",
131 // exclude files from lib/std/compress/lzma/testdata
132 ".lzma",
131 // exclude files from lib/std/compress/xz/testdata133 // exclude files from lib/std/compress/xz/testdata
132 ".xz",134 ".xz",
133 // exclude files from lib/std/tz/135 // exclude files from lib/std/tz/
lib/std/compress.zig+6-2
...@@ -2,8 +2,10 @@ const std = @import("std.zig");...@@ -2,8 +2,10 @@ const std = @import("std.zig");
22
3pub const deflate = @import("compress/deflate.zig");3pub const deflate = @import("compress/deflate.zig");
4pub const gzip = @import("compress/gzip.zig");4pub const gzip = @import("compress/gzip.zig");
5pub const zlib = @import("compress/zlib.zig");5pub const lzma = @import("compress/lzma.zig");
6pub const lzma2 = @import("compress/lzma2.zig");
6pub const xz = @import("compress/xz.zig");7pub const xz = @import("compress/xz.zig");
8pub const zlib = @import("compress/zlib.zig");
79
8pub fn HashedReader(10pub fn HashedReader(
9 comptime ReaderType: anytype,11 comptime ReaderType: anytype,
...@@ -38,6 +40,8 @@ pub fn hashedReader(...@@ -38,6 +40,8 @@ pub fn hashedReader(
38test {40test {
39 _ = deflate;41 _ = deflate;
40 _ = gzip;42 _ = gzip;
41 _ = zlib;43 _ = lzma;
44 _ = lzma2;
42 _ = xz;45 _ = xz;
46 _ = zlib;
43}47}
lib/std/compress/lzma.zig created+90
...@@ -0,0 +1,90 @@
1const std = @import("../std.zig");
2const math = std.math;
3const mem = std.mem;
4const Allocator = std.mem.Allocator;
5
6pub const decode = @import("lzma/decode.zig");
7
8pub fn decompress(
9 allocator: Allocator,
10 reader: anytype,
11) !Decompress(@TypeOf(reader)) {
12 return decompressWithOptions(allocator, reader, .{});
13}
14
15pub fn decompressWithOptions(
16 allocator: Allocator,
17 reader: anytype,
18 options: decode.Options,
19) !Decompress(@TypeOf(reader)) {
20 const params = try decode.Params.readHeader(reader, options);
21 return Decompress(@TypeOf(reader)).init(allocator, reader, params, options.memlimit);
22}
23
24pub fn Decompress(comptime ReaderType: type) type {
25 return struct {
26 const Self = @This();
27
28 pub const Error =
29 ReaderType.Error ||
30 Allocator.Error ||
31 error{ CorruptInput, EndOfStream, Overflow };
32
33 pub const Reader = std.io.Reader(*Self, Error, read);
34
35 allocator: Allocator,
36 in_reader: ReaderType,
37 to_read: std.ArrayListUnmanaged(u8),
38
39 buffer: decode.lzbuffer.LzCircularBuffer,
40 decoder: decode.rangecoder.RangeDecoder,
41 state: decode.DecoderState,
42
43 pub fn init(allocator: Allocator, source: ReaderType, params: decode.Params, memlimit: ?usize) !Self {
44 return Self{
45 .allocator = allocator,
46 .in_reader = source,
47 .to_read = .{},
48
49 .buffer = decode.lzbuffer.LzCircularBuffer.init(params.dict_size, memlimit orelse math.maxInt(usize)),
50 .decoder = try decode.rangecoder.RangeDecoder.init(source),
51 .state = try decode.DecoderState.init(allocator, params.properties, params.unpacked_size),
52 };
53 }
54
55 pub fn reader(self: *Self) Reader {
56 return .{ .context = self };
57 }
58
59 pub fn deinit(self: *Self) void {
60 self.to_read.deinit(self.allocator);
61 self.buffer.deinit(self.allocator);
62 self.state.deinit(self.allocator);
63 self.* = undefined;
64 }
65
66 pub fn read(self: *Self, output: []u8) Error!usize {
67 const writer = self.to_read.writer(self.allocator);
68 while (self.to_read.items.len < output.len) {
69 switch (try self.state.process(self.allocator, self.in_reader, writer, &self.buffer, &self.decoder)) {
70 .continue_ => {},
71 .finished => {
72 try self.buffer.finish(writer);
73 break;
74 },
75 }
76 }
77 const input = self.to_read.items;
78 const n = math.min(input.len, output.len);
79 mem.copy(u8, output[0..n], input[0..n]);
80 mem.copy(u8, input, input[n..]);
81 self.to_read.shrinkRetainingCapacity(input.len - n);
82 return n;
83 }
84 };
85}
86
87test {
88 _ = @import("lzma/test.zig");
89 _ = @import("lzma/vec2d.zig");
90}
lib/std/compress/lzma/decode.zig created+379
...@@ -0,0 +1,379 @@
1const std = @import("../../std.zig");
2const assert = std.debug.assert;
3const math = std.math;
4const Allocator = std.mem.Allocator;
5
6pub const lzbuffer = @import("decode/lzbuffer.zig");
7pub const rangecoder = @import("decode/rangecoder.zig");
8
9const LzCircularBuffer = lzbuffer.LzCircularBuffer;
10const BitTree = rangecoder.BitTree;
11const LenDecoder = rangecoder.LenDecoder;
12const RangeDecoder = rangecoder.RangeDecoder;
13const Vec2D = @import("vec2d.zig").Vec2D;
14
15pub const Options = struct {
16 unpacked_size: UnpackedSize = .read_from_header,
17 memlimit: ?usize = null,
18 allow_incomplete: bool = false,
19};
20
21pub const UnpackedSize = union(enum) {
22 read_from_header,
23 read_header_but_use_provided: ?u64,
24 use_provided: ?u64,
25};
26
27const ProcessingStatus = enum {
28 continue_,
29 finished,
30};
31
32pub const Properties = struct {
33 lc: u4,
34 lp: u3,
35 pb: u3,
36
37 fn validate(self: Properties) void {
38 assert(self.lc <= 8);
39 assert(self.lp <= 4);
40 assert(self.pb <= 4);
41 }
42};
43
44pub const Params = struct {
45 properties: Properties,
46 dict_size: u32,
47 unpacked_size: ?u64,
48
49 pub fn readHeader(reader: anytype, options: Options) !Params {
50 var props = try reader.readByte();
51 if (props >= 225) {
52 return error.CorruptInput;
53 }
54
55 const lc = @intCast(u4, props % 9);
56 props /= 9;
57 const lp = @intCast(u3, props % 5);
58 props /= 5;
59 const pb = @intCast(u3, props);
60
61 const dict_size_provided = try reader.readIntLittle(u32);
62 const dict_size = math.max(0x1000, dict_size_provided);
63
64 const unpacked_size = switch (options.unpacked_size) {
65 .read_from_header => blk: {
66 const unpacked_size_provided = try reader.readIntLittle(u64);
67 const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
68 break :blk if (marker_mandatory)
69 null
70 else
71 unpacked_size_provided;
72 },
73 .read_header_but_use_provided => |x| blk: {
74 _ = try reader.readIntLittle(u64);
75 break :blk x;
76 },
77 .use_provided => |x| x,
78 };
79
80 return Params{
81 .properties = Properties{ .lc = lc, .lp = lp, .pb = pb },
82 .dict_size = dict_size,
83 .unpacked_size = unpacked_size,
84 };
85 }
86};
87
88pub const DecoderState = struct {
89 lzma_props: Properties,
90 unpacked_size: ?u64,
91 literal_probs: Vec2D(u16),
92 pos_slot_decoder: [4]BitTree(6),
93 align_decoder: BitTree(4),
94 pos_decoders: [115]u16,
95 is_match: [192]u16,
96 is_rep: [12]u16,
97 is_rep_g0: [12]u16,
98 is_rep_g1: [12]u16,
99 is_rep_g2: [12]u16,
100 is_rep_0long: [192]u16,
101 state: usize,
102 rep: [4]usize,
103 len_decoder: LenDecoder,
104 rep_len_decoder: LenDecoder,
105
106 pub fn init(
107 allocator: Allocator,
108 lzma_props: Properties,
109 unpacked_size: ?u64,
110 ) !DecoderState {
111 return .{
112 .lzma_props = lzma_props,
113 .unpacked_size = unpacked_size,
114 .literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }),
115 .pos_slot_decoder = .{.{}} ** 4,
116 .align_decoder = .{},
117 .pos_decoders = .{0x400} ** 115,
118 .is_match = .{0x400} ** 192,
119 .is_rep = .{0x400} ** 12,
120 .is_rep_g0 = .{0x400} ** 12,
121 .is_rep_g1 = .{0x400} ** 12,
122 .is_rep_g2 = .{0x400} ** 12,
123 .is_rep_0long = .{0x400} ** 192,
124 .state = 0,
125 .rep = .{0} ** 4,
126 .len_decoder = .{},
127 .rep_len_decoder = .{},
128 };
129 }
130
131 pub fn deinit(self: *DecoderState, allocator: Allocator) void {
132 self.literal_probs.deinit(allocator);
133 self.* = undefined;
134 }
135
136 pub fn resetState(self: *DecoderState, allocator: Allocator, new_props: Properties) !void {
137 new_props.validate();
138 if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
139 self.literal_probs.fill(0x400);
140 } else {
141 self.literal_probs.deinit(allocator);
142 self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
143 }
144
145 self.lzma_props = new_props;
146 for (self.pos_slot_decoder) |*t| t.reset();
147 self.align_decoder.reset();
148 self.pos_decoders = .{0x400} ** 115;
149 self.is_match = .{0x400} ** 192;
150 self.is_rep = .{0x400} ** 12;
151 self.is_rep_g0 = .{0x400} ** 12;
152 self.is_rep_g1 = .{0x400} ** 12;
153 self.is_rep_g2 = .{0x400} ** 12;
154 self.is_rep_0long = .{0x400} ** 192;
155 self.state = 0;
156 self.rep = .{0} ** 4;
157 self.len_decoder.reset();
158 self.rep_len_decoder.reset();
159 }
160
161 fn processNextInner(
162 self: *DecoderState,
163 allocator: Allocator,
164 reader: anytype,
165 writer: anytype,
166 buffer: anytype,
167 decoder: *RangeDecoder,
168 update: bool,
169 ) !ProcessingStatus {
170 const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1);
171
172 if (!try decoder.decodeBit(
173 reader,
174 &self.is_match[(self.state << 4) + pos_state],
175 update,
176 )) {
177 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);
178
179 if (update) {
180 try buffer.appendLiteral(allocator, byte, writer);
181
182 self.state = if (self.state < 4)
183 0
184 else if (self.state < 10)
185 self.state - 3
186 else
187 self.state - 6;
188 }
189 return .continue_;
190 }
191
192 var len: usize = undefined;
193 if (try decoder.decodeBit(reader, &self.is_rep[self.state], update)) {
194 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], update)) {
195 if (!try decoder.decodeBit(
196 reader,
197 &self.is_rep_0long[(self.state << 4) + pos_state],
198 update,
199 )) {
200 if (update) {
201 self.state = if (self.state < 7) 9 else 11;
202 const dist = self.rep[0] + 1;
203 try buffer.appendLz(allocator, 1, dist, writer);
204 }
205 return .continue_;
206 }
207 } else {
208 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))
209 1
210 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], update))
211 2
212 else
213 3;
214 if (update) {
215 const dist = self.rep[idx];
216 var i = idx;
217 while (i > 0) : (i -= 1) {
218 self.rep[i] = self.rep[i - 1];
219 }
220 self.rep[0] = dist;
221 }
222 }
223
224 len = try self.rep_len_decoder.decode(reader, decoder, pos_state, update);
225
226 if (update) {
227 self.state = if (self.state < 7) 8 else 11;
228 }
229 } else {
230 if (update) {
231 self.rep[3] = self.rep[2];
232 self.rep[2] = self.rep[1];
233 self.rep[1] = self.rep[0];
234 }
235
236 len = try self.len_decoder.decode(reader, decoder, pos_state, update);
237
238 if (update) {
239 self.state = if (self.state < 7) 7 else 10;
240 }
241
242 const rep_0 = try self.decodeDistance(reader, decoder, len, update);
243
244 if (update) {
245 self.rep[0] = rep_0;
246 if (self.rep[0] == 0xFFFF_FFFF) {
247 if (decoder.isFinished()) {
248 return .finished;
249 }
250 return error.CorruptInput;
251 }
252 }
253 }
254
255 if (update) {
256 len += 2;
257
258 const dist = self.rep[0] + 1;
259 try buffer.appendLz(allocator, len, dist, writer);
260 }
261
262 return .continue_;
263 }
264
265 fn processNext(
266 self: *DecoderState,
267 allocator: Allocator,
268 reader: anytype,
269 writer: anytype,
270 buffer: anytype,
271 decoder: *RangeDecoder,
272 ) !ProcessingStatus {
273 return self.processNextInner(allocator, reader, writer, buffer, decoder, true);
274 }
275
276 pub fn process(
277 self: *DecoderState,
278 allocator: Allocator,
279 reader: anytype,
280 writer: anytype,
281 buffer: anytype,
282 decoder: *RangeDecoder,
283 ) !ProcessingStatus {
284 process_next: {
285 if (self.unpacked_size) |unpacked_size| {
286 if (buffer.len >= unpacked_size) {
287 break :process_next;
288 }
289 } else if (decoder.isFinished()) {
290 break :process_next;
291 }
292
293 switch (try self.processNext(allocator, reader, writer, buffer, decoder)) {
294 .continue_ => return .continue_,
295 .finished => break :process_next,
296 }
297 }
298
299 if (self.unpacked_size) |unpacked_size| {
300 if (buffer.len != unpacked_size) {
301 return error.CorruptInput;
302 }
303 }
304
305 return .finished;
306 }
307
308 fn decodeLiteral(
309 self: *DecoderState,
310 reader: anytype,
311 buffer: anytype,
312 decoder: *RangeDecoder,
313 update: bool,
314 ) !u8 {
315 const def_prev_byte = 0;
316 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
317
318 var result: usize = 1;
319 const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
320 (prev_byte >> (8 - self.lzma_props.lc));
321 const probs = try self.literal_probs.getMut(lit_state);
322
323 if (self.state >= 7) {
324 var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
325
326 while (result < 0x100) {
327 const match_bit = (match_byte >> 7) & 1;
328 match_byte <<= 1;
329 const bit = @boolToInt(try decoder.decodeBit(
330 reader,
331 &probs[((@as(usize, 1) + match_bit) << 8) + result],
332 update,
333 ));
334 result = (result << 1) ^ bit;
335 if (match_bit != bit) {
336 break;
337 }
338 }
339 }
340
341 while (result < 0x100) {
342 result = (result << 1) ^ @boolToInt(try decoder.decodeBit(reader, &probs[result], update));
343 }
344
345 return @truncate(u8, result - 0x100);
346 }
347
348 fn decodeDistance(
349 self: *DecoderState,
350 reader: anytype,
351 decoder: *RangeDecoder,
352 length: usize,
353 update: bool,
354 ) !usize {
355 const len_state = if (length > 3) 3 else length;
356
357 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder, update));
358 if (pos_slot < 4)
359 return pos_slot;
360
361 const num_direct_bits = @intCast(u5, (pos_slot >> 1) - 1);
362 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
363
364 if (pos_slot < 14) {
365 result += try decoder.parseReverseBitTree(
366 reader,
367 num_direct_bits,
368 &self.pos_decoders,
369 result - pos_slot,
370 update,
371 );
372 } else {
373 result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4;
374 result += try self.align_decoder.parseReverse(reader, decoder, update);
375 }
376
377 return result;
378 }
379};
lib/std/compress/lzma/decode/lzbuffer.zig created+228
...@@ -0,0 +1,228 @@
1const std = @import("../../../std.zig");
2const math = std.math;
3const mem = std.mem;
4const Allocator = std.mem.Allocator;
5const ArrayListUnmanaged = std.ArrayListUnmanaged;
6
7/// An accumulating buffer for LZ sequences
8pub const LzAccumBuffer = struct {
9 /// Buffer
10 buf: ArrayListUnmanaged(u8),
11
12 /// Buffer memory limit
13 memlimit: usize,
14
15 /// Total number of bytes sent through the buffer
16 len: usize,
17
18 const Self = @This();
19
20 pub fn init(memlimit: usize) Self {
21 return Self{
22 .buf = .{},
23 .memlimit = memlimit,
24 .len = 0,
25 };
26 }
27
28 pub fn appendByte(self: *Self, allocator: Allocator, byte: u8) !void {
29 try self.buf.append(allocator, byte);
30 self.len += 1;
31 }
32
33 /// Reset the internal dictionary
34 pub fn reset(self: *Self, writer: anytype) !void {
35 try writer.writeAll(self.buf.items);
36 self.buf.clearRetainingCapacity();
37 self.len = 0;
38 }
39
40 /// Retrieve the last byte or return a default
41 pub fn lastOr(self: Self, lit: u8) u8 {
42 const buf_len = self.buf.items.len;
43 return if (buf_len == 0)
44 lit
45 else
46 self.buf.items[buf_len - 1];
47 }
48
49 /// Retrieve the n-th last byte
50 pub fn lastN(self: Self, dist: usize) !u8 {
51 const buf_len = self.buf.items.len;
52 if (dist > buf_len) {
53 return error.CorruptInput;
54 }
55
56 return self.buf.items[buf_len - dist];
57 }
58
59 /// Append a literal
60 pub fn appendLiteral(
61 self: *Self,
62 allocator: Allocator,
63 lit: u8,
64 writer: anytype,
65 ) !void {
66 _ = writer;
67 if (self.len >= self.memlimit) {
68 return error.CorruptInput;
69 }
70 try self.buf.append(allocator, lit);
71 self.len += 1;
72 }
73
74 /// Fetch an LZ sequence (length, distance) from inside the buffer
75 pub fn appendLz(
76 self: *Self,
77 allocator: Allocator,
78 len: usize,
79 dist: usize,
80 writer: anytype,
81 ) !void {
82 _ = writer;
83
84 const buf_len = self.buf.items.len;
85 if (dist > buf_len) {
86 return error.CorruptInput;
87 }
88
89 var offset = buf_len - dist;
90 var i: usize = 0;
91 while (i < len) : (i += 1) {
92 const x = self.buf.items[offset];
93 try self.buf.append(allocator, x);
94 offset += 1;
95 }
96 self.len += len;
97 }
98
99 pub fn finish(self: *Self, writer: anytype) !void {
100 try writer.writeAll(self.buf.items);
101 self.buf.clearRetainingCapacity();
102 }
103
104 pub fn deinit(self: *Self, allocator: Allocator) void {
105 self.buf.deinit(allocator);
106 self.* = undefined;
107 }
108};
109
110/// A circular buffer for LZ sequences
111pub const LzCircularBuffer = struct {
112 /// Circular buffer
113 buf: ArrayListUnmanaged(u8),
114
115 /// Length of the buffer
116 dict_size: usize,
117
118 /// Buffer memory limit
119 memlimit: usize,
120
121 /// Current position
122 cursor: usize,
123
124 /// Total number of bytes sent through the buffer
125 len: usize,
126
127 const Self = @This();
128
129 pub fn init(dict_size: usize, memlimit: usize) Self {
130 return Self{
131 .buf = .{},
132 .dict_size = dict_size,
133 .memlimit = memlimit,
134 .cursor = 0,
135 .len = 0,
136 };
137 }
138
139 pub fn get(self: Self, index: usize) u8 {
140 return if (0 <= index and index < self.buf.items.len)
141 self.buf.items[index]
142 else
143 0;
144 }
145
146 pub fn set(self: *Self, allocator: Allocator, index: usize, value: u8) !void {
147 if (index >= self.memlimit) {
148 return error.CorruptInput;
149 }
150 try self.buf.ensureTotalCapacity(allocator, index + 1);
151 while (self.buf.items.len < index) {
152 self.buf.appendAssumeCapacity(0);
153 }
154 self.buf.appendAssumeCapacity(value);
155 }
156
157 /// Retrieve the last byte or return a default
158 pub fn lastOr(self: Self, lit: u8) u8 {
159 return if (self.len == 0)
160 lit
161 else
162 self.get((self.dict_size + self.cursor - 1) % self.dict_size);
163 }
164
165 /// Retrieve the n-th last byte
166 pub fn lastN(self: Self, dist: usize) !u8 {
167 if (dist > self.dict_size or dist > self.len) {
168 return error.CorruptInput;
169 }
170
171 const offset = (self.dict_size + self.cursor - dist) % self.dict_size;
172 return self.get(offset);
173 }
174
175 /// Append a literal
176 pub fn appendLiteral(
177 self: *Self,
178 allocator: Allocator,
179 lit: u8,
180 writer: anytype,
181 ) !void {
182 try self.set(allocator, self.cursor, lit);
183 self.cursor += 1;
184 self.len += 1;
185
186 // Flush the circular buffer to the output
187 if (self.cursor == self.dict_size) {
188 try writer.writeAll(self.buf.items);
189 self.cursor = 0;
190 }
191 }
192
193 /// Fetch an LZ sequence (length, distance) from inside the buffer
194 pub fn appendLz(
195 self: *Self,
196 allocator: Allocator,
197 len: usize,
198 dist: usize,
199 writer: anytype,
200 ) !void {
201 if (dist > self.dict_size or dist > self.len) {
202 return error.CorruptInput;
203 }
204
205 var offset = (self.dict_size + self.cursor - dist) % self.dict_size;
206 var i: usize = 0;
207 while (i < len) : (i += 1) {
208 const x = self.get(offset);
209 try self.appendLiteral(allocator, x, writer);
210 offset += 1;
211 if (offset == self.dict_size) {
212 offset = 0;
213 }
214 }
215 }
216
217 pub fn finish(self: *Self, writer: anytype) !void {
218 if (self.cursor > 0) {
219 try writer.writeAll(self.buf.items[0..self.cursor]);
220 self.cursor = 0;
221 }
222 }
223
224 pub fn deinit(self: *Self, allocator: Allocator) void {
225 self.buf.deinit(allocator);
226 self.* = undefined;
227 }
228};
lib/std/compress/lzma/decode/rangecoder.zig created+181
...@@ -0,0 +1,181 @@
1const std = @import("../../../std.zig");
2const mem = std.mem;
3
4pub const RangeDecoder = struct {
5 range: u32,
6 code: u32,
7
8 pub fn init(reader: anytype) !RangeDecoder {
9 const reserved = try reader.readByte();
10 if (reserved != 0) {
11 return error.CorruptInput;
12 }
13 return RangeDecoder{
14 .range = 0xFFFF_FFFF,
15 .code = try reader.readIntBig(u32),
16 };
17 }
18
19 pub fn fromParts(
20 range: u32,
21 code: u32,
22 ) RangeDecoder {
23 return .{
24 .range = range,
25 .code = code,
26 };
27 }
28
29 pub fn set(self: *RangeDecoder, range: u32, code: u32) void {
30 self.range = range;
31 self.code = code;
32 }
33
34 pub inline fn isFinished(self: RangeDecoder) bool {
35 return self.code == 0;
36 }
37
38 inline fn normalize(self: *RangeDecoder, reader: anytype) !void {
39 if (self.range < 0x0100_0000) {
40 self.range <<= 8;
41 self.code = (self.code << 8) ^ @as(u32, try reader.readByte());
42 }
43 }
44
45 inline fn getBit(self: *RangeDecoder, reader: anytype) !bool {
46 self.range >>= 1;
47
48 const bit = self.code >= self.range;
49 if (bit)
50 self.code -= self.range;
51
52 try self.normalize(reader);
53 return bit;
54 }
55
56 pub fn get(self: *RangeDecoder, reader: anytype, count: usize) !u32 {
57 var result: u32 = 0;
58 var i: usize = 0;
59 while (i < count) : (i += 1)
60 result = (result << 1) ^ @boolToInt(try self.getBit(reader));
61 return result;
62 }
63
64 pub inline fn decodeBit(self: *RangeDecoder, reader: anytype, prob: *u16, update: bool) !bool {
65 const bound = (self.range >> 11) * prob.*;
66
67 if (self.code < bound) {
68 if (update)
69 prob.* += (0x800 - prob.*) >> 5;
70 self.range = bound;
71
72 try self.normalize(reader);
73 return false;
74 } else {
75 if (update)
76 prob.* -= prob.* >> 5;
77 self.code -= bound;
78 self.range -= bound;
79
80 try self.normalize(reader);
81 return true;
82 }
83 }
84
85 fn parseBitTree(
86 self: *RangeDecoder,
87 reader: anytype,
88 num_bits: u5,
89 probs: []u16,
90 update: bool,
91 ) !u32 {
92 var tmp: u32 = 1;
93 var i: @TypeOf(num_bits) = 0;
94 while (i < num_bits) : (i += 1) {
95 const bit = try self.decodeBit(reader, &probs[tmp], update);
96 tmp = (tmp << 1) ^ @boolToInt(bit);
97 }
98 return tmp - (@as(u32, 1) << num_bits);
99 }
100
101 pub fn parseReverseBitTree(
102 self: *RangeDecoder,
103 reader: anytype,
104 num_bits: u5,
105 probs: []u16,
106 offset: usize,
107 update: bool,
108 ) !u32 {
109 var result: u32 = 0;
110 var tmp: usize = 1;
111 var i: @TypeOf(num_bits) = 0;
112 while (i < num_bits) : (i += 1) {
113 const bit = @boolToInt(try self.decodeBit(reader, &probs[offset + tmp], update));
114 tmp = (tmp << 1) ^ bit;
115 result ^= @as(u32, bit) << i;
116 }
117 return result;
118 }
119};
120
121pub fn BitTree(comptime num_bits: usize) type {
122 return struct {
123 probs: [1 << num_bits]u16 = .{0x400} ** (1 << num_bits),
124
125 const Self = @This();
126
127 pub fn parse(
128 self: *Self,
129 reader: anytype,
130 decoder: *RangeDecoder,
131 update: bool,
132 ) !u32 {
133 return decoder.parseBitTree(reader, num_bits, &self.probs, update);
134 }
135
136 pub fn parseReverse(
137 self: *Self,
138 reader: anytype,
139 decoder: *RangeDecoder,
140 update: bool,
141 ) !u32 {
142 return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, update);
143 }
144
145 pub fn reset(self: *Self) void {
146 mem.set(u16, &self.probs, 0x400);
147 }
148 };
149}
150
151pub const LenDecoder = struct {
152 choice: u16 = 0x400,
153 choice2: u16 = 0x400,
154 low_coder: [16]BitTree(3) = .{.{}} ** 16,
155 mid_coder: [16]BitTree(3) = .{.{}} ** 16,
156 high_coder: BitTree(8) = .{},
157
158 pub fn decode(
159 self: *LenDecoder,
160 reader: anytype,
161 decoder: *RangeDecoder,
162 pos_state: usize,
163 update: bool,
164 ) !usize {
165 if (!try decoder.decodeBit(reader, &self.choice, update)) {
166 return @as(usize, try self.low_coder[pos_state].parse(reader, decoder, update));
167 } else if (!try decoder.decodeBit(reader, &self.choice2, update)) {
168 return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder, update)) + 8;
169 } else {
170 return @as(usize, try self.high_coder.parse(reader, decoder, update)) + 16;
171 }
172 }
173
174 pub fn reset(self: *LenDecoder) void {
175 self.choice = 0x400;
176 self.choice2 = 0x400;
177 for (self.low_coder) |*t| t.reset();
178 for (self.mid_coder) |*t| t.reset();
179 self.high_coder.reset();
180 }
181};
lib/std/compress/lzma/test.zig created+89
...@@ -0,0 +1,89 @@
1const std = @import("../../std.zig");
2const lzma = @import("../lzma.zig");
3
4fn testDecompress(compressed: []const u8) ![]u8 {
5 const allocator = std.testing.allocator;
6 var stream = std.io.fixedBufferStream(compressed);
7 var decompressor = try lzma.decompress(allocator, stream.reader());
8 defer decompressor.deinit();
9 const reader = decompressor.reader();
10 return reader.readAllAlloc(allocator, std.math.maxInt(usize));
11}
12
13fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
14 const allocator = std.testing.allocator;
15 const decomp = try testDecompress(compressed);
16 defer allocator.free(decomp);
17 try std.testing.expectEqualSlices(u8, expected, decomp);
18}
19
20fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
21 return std.testing.expectError(expected, testDecompress(compressed));
22}
23
24test "LZMA: decompress empty world" {
25 try testDecompressEqual(
26 "",
27 &[_]u8{
28 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x83, 0xff,
29 0xfb, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00,
30 },
31 );
32}
33
34test "LZMA: decompress hello world" {
35 try testDecompressEqual(
36 "Hello world\n",
37 &[_]u8{
38 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
39 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
40 0xa5, 0xb0, 0x00,
41 },
42 );
43}
44
45test "LZMA: decompress huge dict" {
46 try testDecompressEqual(
47 "Hello world\n",
48 &[_]u8{
49 0x5d, 0x7f, 0x7f, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
50 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
51 0xa5, 0xb0, 0x00,
52 },
53 );
54}
55
56test "LZMA: unknown size with end of payload marker" {
57 try testDecompressEqual(
58 "Hello\nWorld!\n",
59 @embedFile("testdata/good-unknown_size-with_eopm.lzma"),
60 );
61}
62
63test "LZMA: known size without end of payload marker" {
64 try testDecompressEqual(
65 "Hello\nWorld!\n",
66 @embedFile("testdata/good-known_size-without_eopm.lzma"),
67 );
68}
69
70test "LZMA: known size with end of payload marker" {
71 try testDecompressEqual(
72 "Hello\nWorld!\n",
73 @embedFile("testdata/good-known_size-with_eopm.lzma"),
74 );
75}
76
77test "LZMA: too big uncompressed size in header" {
78 try testDecompressError(
79 error.CorruptInput,
80 @embedFile("testdata/bad-too_big_size-with_eopm.lzma"),
81 );
82}
83
84test "LZMA: too small uncompressed size in header" {
85 try testDecompressError(
86 error.CorruptInput,
87 @embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"),
88 );
89}
lib/std/compress/lzma/testdata/bad-too_big_size-with_eopm.lzma created
Binary files /dev/null and b/lib/std/compress/lzma/testdata/bad-too_big_size-with_eopm.lzma differ
lib/std/compress/lzma/testdata/bad-too_small_size-without_eopm-3.lzma created
Binary files /dev/null and b/lib/std/compress/lzma/testdata/bad-too_small_size-without_eopm-3.lzma differ
lib/std/compress/lzma/testdata/good-known_size-with_eopm.lzma created
Binary files /dev/null and b/lib/std/compress/lzma/testdata/good-known_size-with_eopm.lzma differ
lib/std/compress/lzma/testdata/good-known_size-without_eopm.lzma created
Binary files /dev/null and b/lib/std/compress/lzma/testdata/good-known_size-without_eopm.lzma differ
lib/std/compress/lzma/testdata/good-unknown_size-with_eopm.lzma created
Binary files /dev/null and b/lib/std/compress/lzma/testdata/good-unknown_size-with_eopm.lzma differ
lib/std/compress/lzma/vec2d.zig created+128
...@@ -0,0 +1,128 @@
1const std = @import("../../std.zig");
2const math = std.math;
3const mem = std.mem;
4const Allocator = std.mem.Allocator;
5
6pub fn Vec2D(comptime T: type) type {
7 return struct {
8 data: []T,
9 cols: usize,
10
11 const Self = @This();
12
13 pub fn init(allocator: Allocator, value: T, size: struct { usize, usize }) !Self {
14 const len = try math.mul(usize, size[0], size[1]);
15 const data = try allocator.alloc(T, len);
16 mem.set(T, data, value);
17 return Self{
18 .data = data,
19 .cols = size[1],
20 };
21 }
22
23 pub fn deinit(self: *Self, allocator: Allocator) void {
24 allocator.free(self.data);
25 self.* = undefined;
26 }
27
28 pub fn fill(self: *Self, value: T) void {
29 mem.set(T, self.data, value);
30 }
31
32 inline fn _get(self: Self, row: usize) ![]T {
33 const start_row = try math.mul(usize, row, self.cols);
34 const end_row = try math.add(usize, start_row, self.cols);
35 return self.data[start_row..end_row];
36 }
37
38 pub fn get(self: Self, row: usize) ![]const T {
39 return self._get(row);
40 }
41
42 pub fn getMut(self: *Self, row: usize) ![]T {
43 return self._get(row);
44 }
45 };
46}
47
48const testing = std.testing;
49const expectEqualSlices = std.testing.expectEqualSlices;
50const expectError = std.testing.expectError;
51
52test "Vec2D.init" {
53 const allocator = testing.allocator;
54 var vec2d = try Vec2D(i32).init(allocator, 1, .{ 2, 3 });
55 defer vec2d.deinit(allocator);
56
57 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(0));
58 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(1));
59}
60
61test "Vec2D.init overflow" {
62 const allocator = testing.allocator;
63 try expectError(
64 error.Overflow,
65 Vec2D(i32).init(allocator, 1, .{ math.maxInt(usize), math.maxInt(usize) }),
66 );
67}
68
69test "Vec2D.fill" {
70 const allocator = testing.allocator;
71 var vec2d = try Vec2D(i32).init(allocator, 0, .{ 2, 3 });
72 defer vec2d.deinit(allocator);
73
74 vec2d.fill(7);
75
76 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(0));
77 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(1));
78}
79
80test "Vec2D.get" {
81 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
82 const vec2d = Vec2D(i32){
83 .data = &data,
84 .cols = 2,
85 };
86
87 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
88 try expectEqualSlices(i32, &.{ 2, 3 }, try vec2d.get(1));
89 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
90 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
91}
92
93test "Vec2D.getMut" {
94 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
95 var vec2d = Vec2D(i32){
96 .data = &data,
97 .cols = 2,
98 };
99
100 const row = try vec2d.getMut(1);
101 row[1] = 9;
102
103 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
104 // (1, 1) should be 9.
105 try expectEqualSlices(i32, &.{ 2, 9 }, try vec2d.get(1));
106 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
107 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
108}
109
110test "Vec2D.get multiplication overflow" {
111 const allocator = testing.allocator;
112 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 4 });
113 defer matrix.deinit(allocator);
114
115 const row = (math.maxInt(usize) / 4) + 1;
116 try expectError(error.Overflow, matrix.get(row));
117 try expectError(error.Overflow, matrix.getMut(row));
118}
119
120test "Vec2D.get addition overflow" {
121 const allocator = testing.allocator;
122 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 5 });
123 defer matrix.deinit(allocator);
124
125 const row = math.maxInt(usize) / 5;
126 try expectError(error.Overflow, matrix.get(row));
127 try expectError(error.Overflow, matrix.getMut(row));
128}
lib/std/compress/lzma2.zig created+26
...@@ -0,0 +1,26 @@
1const std = @import("../std.zig");
2const Allocator = std.mem.Allocator;
3
4pub const decode = @import("lzma2/decode.zig");
5
6pub fn decompress(
7 allocator: Allocator,
8 reader: anytype,
9 writer: anytype,
10) !void {
11 var decoder = try decode.Decoder.init(allocator);
12 defer decoder.deinit(allocator);
13 return decoder.decompress(allocator, reader, writer);
14}
15
16test {
17 const expected = "Hello\nWorld!\n";
18 const compressed = &[_]u8{ 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02, 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00 };
19
20 const allocator = std.testing.allocator;
21 var decomp = std.ArrayList(u8).init(allocator);
22 defer decomp.deinit();
23 var stream = std.io.fixedBufferStream(compressed);
24 try decompress(allocator, stream.reader(), decomp.writer());
25 try std.testing.expectEqualSlices(u8, expected, decomp.items);
26}
lib/std/compress/lzma2/decode.zig created+169
...@@ -0,0 +1,169 @@
1const std = @import("../../std.zig");
2const Allocator = std.mem.Allocator;
3
4const lzma = @import("../lzma.zig");
5const DecoderState = lzma.decode.DecoderState;
6const LzAccumBuffer = lzma.decode.lzbuffer.LzAccumBuffer;
7const Properties = lzma.decode.Properties;
8const RangeDecoder = lzma.decode.rangecoder.RangeDecoder;
9
10pub const Decoder = struct {
11 lzma_state: DecoderState,
12
13 pub fn init(allocator: Allocator) !Decoder {
14 return Decoder{
15 .lzma_state = try DecoderState.init(
16 allocator,
17 Properties{
18 .lc = 0,
19 .lp = 0,
20 .pb = 0,
21 },
22 null,
23 ),
24 };
25 }
26
27 pub fn deinit(self: *Decoder, allocator: Allocator) void {
28 self.lzma_state.deinit(allocator);
29 self.* = undefined;
30 }
31
32 pub fn decompress(
33 self: *Decoder,
34 allocator: Allocator,
35 reader: anytype,
36 writer: anytype,
37 ) !void {
38 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
39 defer accum.deinit(allocator);
40
41 while (true) {
42 const status = try reader.readByte();
43
44 switch (status) {
45 0 => break,
46 1 => try parseUncompressed(allocator, reader, writer, &accum, true),
47 2 => try parseUncompressed(allocator, reader, writer, &accum, false),
48 else => try self.parseLzma(allocator, reader, writer, &accum, status),
49 }
50 }
51
52 try accum.finish(writer);
53 }
54
55 fn parseLzma(
56 self: *Decoder,
57 allocator: Allocator,
58 reader: anytype,
59 writer: anytype,
60 accum: *LzAccumBuffer,
61 status: u8,
62 ) !void {
63 if (status & 0x80 == 0) {
64 return error.CorruptInput;
65 }
66
67 const Reset = struct {
68 dict: bool,
69 state: bool,
70 props: bool,
71 };
72
73 const reset = switch ((status >> 5) & 0x3) {
74 0 => Reset{
75 .dict = false,
76 .state = false,
77 .props = false,
78 },
79 1 => Reset{
80 .dict = false,
81 .state = true,
82 .props = false,
83 },
84 2 => Reset{
85 .dict = false,
86 .state = true,
87 .props = true,
88 },
89 3 => Reset{
90 .dict = true,
91 .state = true,
92 .props = true,
93 },
94 else => unreachable,
95 };
96
97 const unpacked_size = blk: {
98 var tmp: u64 = status & 0x1F;
99 tmp <<= 16;
100 tmp |= try reader.readIntBig(u16);
101 break :blk tmp + 1;
102 };
103
104 const packed_size = blk: {
105 const tmp: u17 = try reader.readIntBig(u16);
106 break :blk tmp + 1;
107 };
108
109 if (reset.dict) {
110 try accum.reset(writer);
111 }
112
113 if (reset.state) {
114 var new_props = self.lzma_state.lzma_props;
115
116 if (reset.props) {
117 var props = try reader.readByte();
118 if (props >= 225) {
119 return error.CorruptInput;
120 }
121
122 const lc = @intCast(u4, props % 9);
123 props /= 9;
124 const lp = @intCast(u3, props % 5);
125 props /= 5;
126 const pb = @intCast(u3, props);
127
128 if (lc + lp > 4) {
129 return error.CorruptInput;
130 }
131
132 new_props = Properties{ .lc = lc, .lp = lp, .pb = pb };
133 }
134
135 try self.lzma_state.resetState(allocator, new_props);
136 }
137
138 self.lzma_state.unpacked_size = unpacked_size + accum.len;
139
140 var counter = std.io.countingReader(reader);
141 const counter_reader = counter.reader();
142
143 var rangecoder = try RangeDecoder.init(counter_reader);
144 while (try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
145
146 if (counter.bytes_read != packed_size) {
147 return error.CorruptInput;
148 }
149 }
150
151 fn parseUncompressed(
152 allocator: Allocator,
153 reader: anytype,
154 writer: anytype,
155 accum: *LzAccumBuffer,
156 reset_dict: bool,
157 ) !void {
158 const unpacked_size = @as(u17, try reader.readIntBig(u16)) + 1;
159
160 if (reset_dict) {
161 try accum.reset(writer);
162 }
163
164 var i: @TypeOf(unpacked_size) = 0;
165 while (i < unpacked_size) : (i += 1) {
166 try accum.appendByte(allocator, try reader.readByte());
167 }
168 }
169};
lib/std/compress/xz.zig+1-1
...@@ -118,7 +118,7 @@ pub fn Decompress(comptime ReaderType: type) type {...@@ -118,7 +118,7 @@ pub fn Decompress(comptime ReaderType: type) type {
118 var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());118 var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
119 const hashed_reader = hasher.reader();119 const hashed_reader = hasher.reader();
120120
121 const backward_size = (try hashed_reader.readIntLittle(u32) + 1) * 4;121 const backward_size = (@as(u64, try hashed_reader.readIntLittle(u32)) + 1) * 4;
122 if (backward_size != index_size)122 if (backward_size != index_size)
123 return error.CorruptInput;123 return error.CorruptInput;
124124
lib/std/compress/xz/block.zig+20-126
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1const std = @import("../../std.zig");1const std = @import("../../std.zig");
2const lzma = @import("lzma.zig");2const lzma2 = std.compress.lzma2;
3const Allocator = std.mem.Allocator;3const Allocator = std.mem.Allocator;
4const ArrayListUnmanaged = std.ArrayListUnmanaged;
4const Crc32 = std.hash.Crc32;5const Crc32 = std.hash.Crc32;
5const Crc64 = std.hash.crc.Crc64Xz;6const Crc64 = std.hash.crc.Crc64Xz;
6const Sha256 = std.crypto.hash.sha2.Sha256;7const Sha256 = std.crypto.hash.sha2.Sha256;
...@@ -32,8 +33,7 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -32,8 +33,7 @@ pub fn Decoder(comptime ReaderType: type) type {
32 inner_reader: ReaderType,33 inner_reader: ReaderType,
33 check: xz.Check,34 check: xz.Check,
34 err: ?Error,35 err: ?Error,
35 accum: lzma.LzAccumBuffer,36 to_read: ArrayListUnmanaged(u8),
36 lzma_state: lzma.DecoderState,
37 block_count: usize,37 block_count: usize,
3838
39 fn init(allocator: Allocator, in_reader: ReaderType, check: xz.Check) !Self {39 fn init(allocator: Allocator, in_reader: ReaderType, check: xz.Check) !Self {
...@@ -42,15 +42,13 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -42,15 +42,13 @@ pub fn Decoder(comptime ReaderType: type) type {
42 .inner_reader = in_reader,42 .inner_reader = in_reader,
43 .check = check,43 .check = check,
44 .err = null,44 .err = null,
45 .accum = .{},45 .to_read = .{},
46 .lzma_state = try lzma.DecoderState.init(allocator),
47 .block_count = 0,46 .block_count = 0,
48 };47 };
49 }48 }
5049
51 pub fn deinit(self: *Self) void {50 pub fn deinit(self: *Self) void {
52 self.accum.deinit(self.allocator);51 self.to_read.deinit(self.allocator);
53 self.lzma_state.deinit(self.allocator);
54 }52 }
5553
56 pub fn reader(self: *Self) Reader {54 pub fn reader(self: *Self) Reader {
...@@ -59,9 +57,13 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -59,9 +57,13 @@ pub fn Decoder(comptime ReaderType: type) type {
5957
60 pub fn read(self: *Self, output: []u8) Error!usize {58 pub fn read(self: *Self, output: []u8) Error!usize {
61 while (true) {59 while (true) {
62 if (self.accum.to_read.items.len > 0) {60 if (self.to_read.items.len > 0) {
63 const n = self.accum.read(output);61 const input = self.to_read.items;
64 if (self.accum.to_read.items.len == 0 and self.err != null) {62 const n = std.math.min(input.len, output.len);
63 std.mem.copy(u8, output[0..n], input[0..n]);
64 std.mem.copy(u8, input, input[n..]);
65 self.to_read.shrinkRetainingCapacity(input.len - n);
66 if (self.to_read.items.len == 0 and self.err != null) {
65 if (self.err.? == DecodeError.EndOfStreamWithNoError) {67 if (self.err.? == DecodeError.EndOfStreamWithNoError) {
66 return n;68 return n;
67 }69 }
...@@ -77,15 +79,12 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -77,15 +79,12 @@ pub fn Decoder(comptime ReaderType: type) type {
77 }79 }
78 self.readBlock() catch |e| {80 self.readBlock() catch |e| {
79 self.err = e;81 self.err = e;
80 if (self.accum.to_read.items.len == 0) {
81 try self.accum.reset(self.allocator);
82 }
83 };82 };
84 }83 }
85 }84 }
8685
87 fn readBlock(self: *Self) Error!void {86 fn readBlock(self: *Self) Error!void {
88 const unpacked_pos = self.accum.to_read.items.len;87 const unpacked_pos = self.to_read.items.len;
8988
90 var block_counter = std.io.countingReader(self.inner_reader);89 var block_counter = std.io.countingReader(self.inner_reader);
91 const block_reader = block_counter.reader();90 const block_reader = block_counter.reader();
...@@ -98,7 +97,7 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -98,7 +97,7 @@ pub fn Decoder(comptime ReaderType: type) type {
98 var header_hasher = std.compress.hashedReader(block_reader, Crc32.init());97 var header_hasher = std.compress.hashedReader(block_reader, Crc32.init());
99 const header_reader = header_hasher.reader();98 const header_reader = header_hasher.reader();
10099
101 const header_size = try header_reader.readByte() * 4;100 const header_size = @as(u64, try header_reader.readByte()) * 4;
102 if (header_size == 0)101 if (header_size == 0)
103 return error.EndOfStreamWithNoError;102 return error.EndOfStreamWithNoError;
104103
...@@ -156,15 +155,18 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -156,15 +155,18 @@ pub fn Decoder(comptime ReaderType: type) type {
156155
157 // Compressed Data156 // Compressed Data
158 var packed_counter = std.io.countingReader(block_reader);157 var packed_counter = std.io.countingReader(block_reader);
159 const packed_reader = packed_counter.reader();158 try lzma2.decompress(
160 while (try self.readLzma2Chunk(packed_reader)) {}159 self.allocator,
160 packed_counter.reader(),
161 self.to_read.writer(self.allocator),
162 );
161163
162 if (packed_size) |s| {164 if (packed_size) |s| {
163 if (s != packed_counter.bytes_read)165 if (s != packed_counter.bytes_read)
164 return error.CorruptInput;166 return error.CorruptInput;
165 }167 }
166168
167 const unpacked_bytes = self.accum.to_read.items[unpacked_pos..];169 const unpacked_bytes = self.to_read.items[unpacked_pos..];
168 if (unpacked_size) |s| {170 if (unpacked_size) |s| {
169 if (s != unpacked_bytes.len)171 if (s != unpacked_bytes.len)
170 return error.CorruptInput;172 return error.CorruptInput;
...@@ -205,113 +207,5 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -205,113 +207,5 @@ pub fn Decoder(comptime ReaderType: type) type {
205207
206 self.block_count += 1;208 self.block_count += 1;
207 }209 }
208
209 fn readLzma2Chunk(self: *Self, packed_reader: anytype) Error!bool {
210 const status = try packed_reader.readByte();
211 switch (status) {
212 0 => {
213 try self.accum.reset(self.allocator);
214 return false;
215 },
216 1, 2 => {
217 if (status == 1)
218 try self.accum.reset(self.allocator);
219
220 const size = try packed_reader.readIntBig(u16) + 1;
221 try self.accum.ensureUnusedCapacity(self.allocator, size);
222
223 var i: usize = 0;
224 while (i < size) : (i += 1)
225 self.accum.appendAssumeCapacity(try packed_reader.readByte());
226
227 return true;
228 },
229 else => {
230 if (status & 0x80 == 0)
231 return error.CorruptInput;
232
233 const Reset = struct {
234 dict: bool,
235 state: bool,
236 props: bool,
237 };
238
239 const reset = switch ((status >> 5) & 0x3) {
240 0 => Reset{
241 .dict = false,
242 .state = false,
243 .props = false,
244 },
245 1 => Reset{
246 .dict = false,
247 .state = true,
248 .props = false,
249 },
250 2 => Reset{
251 .dict = false,
252 .state = true,
253 .props = true,
254 },
255 3 => Reset{
256 .dict = true,
257 .state = true,
258 .props = true,
259 },
260 else => unreachable,
261 };
262
263 const unpacked_size = blk: {
264 var tmp: u64 = status & 0x1F;
265 tmp <<= 16;
266 tmp |= try packed_reader.readIntBig(u16);
267 break :blk tmp + 1;
268 };
269
270 const packed_size = blk: {
271 const tmp: u17 = try packed_reader.readIntBig(u16);
272 break :blk tmp + 1;
273 };
274
275 if (reset.dict)
276 try self.accum.reset(self.allocator);
277
278 if (reset.state) {
279 var new_props = self.lzma_state.lzma_props;
280
281 if (reset.props) {
282 var props = try packed_reader.readByte();
283 if (props >= 225)
284 return error.CorruptInput;
285
286 const lc = @intCast(u4, props % 9);
287 props /= 9;
288 const lp = @intCast(u3, props % 5);
289 props /= 5;
290 const pb = @intCast(u3, props);
291
292 if (lc + lp > 4)
293 return error.CorruptInput;
294
295 new_props = .{ .lc = lc, .lp = lp, .pb = pb };
296 }
297
298 try self.lzma_state.reset_state(self.allocator, new_props);
299 }
300
301 self.lzma_state.unpacked_size = unpacked_size + self.accum.len();
302
303 const buffer = try self.allocator.alloc(u8, packed_size);
304 defer self.allocator.free(buffer);
305
306 for (buffer) |*b|
307 b.* = try packed_reader.readByte();
308
309 var rangecoder = try lzma.RangeDecoder.init(buffer);
310 try self.lzma_state.process(self.allocator, &self.accum, &rangecoder);
311
312 return true;
313 },
314 }
315 }
316 };210 };
317}211}
lib/std/compress/xz/lzma.zig deleted-658
...@@ -1,658 +0,0 @@
1// Ported from https://github.com/gendx/lzma-rs
2
3const std = @import("../../std.zig");
4const assert = std.debug.assert;
5const Allocator = std.mem.Allocator;
6const ArrayListUnmanaged = std.ArrayListUnmanaged;
7
8const LzmaProperties = struct {
9 lc: u4,
10 lp: u3,
11 pb: u3,
12
13 fn validate(self: LzmaProperties) void {
14 assert(self.lc <= 8);
15 assert(self.lp <= 4);
16 assert(self.pb <= 4);
17 }
18};
19
20pub const DecoderState = struct {
21 lzma_props: LzmaProperties,
22 unpacked_size: ?u64,
23 literal_probs: Vec2D(u16),
24 pos_slot_decoder: [4]BitTree,
25 align_decoder: BitTree,
26 pos_decoders: [115]u16,
27 is_match: [192]u16,
28 is_rep: [12]u16,
29 is_rep_g0: [12]u16,
30 is_rep_g1: [12]u16,
31 is_rep_g2: [12]u16,
32 is_rep_0long: [192]u16,
33 state: usize,
34 rep: [4]usize,
35 len_decoder: LenDecoder,
36 rep_len_decoder: LenDecoder,
37
38 pub fn init(allocator: Allocator) !DecoderState {
39 return .{
40 .lzma_props = LzmaProperties{ .lc = 0, .lp = 0, .pb = 0 },
41 .unpacked_size = null,
42 .literal_probs = try Vec2D(u16).init(allocator, 0x400, 1, 0x300),
43 .pos_slot_decoder = .{
44 try BitTree.init(allocator, 6),
45 try BitTree.init(allocator, 6),
46 try BitTree.init(allocator, 6),
47 try BitTree.init(allocator, 6),
48 },
49 .align_decoder = try BitTree.init(allocator, 4),
50 .pos_decoders = .{0x400} ** 115,
51 .is_match = .{0x400} ** 192,
52 .is_rep = .{0x400} ** 12,
53 .is_rep_g0 = .{0x400} ** 12,
54 .is_rep_g1 = .{0x400} ** 12,
55 .is_rep_g2 = .{0x400} ** 12,
56 .is_rep_0long = .{0x400} ** 192,
57 .state = 0,
58 .rep = .{0} ** 4,
59 .len_decoder = try LenDecoder.init(allocator),
60 .rep_len_decoder = try LenDecoder.init(allocator),
61 };
62 }
63
64 pub fn deinit(self: *DecoderState, allocator: Allocator) void {
65 self.literal_probs.deinit(allocator);
66 for (self.pos_slot_decoder) |*t| t.deinit(allocator);
67 self.align_decoder.deinit(allocator);
68 self.len_decoder.deinit(allocator);
69 self.rep_len_decoder.deinit(allocator);
70 }
71
72 pub fn reset_state(self: *DecoderState, allocator: Allocator, new_props: LzmaProperties) !void {
73 new_props.validate();
74 if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
75 self.literal_probs.fill(0x400);
76 } else {
77 self.literal_probs.deinit(allocator);
78 self.literal_probs = try Vec2D(u16).init(allocator, 0x400, @as(usize, 1) << (new_props.lc + new_props.lp), 0x300);
79 }
80
81 self.lzma_props = new_props;
82 for (self.pos_slot_decoder) |*t| t.reset();
83 self.align_decoder.reset();
84 self.pos_decoders = .{0x400} ** 115;
85 self.is_match = .{0x400} ** 192;
86 self.is_rep = .{0x400} ** 12;
87 self.is_rep_g0 = .{0x400} ** 12;
88 self.is_rep_g1 = .{0x400} ** 12;
89 self.is_rep_g2 = .{0x400} ** 12;
90 self.is_rep_0long = .{0x400} ** 192;
91 self.state = 0;
92 self.rep = .{0} ** 4;
93 self.len_decoder.reset();
94 self.rep_len_decoder.reset();
95 }
96
97 fn processNextInner(
98 self: *DecoderState,
99 allocator: Allocator,
100 output: *LzAccumBuffer,
101 rangecoder: *RangeDecoder,
102 update: bool,
103 ) !ProcessingStatus {
104 const pos_state = output.len() & ((@as(usize, 1) << self.lzma_props.pb) - 1);
105
106 if (!try rangecoder.decodeBit(
107 &self.is_match[(self.state << 4) + pos_state],
108 update,
109 )) {
110 const byte: u8 = try self.decodeLiteral(output, rangecoder, update);
111
112 if (update) {
113 try output.appendLiteral(allocator, byte);
114
115 self.state = if (self.state < 4)
116 0
117 else if (self.state < 10)
118 self.state - 3
119 else
120 self.state - 6;
121 }
122 return .continue_;
123 }
124
125 var len: usize = undefined;
126 if (try rangecoder.decodeBit(&self.is_rep[self.state], update)) {
127 if (!try rangecoder.decodeBit(&self.is_rep_g0[self.state], update)) {
128 if (!try rangecoder.decodeBit(
129 &self.is_rep_0long[(self.state << 4) + pos_state],
130 update,
131 )) {
132 if (update) {
133 self.state = if (self.state < 7) 9 else 11;
134 const dist = self.rep[0] + 1;
135 try output.appendLz(allocator, 1, dist);
136 }
137 return .continue_;
138 }
139 } else {
140 const idx: usize = if (!try rangecoder.decodeBit(&self.is_rep_g1[self.state], update))
141 1
142 else if (!try rangecoder.decodeBit(&self.is_rep_g2[self.state], update))
143 2
144 else
145 3;
146 if (update) {
147 const dist = self.rep[idx];
148 var i = idx;
149 while (i > 0) : (i -= 1) {
150 self.rep[i] = self.rep[i - 1];
151 }
152 self.rep[0] = dist;
153 }
154 }
155
156 len = try self.rep_len_decoder.decode(rangecoder, pos_state, update);
157
158 if (update) {
159 self.state = if (self.state < 7) 8 else 11;
160 }
161 } else {
162 if (update) {
163 self.rep[3] = self.rep[2];
164 self.rep[2] = self.rep[1];
165 self.rep[1] = self.rep[0];
166 }
167
168 len = try self.len_decoder.decode(rangecoder, pos_state, update);
169
170 if (update) {
171 self.state = if (self.state < 7) 7 else 10;
172 }
173
174 const rep_0 = try self.decodeDistance(rangecoder, len, update);
175
176 if (update) {
177 self.rep[0] = rep_0;
178 if (self.rep[0] == 0xFFFF_FFFF) {
179 if (rangecoder.isFinished()) {
180 return .finished;
181 }
182 return error.CorruptInput;
183 }
184 }
185 }
186
187 if (update) {
188 len += 2;
189
190 const dist = self.rep[0] + 1;
191 try output.appendLz(allocator, len, dist);
192 }
193
194 return .continue_;
195 }
196
197 fn processNext(
198 self: *DecoderState,
199 allocator: Allocator,
200 output: *LzAccumBuffer,
201 rangecoder: *RangeDecoder,
202 ) !ProcessingStatus {
203 return self.processNextInner(allocator, output, rangecoder, true);
204 }
205
206 pub fn process(
207 self: *DecoderState,
208 allocator: Allocator,
209 output: *LzAccumBuffer,
210 rangecoder: *RangeDecoder,
211 ) !void {
212 while (true) {
213 if (self.unpacked_size) |unpacked_size| {
214 if (output.len() >= unpacked_size) {
215 break;
216 }
217 } else if (rangecoder.isFinished()) {
218 break;
219 }
220
221 if (try self.processNext(allocator, output, rangecoder) == .finished) {
222 break;
223 }
224 }
225
226 if (self.unpacked_size) |len| {
227 if (len != output.len()) {
228 return error.CorruptInput;
229 }
230 }
231 }
232
233 fn decodeLiteral(
234 self: *DecoderState,
235 output: *LzAccumBuffer,
236 rangecoder: *RangeDecoder,
237 update: bool,
238 ) !u8 {
239 const def_prev_byte = 0;
240 const prev_byte = @as(usize, output.lastOr(def_prev_byte));
241
242 var result: usize = 1;
243 const lit_state = ((output.len() & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
244 (prev_byte >> (8 - self.lzma_props.lc));
245 const probs = try self.literal_probs.get(lit_state);
246
247 if (self.state >= 7) {
248 var match_byte = @as(usize, try output.lastN(self.rep[0] + 1));
249
250 while (result < 0x100) {
251 const match_bit = (match_byte >> 7) & 1;
252 match_byte <<= 1;
253 const bit = @boolToInt(try rangecoder.decodeBit(
254 &probs[((@as(usize, 1) + match_bit) << 8) + result],
255 update,
256 ));
257 result = (result << 1) ^ bit;
258 if (match_bit != bit) {
259 break;
260 }
261 }
262 }
263
264 while (result < 0x100) {
265 result = (result << 1) ^ @boolToInt(try rangecoder.decodeBit(&probs[result], update));
266 }
267
268 return @truncate(u8, result - 0x100);
269 }
270
271 fn decodeDistance(
272 self: *DecoderState,
273 rangecoder: *RangeDecoder,
274 length: usize,
275 update: bool,
276 ) !usize {
277 const len_state = if (length > 3) 3 else length;
278
279 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(rangecoder, update));
280 if (pos_slot < 4)
281 return pos_slot;
282
283 const num_direct_bits = @intCast(u5, (pos_slot >> 1) - 1);
284 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
285
286 if (pos_slot < 14) {
287 result += try rangecoder.parseReverseBitTree(
288 num_direct_bits,
289 &self.pos_decoders,
290 result - pos_slot,
291 update,
292 );
293 } else {
294 result += @as(usize, try rangecoder.get(num_direct_bits - 4)) << 4;
295 result += try self.align_decoder.parseReverse(rangecoder, update);
296 }
297
298 return result;
299 }
300};
301
302const ProcessingStatus = enum {
303 continue_,
304 finished,
305};
306
307pub const LzAccumBuffer = struct {
308 to_read: ArrayListUnmanaged(u8) = .{},
309 buf: ArrayListUnmanaged(u8) = .{},
310
311 pub fn deinit(self: *LzAccumBuffer, allocator: Allocator) void {
312 self.to_read.deinit(allocator);
313 self.buf.deinit(allocator);
314 }
315
316 pub fn read(self: *LzAccumBuffer, output: []u8) usize {
317 const input = self.to_read.items;
318 const n = std.math.min(input.len, output.len);
319 std.mem.copy(u8, output[0..n], input[0..n]);
320 std.mem.copy(u8, input, input[n..]);
321 self.to_read.shrinkRetainingCapacity(input.len - n);
322 return n;
323 }
324
325 pub fn ensureUnusedCapacity(
326 self: *LzAccumBuffer,
327 allocator: Allocator,
328 additional_count: usize,
329 ) !void {
330 try self.buf.ensureUnusedCapacity(allocator, additional_count);
331 }
332
333 pub fn appendAssumeCapacity(self: *LzAccumBuffer, byte: u8) void {
334 self.buf.appendAssumeCapacity(byte);
335 }
336
337 pub fn reset(self: *LzAccumBuffer, allocator: Allocator) !void {
338 try self.to_read.appendSlice(allocator, self.buf.items);
339 self.buf.clearRetainingCapacity();
340 }
341
342 pub fn len(self: *const LzAccumBuffer) usize {
343 return self.buf.items.len;
344 }
345
346 pub fn lastOr(self: *const LzAccumBuffer, lit: u8) u8 {
347 const buf_len = self.buf.items.len;
348 return if (buf_len == 0)
349 lit
350 else
351 self.buf.items[buf_len - 1];
352 }
353
354 pub fn lastN(self: *const LzAccumBuffer, dist: usize) !u8 {
355 const buf_len = self.buf.items.len;
356 if (dist > buf_len) {
357 return error.CorruptInput;
358 }
359
360 return self.buf.items[buf_len - dist];
361 }
362
363 pub fn appendLiteral(self: *LzAccumBuffer, allocator: Allocator, lit: u8) !void {
364 try self.buf.append(allocator, lit);
365 }
366
367 pub fn appendLz(self: *LzAccumBuffer, allocator: Allocator, length: usize, dist: usize) !void {
368 const buf_len = self.buf.items.len;
369 if (dist > buf_len) {
370 return error.CorruptInput;
371 }
372
373 var offset = buf_len - dist;
374 var i: usize = 0;
375 while (i < length) : (i += 1) {
376 const x = self.buf.items[offset];
377 try self.buf.append(allocator, x);
378 offset += 1;
379 }
380 }
381};
382
383pub const RangeDecoder = struct {
384 stream: std.io.FixedBufferStream([]const u8),
385 range: u32,
386 code: u32,
387
388 pub fn init(buffer: []const u8) !RangeDecoder {
389 var dec = RangeDecoder{
390 .stream = std.io.fixedBufferStream(buffer),
391 .range = 0xFFFF_FFFF,
392 .code = 0,
393 };
394 const reader = dec.stream.reader();
395 _ = try reader.readByte();
396 dec.code = try reader.readIntBig(u32);
397 return dec;
398 }
399
400 pub fn fromParts(
401 buffer: []const u8,
402 range: u32,
403 code: u32,
404 ) RangeDecoder {
405 return .{
406 .stream = std.io.fixedBufferStream(buffer),
407 .range = range,
408 .code = code,
409 };
410 }
411
412 pub fn set(self: *RangeDecoder, range: u32, code: u32) void {
413 self.range = range;
414 self.code = code;
415 }
416
417 pub fn readInto(self: *RangeDecoder, dest: []u8) !usize {
418 return self.stream.read(dest);
419 }
420
421 pub inline fn isFinished(self: *const RangeDecoder) bool {
422 return self.code == 0 and self.isEof();
423 }
424
425 pub inline fn isEof(self: *const RangeDecoder) bool {
426 return self.stream.pos == self.stream.buffer.len;
427 }
428
429 inline fn normalize(self: *RangeDecoder) !void {
430 if (self.range < 0x0100_0000) {
431 self.range <<= 8;
432 self.code = (self.code << 8) ^ @as(u32, try self.stream.reader().readByte());
433 }
434 }
435
436 inline fn getBit(self: *RangeDecoder) !bool {
437 self.range >>= 1;
438
439 const bit = self.code >= self.range;
440 if (bit)
441 self.code -= self.range;
442
443 try self.normalize();
444 return bit;
445 }
446
447 fn get(self: *RangeDecoder, count: usize) !u32 {
448 var result: u32 = 0;
449 var i: usize = 0;
450 while (i < count) : (i += 1)
451 result = (result << 1) ^ @boolToInt(try self.getBit());
452 return result;
453 }
454
455 pub inline fn decodeBit(self: *RangeDecoder, prob: *u16, update: bool) !bool {
456 const bound = (self.range >> 11) * prob.*;
457
458 if (self.code < bound) {
459 if (update)
460 prob.* += (0x800 - prob.*) >> 5;
461 self.range = bound;
462
463 try self.normalize();
464 return false;
465 } else {
466 if (update)
467 prob.* -= prob.* >> 5;
468 self.code -= bound;
469 self.range -= bound;
470
471 try self.normalize();
472 return true;
473 }
474 }
475
476 fn parseBitTree(
477 self: *RangeDecoder,
478 num_bits: u5,
479 probs: []u16,
480 update: bool,
481 ) !u32 {
482 var tmp: u32 = 1;
483 var i: u5 = 0;
484 while (i < num_bits) : (i += 1) {
485 const bit = try self.decodeBit(&probs[tmp], update);
486 tmp = (tmp << 1) ^ @boolToInt(bit);
487 }
488 return tmp - (@as(u32, 1) << num_bits);
489 }
490
491 pub fn parseReverseBitTree(
492 self: *RangeDecoder,
493 num_bits: u5,
494 probs: []u16,
495 offset: usize,
496 update: bool,
497 ) !u32 {
498 var result: u32 = 0;
499 var tmp: usize = 1;
500 var i: u5 = 0;
501 while (i < num_bits) : (i += 1) {
502 const bit = @boolToInt(try self.decodeBit(&probs[offset + tmp], update));
503 tmp = (tmp << 1) ^ bit;
504 result ^= @as(u32, bit) << i;
505 }
506 return result;
507 }
508};
509
510fn Vec2D(comptime T: type) type {
511 return struct {
512 data: []T,
513 cols: usize,
514
515 const Self = @This();
516
517 pub fn init(allocator: Allocator, data: T, rows: usize, cols: usize) !Self {
518 const len = try std.math.mul(usize, rows, cols);
519 var vec2d = Self{
520 .data = try allocator.alloc(T, len),
521 .cols = cols,
522 };
523 vec2d.fill(data);
524 return vec2d;
525 }
526
527 pub fn deinit(self: *Self, allocator: Allocator) void {
528 allocator.free(self.data);
529 }
530
531 pub fn fill(self: *Self, value: T) void {
532 std.mem.set(T, self.data, value);
533 }
534
535 pub fn get(self: *Self, row: usize) ![]T {
536 const start_row = try std.math.mul(usize, row, self.cols);
537 return self.data[start_row .. start_row + self.cols];
538 }
539 };
540}
541
542const BitTree = struct {
543 num_bits: u5,
544 probs: ArrayListUnmanaged(u16),
545
546 pub fn init(allocator: Allocator, num_bits: u5) !BitTree {
547 var probs_len = @as(usize, 1) << num_bits;
548 var probs = try ArrayListUnmanaged(u16).initCapacity(allocator, probs_len);
549 while (probs_len > 0) : (probs_len -= 1)
550 probs.appendAssumeCapacity(0x400);
551 return .{ .num_bits = num_bits, .probs = probs };
552 }
553
554 pub fn deinit(self: *BitTree, allocator: Allocator) void {
555 self.probs.deinit(allocator);
556 }
557
558 pub fn parse(
559 self: *BitTree,
560 rangecoder: *RangeDecoder,
561 update: bool,
562 ) !u32 {
563 return rangecoder.parseBitTree(self.num_bits, self.probs.items, update);
564 }
565
566 pub fn parseReverse(
567 self: *BitTree,
568 rangecoder: *RangeDecoder,
569 update: bool,
570 ) !u32 {
571 return rangecoder.parseReverseBitTree(self.num_bits, self.probs.items, 0, update);
572 }
573
574 pub fn reset(self: *BitTree) void {
575 std.mem.set(u16, self.probs.items, 0x400);
576 }
577};
578
579const LenDecoder = struct {
580 choice: u16,
581 choice2: u16,
582 low_coder: [16]BitTree,
583 mid_coder: [16]BitTree,
584 high_coder: BitTree,
585
586 pub fn init(allocator: Allocator) !LenDecoder {
587 return .{
588 .choice = 0x400,
589 .choice2 = 0x400,
590 .low_coder = .{
591 try BitTree.init(allocator, 3),
592 try BitTree.init(allocator, 3),
593 try BitTree.init(allocator, 3),
594 try BitTree.init(allocator, 3),
595 try BitTree.init(allocator, 3),
596 try BitTree.init(allocator, 3),
597 try BitTree.init(allocator, 3),
598 try BitTree.init(allocator, 3),
599 try BitTree.init(allocator, 3),
600 try BitTree.init(allocator, 3),
601 try BitTree.init(allocator, 3),
602 try BitTree.init(allocator, 3),
603 try BitTree.init(allocator, 3),
604 try BitTree.init(allocator, 3),
605 try BitTree.init(allocator, 3),
606 try BitTree.init(allocator, 3),
607 },
608 .mid_coder = .{
609 try BitTree.init(allocator, 3),
610 try BitTree.init(allocator, 3),
611 try BitTree.init(allocator, 3),
612 try BitTree.init(allocator, 3),
613 try BitTree.init(allocator, 3),
614 try BitTree.init(allocator, 3),
615 try BitTree.init(allocator, 3),
616 try BitTree.init(allocator, 3),
617 try BitTree.init(allocator, 3),
618 try BitTree.init(allocator, 3),
619 try BitTree.init(allocator, 3),
620 try BitTree.init(allocator, 3),
621 try BitTree.init(allocator, 3),
622 try BitTree.init(allocator, 3),
623 try BitTree.init(allocator, 3),
624 try BitTree.init(allocator, 3),
625 },
626 .high_coder = try BitTree.init(allocator, 8),
627 };
628 }
629
630 pub fn deinit(self: *LenDecoder, allocator: Allocator) void {
631 for (self.low_coder) |*t| t.deinit(allocator);
632 for (self.mid_coder) |*t| t.deinit(allocator);
633 self.high_coder.deinit(allocator);
634 }
635
636 pub fn decode(
637 self: *LenDecoder,
638 rangecoder: *RangeDecoder,
639 pos_state: usize,
640 update: bool,
641 ) !usize {
642 if (!try rangecoder.decodeBit(&self.choice, update)) {
643 return @as(usize, try self.low_coder[pos_state].parse(rangecoder, update));
644 } else if (!try rangecoder.decodeBit(&self.choice2, update)) {
645 return @as(usize, try self.mid_coder[pos_state].parse(rangecoder, update)) + 8;
646 } else {
647 return @as(usize, try self.high_coder.parse(rangecoder, update)) + 16;
648 }
649 }
650
651 pub fn reset(self: *LenDecoder) void {
652 self.choice = 0x400;
653 self.choice2 = 0x400;
654 for (self.low_coder) |*t| t.reset();
655 for (self.mid_coder) |*t| t.reset();
656 self.high_coder.reset();
657 }
658};
lib/std/compress/xz/test.zig+20
...@@ -78,3 +78,23 @@ test "unsupported" {...@@ -78,3 +78,23 @@ test "unsupported" {
78 );78 );
79 }79 }
80}80}
81
82fn testDontPanic(data: []const u8) !void {
83 const buf = decompress(data) catch |err| switch (err) {
84 error.OutOfMemory => |e| return e,
85 else => return,
86 };
87 defer testing.allocator.free(buf);
88}
89
90test "size fields: integer overflow avoidance" {
91 // These cases were found via fuzz testing and each previously caused
92 // an integer overflow when decoding. We just want to ensure they no longer
93 // cause a panic
94 const header_size_overflow = "\xfd7zXZ\x00\x00\x01i\"\xde6z";
95 try testDontPanic(header_size_overflow);
96 const lzma2_chunk_size_overflow = "\xfd7zXZ\x00\x00\x01i\"\xde6\x02\x00!\x01\x08\x00\x00\x00\xd8\x0f#\x13\x01\xff\xff";
97 try testDontPanic(lzma2_chunk_size_overflow);
98 const backward_size_overflow = "\xfd7zXZ\x00\x00\x01i\"\xde6\x00\x00\x00\x00\x1c\xdfD!\x90B\x99\r\x01\x00\x00\xff\xff\x10\x00\x00\x00\x01DD\xff\xff\xff\x01";
99 try testDontPanic(backward_size_overflow);
100}