authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-22 14:03:44-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-26 21:00:58-07:00
logea0ce7afb59d7c7ed33f707f3fea3e0babd785b6
treeb77bdcd8b4ad1b484d15e84ed4123a0382b4a024
parent9399fcddce0bcd8e987b053f3946aa1b0ff2ef0a

std.compress: flatten lzma and lzma2 namespaces

7 files changed, 981 insertions(+), 1098 deletions(-)

lib/std/compress/lzma.zig+715-11
...@@ -2,8 +2,713 @@ const std = @import("../std.zig");...@@ -2,8 +2,713 @@ const std = @import("../std.zig");
2const math = std.math;2const math = std.math;
3const mem = std.mem;3const mem = std.mem;
4const Allocator = std.mem.Allocator;4const Allocator = std.mem.Allocator;
5const assert = std.debug.assert;
6const ArrayList = std.ArrayList;
57
6pub const decode = @import("lzma/decode.zig");8pub const RangeDecoder = struct {
9 range: u32,
10 code: u32,
11
12 pub fn init(reader: anytype) !RangeDecoder {
13 const reserved = try reader.readByte();
14 if (reserved != 0) {
15 return error.CorruptInput;
16 }
17 return RangeDecoder{
18 .range = 0xFFFF_FFFF,
19 .code = try reader.readInt(u32, .big),
20 };
21 }
22
23 pub fn fromParts(
24 range: u32,
25 code: u32,
26 ) RangeDecoder {
27 return .{
28 .range = range,
29 .code = code,
30 };
31 }
32
33 pub fn set(self: *RangeDecoder, range: u32, code: u32) void {
34 self.range = range;
35 self.code = code;
36 }
37
38 pub inline fn isFinished(self: RangeDecoder) bool {
39 return self.code == 0;
40 }
41
42 inline fn normalize(self: *RangeDecoder, reader: anytype) !void {
43 if (self.range < 0x0100_0000) {
44 self.range <<= 8;
45 self.code = (self.code << 8) ^ @as(u32, try reader.readByte());
46 }
47 }
48
49 inline fn getBit(self: *RangeDecoder, reader: anytype) !bool {
50 self.range >>= 1;
51
52 const bit = self.code >= self.range;
53 if (bit)
54 self.code -= self.range;
55
56 try self.normalize(reader);
57 return bit;
58 }
59
60 pub fn get(self: *RangeDecoder, reader: anytype, count: usize) !u32 {
61 var result: u32 = 0;
62 var i: usize = 0;
63 while (i < count) : (i += 1)
64 result = (result << 1) ^ @intFromBool(try self.getBit(reader));
65 return result;
66 }
67
68 pub inline fn decodeBit(self: *RangeDecoder, reader: anytype, prob: *u16, update: bool) !bool {
69 const bound = (self.range >> 11) * prob.*;
70
71 if (self.code < bound) {
72 if (update)
73 prob.* += (0x800 - prob.*) >> 5;
74 self.range = bound;
75
76 try self.normalize(reader);
77 return false;
78 } else {
79 if (update)
80 prob.* -= prob.* >> 5;
81 self.code -= bound;
82 self.range -= bound;
83
84 try self.normalize(reader);
85 return true;
86 }
87 }
88
89 fn parseBitTree(
90 self: *RangeDecoder,
91 reader: anytype,
92 num_bits: u5,
93 probs: []u16,
94 update: bool,
95 ) !u32 {
96 var tmp: u32 = 1;
97 var i: @TypeOf(num_bits) = 0;
98 while (i < num_bits) : (i += 1) {
99 const bit = try self.decodeBit(reader, &probs[tmp], update);
100 tmp = (tmp << 1) ^ @intFromBool(bit);
101 }
102 return tmp - (@as(u32, 1) << num_bits);
103 }
104
105 pub fn parseReverseBitTree(
106 self: *RangeDecoder,
107 reader: anytype,
108 num_bits: u5,
109 probs: []u16,
110 offset: usize,
111 update: bool,
112 ) !u32 {
113 var result: u32 = 0;
114 var tmp: usize = 1;
115 var i: @TypeOf(num_bits) = 0;
116 while (i < num_bits) : (i += 1) {
117 const bit = @intFromBool(try self.decodeBit(reader, &probs[offset + tmp], update));
118 tmp = (tmp << 1) ^ bit;
119 result ^= @as(u32, bit) << i;
120 }
121 return result;
122 }
123};
124
125pub const Decode = struct {
126 lzma_props: Properties,
127 unpacked_size: ?u64,
128 literal_probs: Vec2d,
129 pos_slot_decoder: [4]BitTree(6),
130 align_decoder: BitTree(4),
131 pos_decoders: [115]u16,
132 is_match: [192]u16,
133 is_rep: [12]u16,
134 is_rep_g0: [12]u16,
135 is_rep_g1: [12]u16,
136 is_rep_g2: [12]u16,
137 is_rep_0long: [192]u16,
138 state: usize,
139 rep: [4]usize,
140 len_decoder: LenDecoder,
141 rep_len_decoder: LenDecoder,
142
143 pub fn init(
144 allocator: Allocator,
145 lzma_props: Properties,
146 unpacked_size: ?u64,
147 ) !Decode {
148 return .{
149 .lzma_props = lzma_props,
150 .unpacked_size = unpacked_size,
151 .literal_probs = try Vec2d.init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }),
152 .pos_slot_decoder = @splat(.{}),
153 .align_decoder = .{},
154 .pos_decoders = @splat(0x400),
155 .is_match = @splat(0x400),
156 .is_rep = @splat(0x400),
157 .is_rep_g0 = @splat(0x400),
158 .is_rep_g1 = @splat(0x400),
159 .is_rep_g2 = @splat(0x400),
160 .is_rep_0long = @splat(0x400),
161 .state = 0,
162 .rep = @splat(0),
163 .len_decoder = .{},
164 .rep_len_decoder = .{},
165 };
166 }
167
168 pub fn deinit(self: *Decode, allocator: Allocator) void {
169 self.literal_probs.deinit(allocator);
170 self.* = undefined;
171 }
172
173 pub fn resetState(self: *Decode, allocator: Allocator, new_props: Properties) !void {
174 new_props.validate();
175 if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
176 self.literal_probs.fill(0x400);
177 } else {
178 self.literal_probs.deinit(allocator);
179 self.literal_probs = try Vec2d.init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
180 }
181
182 self.lzma_props = new_props;
183 for (&self.pos_slot_decoder) |*t| t.reset();
184 self.align_decoder.reset();
185 self.pos_decoders = @splat(0x400);
186 self.is_match = @splat(0x400);
187 self.is_rep = @splat(0x400);
188 self.is_rep_g0 = @splat(0x400);
189 self.is_rep_g1 = @splat(0x400);
190 self.is_rep_g2 = @splat(0x400);
191 self.is_rep_0long = @splat(0x400);
192 self.state = 0;
193 self.rep = @splat(0);
194 self.len_decoder.reset();
195 self.rep_len_decoder.reset();
196 }
197
198 fn processNextInner(
199 self: *Decode,
200 allocator: Allocator,
201 reader: anytype,
202 writer: anytype,
203 buffer: anytype,
204 decoder: *RangeDecoder,
205 update: bool,
206 ) !ProcessingStatus {
207 const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1);
208
209 if (!try decoder.decodeBit(
210 reader,
211 &self.is_match[(self.state << 4) + pos_state],
212 update,
213 )) {
214 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);
215
216 if (update) {
217 try buffer.appendLiteral(allocator, byte, writer);
218
219 self.state = if (self.state < 4)
220 0
221 else if (self.state < 10)
222 self.state - 3
223 else
224 self.state - 6;
225 }
226 return .continue_;
227 }
228
229 var len: usize = undefined;
230 if (try decoder.decodeBit(reader, &self.is_rep[self.state], update)) {
231 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], update)) {
232 if (!try decoder.decodeBit(
233 reader,
234 &self.is_rep_0long[(self.state << 4) + pos_state],
235 update,
236 )) {
237 if (update) {
238 self.state = if (self.state < 7) 9 else 11;
239 const dist = self.rep[0] + 1;
240 try buffer.appendLz(allocator, 1, dist, writer);
241 }
242 return .continue_;
243 }
244 } else {
245 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))
246 1
247 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], update))
248 2
249 else
250 3;
251 if (update) {
252 const dist = self.rep[idx];
253 var i = idx;
254 while (i > 0) : (i -= 1) {
255 self.rep[i] = self.rep[i - 1];
256 }
257 self.rep[0] = dist;
258 }
259 }
260
261 len = try self.rep_len_decoder.decode(reader, decoder, pos_state, update);
262
263 if (update) {
264 self.state = if (self.state < 7) 8 else 11;
265 }
266 } else {
267 if (update) {
268 self.rep[3] = self.rep[2];
269 self.rep[2] = self.rep[1];
270 self.rep[1] = self.rep[0];
271 }
272
273 len = try self.len_decoder.decode(reader, decoder, pos_state, update);
274
275 if (update) {
276 self.state = if (self.state < 7) 7 else 10;
277 }
278
279 const rep_0 = try self.decodeDistance(reader, decoder, len, update);
280
281 if (update) {
282 self.rep[0] = rep_0;
283 if (self.rep[0] == 0xFFFF_FFFF) {
284 if (decoder.isFinished()) {
285 return .finished;
286 }
287 return error.CorruptInput;
288 }
289 }
290 }
291
292 if (update) {
293 len += 2;
294
295 const dist = self.rep[0] + 1;
296 try buffer.appendLz(allocator, len, dist, writer);
297 }
298
299 return .continue_;
300 }
301
302 fn processNext(
303 self: *Decode,
304 allocator: Allocator,
305 reader: anytype,
306 writer: anytype,
307 buffer: anytype,
308 decoder: *RangeDecoder,
309 ) !ProcessingStatus {
310 return self.processNextInner(allocator, reader, writer, buffer, decoder, true);
311 }
312
313 pub fn process(
314 self: *Decode,
315 allocator: Allocator,
316 reader: anytype,
317 writer: anytype,
318 buffer: anytype,
319 decoder: *RangeDecoder,
320 ) !ProcessingStatus {
321 process_next: {
322 if (self.unpacked_size) |unpacked_size| {
323 if (buffer.len >= unpacked_size) {
324 break :process_next;
325 }
326 } else if (decoder.isFinished()) {
327 break :process_next;
328 }
329
330 switch (try self.processNext(allocator, reader, writer, buffer, decoder)) {
331 .continue_ => return .continue_,
332 .finished => break :process_next,
333 }
334 }
335
336 if (self.unpacked_size) |unpacked_size| {
337 if (buffer.len != unpacked_size) {
338 return error.CorruptInput;
339 }
340 }
341
342 return .finished;
343 }
344
345 fn decodeLiteral(
346 self: *Decode,
347 reader: anytype,
348 buffer: anytype,
349 decoder: *RangeDecoder,
350 update: bool,
351 ) !u8 {
352 const def_prev_byte = 0;
353 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
354
355 var result: usize = 1;
356 const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
357 (prev_byte >> (8 - self.lzma_props.lc));
358 const probs = try self.literal_probs.getMut(lit_state);
359
360 if (self.state >= 7) {
361 var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
362
363 while (result < 0x100) {
364 const match_bit = (match_byte >> 7) & 1;
365 match_byte <<= 1;
366 const bit = @intFromBool(try decoder.decodeBit(
367 reader,
368 &probs[((@as(usize, 1) + match_bit) << 8) + result],
369 update,
370 ));
371 result = (result << 1) ^ bit;
372 if (match_bit != bit) {
373 break;
374 }
375 }
376 }
377
378 while (result < 0x100) {
379 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result], update));
380 }
381
382 return @as(u8, @truncate(result - 0x100));
383 }
384
385 fn decodeDistance(
386 self: *Decode,
387 reader: anytype,
388 decoder: *RangeDecoder,
389 length: usize,
390 update: bool,
391 ) !usize {
392 const len_state = if (length > 3) 3 else length;
393
394 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder, update));
395 if (pos_slot < 4)
396 return pos_slot;
397
398 const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
399 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
400
401 if (pos_slot < 14) {
402 result += try decoder.parseReverseBitTree(
403 reader,
404 num_direct_bits,
405 &self.pos_decoders,
406 result - pos_slot,
407 update,
408 );
409 } else {
410 result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4;
411 result += try self.align_decoder.parseReverse(reader, decoder, update);
412 }
413
414 return result;
415 }
416
417 /// A circular buffer for LZ sequences
418 pub const LzCircularBuffer = struct {
419 /// Circular buffer
420 buf: ArrayList(u8),
421
422 /// Length of the buffer
423 dict_size: usize,
424
425 /// Buffer memory limit
426 memlimit: usize,
427
428 /// Current position
429 cursor: usize,
430
431 /// Total number of bytes sent through the buffer
432 len: usize,
433
434 const Self = @This();
435
436 pub fn init(dict_size: usize, memlimit: usize) Self {
437 return Self{
438 .buf = .{},
439 .dict_size = dict_size,
440 .memlimit = memlimit,
441 .cursor = 0,
442 .len = 0,
443 };
444 }
445
446 pub fn get(self: Self, index: usize) u8 {
447 return if (0 <= index and index < self.buf.items.len)
448 self.buf.items[index]
449 else
450 0;
451 }
452
453 pub fn set(self: *Self, allocator: Allocator, index: usize, value: u8) !void {
454 if (index >= self.memlimit) {
455 return error.CorruptInput;
456 }
457 try self.buf.ensureTotalCapacity(allocator, index + 1);
458 while (self.buf.items.len < index) {
459 self.buf.appendAssumeCapacity(0);
460 }
461 self.buf.appendAssumeCapacity(value);
462 }
463
464 /// Retrieve the last byte or return a default
465 pub fn lastOr(self: Self, lit: u8) u8 {
466 return if (self.len == 0)
467 lit
468 else
469 self.get((self.dict_size + self.cursor - 1) % self.dict_size);
470 }
471
472 /// Retrieve the n-th last byte
473 pub fn lastN(self: Self, dist: usize) !u8 {
474 if (dist > self.dict_size or dist > self.len) {
475 return error.CorruptInput;
476 }
477
478 const offset = (self.dict_size + self.cursor - dist) % self.dict_size;
479 return self.get(offset);
480 }
481
482 /// Append a literal
483 pub fn appendLiteral(
484 self: *Self,
485 allocator: Allocator,
486 lit: u8,
487 writer: anytype,
488 ) !void {
489 try self.set(allocator, self.cursor, lit);
490 self.cursor += 1;
491 self.len += 1;
492
493 // Flush the circular buffer to the output
494 if (self.cursor == self.dict_size) {
495 try writer.writeAll(self.buf.items);
496 self.cursor = 0;
497 }
498 }
499
500 /// Fetch an LZ sequence (length, distance) from inside the buffer
501 pub fn appendLz(
502 self: *Self,
503 allocator: Allocator,
504 len: usize,
505 dist: usize,
506 writer: anytype,
507 ) !void {
508 if (dist > self.dict_size or dist > self.len) {
509 return error.CorruptInput;
510 }
511
512 var offset = (self.dict_size + self.cursor - dist) % self.dict_size;
513 var i: usize = 0;
514 while (i < len) : (i += 1) {
515 const x = self.get(offset);
516 try self.appendLiteral(allocator, x, writer);
517 offset += 1;
518 if (offset == self.dict_size) {
519 offset = 0;
520 }
521 }
522 }
523
524 pub fn finish(self: *Self, writer: anytype) !void {
525 if (self.cursor > 0) {
526 try writer.writeAll(self.buf.items[0..self.cursor]);
527 self.cursor = 0;
528 }
529 }
530
531 pub fn deinit(self: *Self, allocator: Allocator) void {
532 self.buf.deinit(allocator);
533 self.* = undefined;
534 }
535 };
536
537 pub fn BitTree(comptime num_bits: usize) type {
538 return struct {
539 probs: [1 << num_bits]u16 = @splat(0x400),
540
541 const Self = @This();
542
543 pub fn parse(
544 self: *Self,
545 reader: anytype,
546 decoder: *RangeDecoder,
547 update: bool,
548 ) !u32 {
549 return decoder.parseBitTree(reader, num_bits, &self.probs, update);
550 }
551
552 pub fn parseReverse(
553 self: *Self,
554 reader: anytype,
555 decoder: *RangeDecoder,
556 update: bool,
557 ) !u32 {
558 return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, update);
559 }
560
561 pub fn reset(self: *Self) void {
562 @memset(&self.probs, 0x400);
563 }
564 };
565 }
566
567 pub const LenDecoder = struct {
568 choice: u16 = 0x400,
569 choice2: u16 = 0x400,
570 low_coder: [16]BitTree(3) = @splat(.{}),
571 mid_coder: [16]BitTree(3) = @splat(.{}),
572 high_coder: BitTree(8) = .{},
573
574 pub fn decode(
575 self: *LenDecoder,
576 reader: anytype,
577 decoder: *RangeDecoder,
578 pos_state: usize,
579 update: bool,
580 ) !usize {
581 if (!try decoder.decodeBit(reader, &self.choice, update)) {
582 return @as(usize, try self.low_coder[pos_state].parse(reader, decoder, update));
583 } else if (!try decoder.decodeBit(reader, &self.choice2, update)) {
584 return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder, update)) + 8;
585 } else {
586 return @as(usize, try self.high_coder.parse(reader, decoder, update)) + 16;
587 }
588 }
589
590 pub fn reset(self: *LenDecoder) void {
591 self.choice = 0x400;
592 self.choice2 = 0x400;
593 for (&self.low_coder) |*t| t.reset();
594 for (&self.mid_coder) |*t| t.reset();
595 self.high_coder.reset();
596 }
597 };
598
599 pub const Vec2d = struct {
600 data: []u16,
601 cols: usize,
602
603 const Self = @This();
604
605 pub fn init(allocator: Allocator, value: u16, size: struct { usize, usize }) !Self {
606 const len = try math.mul(usize, size[0], size[1]);
607 const data = try allocator.alloc(u16, len);
608 @memset(data, value);
609 return Self{
610 .data = data,
611 .cols = size[1],
612 };
613 }
614
615 pub fn deinit(self: *Self, allocator: Allocator) void {
616 allocator.free(self.data);
617 self.* = undefined;
618 }
619
620 pub fn fill(self: *Self, value: u16) void {
621 @memset(self.data, value);
622 }
623
624 inline fn _get(self: Self, row: usize) ![]u16 {
625 const start_row = try math.mul(usize, row, self.cols);
626 const end_row = try math.add(usize, start_row, self.cols);
627 return self.data[start_row..end_row];
628 }
629
630 pub fn get(self: Self, row: usize) ![]const u16 {
631 return self._get(row);
632 }
633
634 pub fn getMut(self: *Self, row: usize) ![]u16 {
635 return self._get(row);
636 }
637 };
638
639 pub const Options = struct {
640 unpacked_size: UnpackedSize = .read_from_header,
641 memlimit: ?usize = null,
642 allow_incomplete: bool = false,
643 };
644
645 pub const UnpackedSize = union(enum) {
646 read_from_header,
647 read_header_but_use_provided: ?u64,
648 use_provided: ?u64,
649 };
650
651 const ProcessingStatus = enum {
652 continue_,
653 finished,
654 };
655
656 pub const Properties = struct {
657 lc: u4,
658 lp: u3,
659 pb: u3,
660
661 fn validate(self: Properties) void {
662 assert(self.lc <= 8);
663 assert(self.lp <= 4);
664 assert(self.pb <= 4);
665 }
666 };
667
668 pub const Params = struct {
669 properties: Properties,
670 dict_size: u32,
671 unpacked_size: ?u64,
672
673 pub fn readHeader(reader: anytype, options: Options) !Params {
674 var props = try reader.readByte();
675 if (props >= 225) {
676 return error.CorruptInput;
677 }
678
679 const lc = @as(u4, @intCast(props % 9));
680 props /= 9;
681 const lp = @as(u3, @intCast(props % 5));
682 props /= 5;
683 const pb = @as(u3, @intCast(props));
684
685 const dict_size_provided = try reader.readInt(u32, .little);
686 const dict_size = @max(0x1000, dict_size_provided);
687
688 const unpacked_size = switch (options.unpacked_size) {
689 .read_from_header => blk: {
690 const unpacked_size_provided = try reader.readInt(u64, .little);
691 const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
692 break :blk if (marker_mandatory)
693 null
694 else
695 unpacked_size_provided;
696 },
697 .read_header_but_use_provided => |x| blk: {
698 _ = try reader.readInt(u64, .little);
699 break :blk x;
700 },
701 .use_provided => |x| x,
702 };
703
704 return Params{
705 .properties = Properties{ .lc = lc, .lp = lp, .pb = pb },
706 .dict_size = dict_size,
707 .unpacked_size = unpacked_size,
708 };
709 }
710 };
711};
7712
8pub fn decompress(713pub fn decompress(
9 allocator: Allocator,714 allocator: Allocator,
...@@ -15,9 +720,9 @@ pub fn decompress(...@@ -15,9 +720,9 @@ pub fn decompress(
15pub fn decompressWithOptions(720pub fn decompressWithOptions(
16 allocator: Allocator,721 allocator: Allocator,
17 reader: anytype,722 reader: anytype,
18 options: decode.Options,723 options: Decode.Options,
19) !Decompress(@TypeOf(reader)) {724) !Decompress(@TypeOf(reader)) {
20 const params = try decode.Params.readHeader(reader, options);725 const params = try Decode.Params.readHeader(reader, options);
21 return Decompress(@TypeOf(reader)).init(allocator, reader, params, options.memlimit);726 return Decompress(@TypeOf(reader)).init(allocator, reader, params, options.memlimit);
22}727}
23728
...@@ -36,19 +741,19 @@ pub fn Decompress(comptime ReaderType: type) type {...@@ -36,19 +741,19 @@ pub fn Decompress(comptime ReaderType: type) type {
36 in_reader: ReaderType,741 in_reader: ReaderType,
37 to_read: std.ArrayListUnmanaged(u8),742 to_read: std.ArrayListUnmanaged(u8),
38743
39 buffer: decode.lzbuffer.LzCircularBuffer,744 buffer: Decode.LzCircularBuffer,
40 decoder: decode.rangecoder.RangeDecoder,745 decoder: RangeDecoder,
41 state: decode.DecoderState,746 state: Decode,
42747
43 pub fn init(allocator: Allocator, source: ReaderType, params: decode.Params, memlimit: ?usize) !Self {748 pub fn init(allocator: Allocator, source: ReaderType, params: Decode.Params, memlimit: ?usize) !Self {
44 return Self{749 return Self{
45 .allocator = allocator,750 .allocator = allocator,
46 .in_reader = source,751 .in_reader = source,
47 .to_read = .{},752 .to_read = .{},
48753
49 .buffer = decode.lzbuffer.LzCircularBuffer.init(params.dict_size, memlimit orelse math.maxInt(usize)),754 .buffer = Decode.LzCircularBuffer.init(params.dict_size, memlimit orelse math.maxInt(usize)),
50 .decoder = try decode.rangecoder.RangeDecoder.init(source),755 .decoder = try RangeDecoder.init(source),
51 .state = try decode.DecoderState.init(allocator, params.properties, params.unpacked_size),756 .state = try Decode.init(allocator, params.properties, params.unpacked_size),
52 };757 };
53 }758 }
54759
...@@ -86,5 +791,4 @@ pub fn Decompress(comptime ReaderType: type) type {...@@ -86,5 +791,4 @@ pub fn Decompress(comptime ReaderType: type) type {
86791
87test {792test {
88 _ = @import("lzma/test.zig");793 _ = @import("lzma/test.zig");
89 _ = @import("lzma/vec2d.zig");
90}794}
lib/std/compress/lzma/decode.zig deleted-379
...@@ -1,379 +0,0 @@
1const std = @import("../../std.zig");
2const assert = std.debug.assert;
3const math = std.math;
4const Allocator = std.mem.Allocator;
5
6pub const lzbuffer = @import("decode/lzbuffer.zig");
7pub const rangecoder = @import("decode/rangecoder.zig");
8
9const LzCircularBuffer = lzbuffer.LzCircularBuffer;
10const BitTree = rangecoder.BitTree;
11const LenDecoder = rangecoder.LenDecoder;
12const RangeDecoder = rangecoder.RangeDecoder;
13const Vec2D = @import("vec2d.zig").Vec2D;
14
15pub const Options = struct {
16 unpacked_size: UnpackedSize = .read_from_header,
17 memlimit: ?usize = null,
18 allow_incomplete: bool = false,
19};
20
21pub const UnpackedSize = union(enum) {
22 read_from_header,
23 read_header_but_use_provided: ?u64,
24 use_provided: ?u64,
25};
26
27const ProcessingStatus = enum {
28 continue_,
29 finished,
30};
31
32pub const Properties = struct {
33 lc: u4,
34 lp: u3,
35 pb: u3,
36
37 fn validate(self: Properties) void {
38 assert(self.lc <= 8);
39 assert(self.lp <= 4);
40 assert(self.pb <= 4);
41 }
42};
43
44pub const Params = struct {
45 properties: Properties,
46 dict_size: u32,
47 unpacked_size: ?u64,
48
49 pub fn readHeader(reader: anytype, options: Options) !Params {
50 var props = try reader.readByte();
51 if (props >= 225) {
52 return error.CorruptInput;
53 }
54
55 const lc = @as(u4, @intCast(props % 9));
56 props /= 9;
57 const lp = @as(u3, @intCast(props % 5));
58 props /= 5;
59 const pb = @as(u3, @intCast(props));
60
61 const dict_size_provided = try reader.readInt(u32, .little);
62 const dict_size = @max(0x1000, dict_size_provided);
63
64 const unpacked_size = switch (options.unpacked_size) {
65 .read_from_header => blk: {
66 const unpacked_size_provided = try reader.readInt(u64, .little);
67 const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
68 break :blk if (marker_mandatory)
69 null
70 else
71 unpacked_size_provided;
72 },
73 .read_header_but_use_provided => |x| blk: {
74 _ = try reader.readInt(u64, .little);
75 break :blk x;
76 },
77 .use_provided => |x| x,
78 };
79
80 return Params{
81 .properties = Properties{ .lc = lc, .lp = lp, .pb = pb },
82 .dict_size = dict_size,
83 .unpacked_size = unpacked_size,
84 };
85 }
86};
87
88pub const DecoderState = struct {
89 lzma_props: Properties,
90 unpacked_size: ?u64,
91 literal_probs: Vec2D(u16),
92 pos_slot_decoder: [4]BitTree(6),
93 align_decoder: BitTree(4),
94 pos_decoders: [115]u16,
95 is_match: [192]u16,
96 is_rep: [12]u16,
97 is_rep_g0: [12]u16,
98 is_rep_g1: [12]u16,
99 is_rep_g2: [12]u16,
100 is_rep_0long: [192]u16,
101 state: usize,
102 rep: [4]usize,
103 len_decoder: LenDecoder,
104 rep_len_decoder: LenDecoder,
105
106 pub fn init(
107 allocator: Allocator,
108 lzma_props: Properties,
109 unpacked_size: ?u64,
110 ) !DecoderState {
111 return .{
112 .lzma_props = lzma_props,
113 .unpacked_size = unpacked_size,
114 .literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }),
115 .pos_slot_decoder = @splat(.{}),
116 .align_decoder = .{},
117 .pos_decoders = @splat(0x400),
118 .is_match = @splat(0x400),
119 .is_rep = @splat(0x400),
120 .is_rep_g0 = @splat(0x400),
121 .is_rep_g1 = @splat(0x400),
122 .is_rep_g2 = @splat(0x400),
123 .is_rep_0long = @splat(0x400),
124 .state = 0,
125 .rep = @splat(0),
126 .len_decoder = .{},
127 .rep_len_decoder = .{},
128 };
129 }
130
131 pub fn deinit(self: *DecoderState, allocator: Allocator) void {
132 self.literal_probs.deinit(allocator);
133 self.* = undefined;
134 }
135
136 pub fn resetState(self: *DecoderState, allocator: Allocator, new_props: Properties) !void {
137 new_props.validate();
138 if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
139 self.literal_probs.fill(0x400);
140 } else {
141 self.literal_probs.deinit(allocator);
142 self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
143 }
144
145 self.lzma_props = new_props;
146 for (&self.pos_slot_decoder) |*t| t.reset();
147 self.align_decoder.reset();
148 self.pos_decoders = @splat(0x400);
149 self.is_match = @splat(0x400);
150 self.is_rep = @splat(0x400);
151 self.is_rep_g0 = @splat(0x400);
152 self.is_rep_g1 = @splat(0x400);
153 self.is_rep_g2 = @splat(0x400);
154 self.is_rep_0long = @splat(0x400);
155 self.state = 0;
156 self.rep = @splat(0);
157 self.len_decoder.reset();
158 self.rep_len_decoder.reset();
159 }
160
161 fn processNextInner(
162 self: *DecoderState,
163 allocator: Allocator,
164 reader: anytype,
165 writer: anytype,
166 buffer: anytype,
167 decoder: *RangeDecoder,
168 update: bool,
169 ) !ProcessingStatus {
170 const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1);
171
172 if (!try decoder.decodeBit(
173 reader,
174 &self.is_match[(self.state << 4) + pos_state],
175 update,
176 )) {
177 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);
178
179 if (update) {
180 try buffer.appendLiteral(allocator, byte, writer);
181
182 self.state = if (self.state < 4)
183 0
184 else if (self.state < 10)
185 self.state - 3
186 else
187 self.state - 6;
188 }
189 return .continue_;
190 }
191
192 var len: usize = undefined;
193 if (try decoder.decodeBit(reader, &self.is_rep[self.state], update)) {
194 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], update)) {
195 if (!try decoder.decodeBit(
196 reader,
197 &self.is_rep_0long[(self.state << 4) + pos_state],
198 update,
199 )) {
200 if (update) {
201 self.state = if (self.state < 7) 9 else 11;
202 const dist = self.rep[0] + 1;
203 try buffer.appendLz(allocator, 1, dist, writer);
204 }
205 return .continue_;
206 }
207 } else {
208 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))
209 1
210 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], update))
211 2
212 else
213 3;
214 if (update) {
215 const dist = self.rep[idx];
216 var i = idx;
217 while (i > 0) : (i -= 1) {
218 self.rep[i] = self.rep[i - 1];
219 }
220 self.rep[0] = dist;
221 }
222 }
223
224 len = try self.rep_len_decoder.decode(reader, decoder, pos_state, update);
225
226 if (update) {
227 self.state = if (self.state < 7) 8 else 11;
228 }
229 } else {
230 if (update) {
231 self.rep[3] = self.rep[2];
232 self.rep[2] = self.rep[1];
233 self.rep[1] = self.rep[0];
234 }
235
236 len = try self.len_decoder.decode(reader, decoder, pos_state, update);
237
238 if (update) {
239 self.state = if (self.state < 7) 7 else 10;
240 }
241
242 const rep_0 = try self.decodeDistance(reader, decoder, len, update);
243
244 if (update) {
245 self.rep[0] = rep_0;
246 if (self.rep[0] == 0xFFFF_FFFF) {
247 if (decoder.isFinished()) {
248 return .finished;
249 }
250 return error.CorruptInput;
251 }
252 }
253 }
254
255 if (update) {
256 len += 2;
257
258 const dist = self.rep[0] + 1;
259 try buffer.appendLz(allocator, len, dist, writer);
260 }
261
262 return .continue_;
263 }
264
265 fn processNext(
266 self: *DecoderState,
267 allocator: Allocator,
268 reader: anytype,
269 writer: anytype,
270 buffer: anytype,
271 decoder: *RangeDecoder,
272 ) !ProcessingStatus {
273 return self.processNextInner(allocator, reader, writer, buffer, decoder, true);
274 }
275
276 pub fn process(
277 self: *DecoderState,
278 allocator: Allocator,
279 reader: anytype,
280 writer: anytype,
281 buffer: anytype,
282 decoder: *RangeDecoder,
283 ) !ProcessingStatus {
284 process_next: {
285 if (self.unpacked_size) |unpacked_size| {
286 if (buffer.len >= unpacked_size) {
287 break :process_next;
288 }
289 } else if (decoder.isFinished()) {
290 break :process_next;
291 }
292
293 switch (try self.processNext(allocator, reader, writer, buffer, decoder)) {
294 .continue_ => return .continue_,
295 .finished => break :process_next,
296 }
297 }
298
299 if (self.unpacked_size) |unpacked_size| {
300 if (buffer.len != unpacked_size) {
301 return error.CorruptInput;
302 }
303 }
304
305 return .finished;
306 }
307
308 fn decodeLiteral(
309 self: *DecoderState,
310 reader: anytype,
311 buffer: anytype,
312 decoder: *RangeDecoder,
313 update: bool,
314 ) !u8 {
315 const def_prev_byte = 0;
316 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
317
318 var result: usize = 1;
319 const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
320 (prev_byte >> (8 - self.lzma_props.lc));
321 const probs = try self.literal_probs.getMut(lit_state);
322
323 if (self.state >= 7) {
324 var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
325
326 while (result < 0x100) {
327 const match_bit = (match_byte >> 7) & 1;
328 match_byte <<= 1;
329 const bit = @intFromBool(try decoder.decodeBit(
330 reader,
331 &probs[((@as(usize, 1) + match_bit) << 8) + result],
332 update,
333 ));
334 result = (result << 1) ^ bit;
335 if (match_bit != bit) {
336 break;
337 }
338 }
339 }
340
341 while (result < 0x100) {
342 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result], update));
343 }
344
345 return @as(u8, @truncate(result - 0x100));
346 }
347
348 fn decodeDistance(
349 self: *DecoderState,
350 reader: anytype,
351 decoder: *RangeDecoder,
352 length: usize,
353 update: bool,
354 ) !usize {
355 const len_state = if (length > 3) 3 else length;
356
357 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder, update));
358 if (pos_slot < 4)
359 return pos_slot;
360
361 const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
362 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
363
364 if (pos_slot < 14) {
365 result += try decoder.parseReverseBitTree(
366 reader,
367 num_direct_bits,
368 &self.pos_decoders,
369 result - pos_slot,
370 update,
371 );
372 } else {
373 result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4;
374 result += try self.align_decoder.parseReverse(reader, decoder, update);
375 }
376
377 return result;
378 }
379};
lib/std/compress/lzma/decode/lzbuffer.zig deleted-228
...@@ -1,228 +0,0 @@
1const std = @import("../../../std.zig");
2const math = std.math;
3const mem = std.mem;
4const Allocator = std.mem.Allocator;
5const ArrayListUnmanaged = std.ArrayListUnmanaged;
6
7/// An accumulating buffer for LZ sequences
8pub const LzAccumBuffer = struct {
9 /// Buffer
10 buf: ArrayListUnmanaged(u8),
11
12 /// Buffer memory limit
13 memlimit: usize,
14
15 /// Total number of bytes sent through the buffer
16 len: usize,
17
18 const Self = @This();
19
20 pub fn init(memlimit: usize) Self {
21 return Self{
22 .buf = .{},
23 .memlimit = memlimit,
24 .len = 0,
25 };
26 }
27
28 pub fn appendByte(self: *Self, allocator: Allocator, byte: u8) !void {
29 try self.buf.append(allocator, byte);
30 self.len += 1;
31 }
32
33 /// Reset the internal dictionary
34 pub fn reset(self: *Self, writer: anytype) !void {
35 try writer.writeAll(self.buf.items);
36 self.buf.clearRetainingCapacity();
37 self.len = 0;
38 }
39
40 /// Retrieve the last byte or return a default
41 pub fn lastOr(self: Self, lit: u8) u8 {
42 const buf_len = self.buf.items.len;
43 return if (buf_len == 0)
44 lit
45 else
46 self.buf.items[buf_len - 1];
47 }
48
49 /// Retrieve the n-th last byte
50 pub fn lastN(self: Self, dist: usize) !u8 {
51 const buf_len = self.buf.items.len;
52 if (dist > buf_len) {
53 return error.CorruptInput;
54 }
55
56 return self.buf.items[buf_len - dist];
57 }
58
59 /// Append a literal
60 pub fn appendLiteral(
61 self: *Self,
62 allocator: Allocator,
63 lit: u8,
64 writer: anytype,
65 ) !void {
66 _ = writer;
67 if (self.len >= self.memlimit) {
68 return error.CorruptInput;
69 }
70 try self.buf.append(allocator, lit);
71 self.len += 1;
72 }
73
74 /// Fetch an LZ sequence (length, distance) from inside the buffer
75 pub fn appendLz(
76 self: *Self,
77 allocator: Allocator,
78 len: usize,
79 dist: usize,
80 writer: anytype,
81 ) !void {
82 _ = writer;
83
84 const buf_len = self.buf.items.len;
85 if (dist > buf_len) {
86 return error.CorruptInput;
87 }
88
89 var offset = buf_len - dist;
90 var i: usize = 0;
91 while (i < len) : (i += 1) {
92 const x = self.buf.items[offset];
93 try self.buf.append(allocator, x);
94 offset += 1;
95 }
96 self.len += len;
97 }
98
99 pub fn finish(self: *Self, writer: anytype) !void {
100 try writer.writeAll(self.buf.items);
101 self.buf.clearRetainingCapacity();
102 }
103
104 pub fn deinit(self: *Self, allocator: Allocator) void {
105 self.buf.deinit(allocator);
106 self.* = undefined;
107 }
108};
109
110/// A circular buffer for LZ sequences
111pub const LzCircularBuffer = struct {
112 /// Circular buffer
113 buf: ArrayListUnmanaged(u8),
114
115 /// Length of the buffer
116 dict_size: usize,
117
118 /// Buffer memory limit
119 memlimit: usize,
120
121 /// Current position
122 cursor: usize,
123
124 /// Total number of bytes sent through the buffer
125 len: usize,
126
127 const Self = @This();
128
129 pub fn init(dict_size: usize, memlimit: usize) Self {
130 return Self{
131 .buf = .{},
132 .dict_size = dict_size,
133 .memlimit = memlimit,
134 .cursor = 0,
135 .len = 0,
136 };
137 }
138
139 pub fn get(self: Self, index: usize) u8 {
140 return if (0 <= index and index < self.buf.items.len)
141 self.buf.items[index]
142 else
143 0;
144 }
145
146 pub fn set(self: *Self, allocator: Allocator, index: usize, value: u8) !void {
147 if (index >= self.memlimit) {
148 return error.CorruptInput;
149 }
150 try self.buf.ensureTotalCapacity(allocator, index + 1);
151 while (self.buf.items.len < index) {
152 self.buf.appendAssumeCapacity(0);
153 }
154 self.buf.appendAssumeCapacity(value);
155 }
156
157 /// Retrieve the last byte or return a default
158 pub fn lastOr(self: Self, lit: u8) u8 {
159 return if (self.len == 0)
160 lit
161 else
162 self.get((self.dict_size + self.cursor - 1) % self.dict_size);
163 }
164
165 /// Retrieve the n-th last byte
166 pub fn lastN(self: Self, dist: usize) !u8 {
167 if (dist > self.dict_size or dist > self.len) {
168 return error.CorruptInput;
169 }
170
171 const offset = (self.dict_size + self.cursor - dist) % self.dict_size;
172 return self.get(offset);
173 }
174
175 /// Append a literal
176 pub fn appendLiteral(
177 self: *Self,
178 allocator: Allocator,
179 lit: u8,
180 writer: anytype,
181 ) !void {
182 try self.set(allocator, self.cursor, lit);
183 self.cursor += 1;
184 self.len += 1;
185
186 // Flush the circular buffer to the output
187 if (self.cursor == self.dict_size) {
188 try writer.writeAll(self.buf.items);
189 self.cursor = 0;
190 }
191 }
192
193 /// Fetch an LZ sequence (length, distance) from inside the buffer
194 pub fn appendLz(
195 self: *Self,
196 allocator: Allocator,
197 len: usize,
198 dist: usize,
199 writer: anytype,
200 ) !void {
201 if (dist > self.dict_size or dist > self.len) {
202 return error.CorruptInput;
203 }
204
205 var offset = (self.dict_size + self.cursor - dist) % self.dict_size;
206 var i: usize = 0;
207 while (i < len) : (i += 1) {
208 const x = self.get(offset);
209 try self.appendLiteral(allocator, x, writer);
210 offset += 1;
211 if (offset == self.dict_size) {
212 offset = 0;
213 }
214 }
215 }
216
217 pub fn finish(self: *Self, writer: anytype) !void {
218 if (self.cursor > 0) {
219 try writer.writeAll(self.buf.items[0..self.cursor]);
220 self.cursor = 0;
221 }
222 }
223
224 pub fn deinit(self: *Self, allocator: Allocator) void {
225 self.buf.deinit(allocator);
226 self.* = undefined;
227 }
228};
lib/std/compress/lzma/decode/rangecoder.zig deleted-181
...@@ -1,181 +0,0 @@
1const std = @import("../../../std.zig");
2const mem = std.mem;
3
4pub const RangeDecoder = struct {
5 range: u32,
6 code: u32,
7
8 pub fn init(reader: anytype) !RangeDecoder {
9 const reserved = try reader.readByte();
10 if (reserved != 0) {
11 return error.CorruptInput;
12 }
13 return RangeDecoder{
14 .range = 0xFFFF_FFFF,
15 .code = try reader.readInt(u32, .big),
16 };
17 }
18
19 pub fn fromParts(
20 range: u32,
21 code: u32,
22 ) RangeDecoder {
23 return .{
24 .range = range,
25 .code = code,
26 };
27 }
28
29 pub fn set(self: *RangeDecoder, range: u32, code: u32) void {
30 self.range = range;
31 self.code = code;
32 }
33
34 pub inline fn isFinished(self: RangeDecoder) bool {
35 return self.code == 0;
36 }
37
38 inline fn normalize(self: *RangeDecoder, reader: anytype) !void {
39 if (self.range < 0x0100_0000) {
40 self.range <<= 8;
41 self.code = (self.code << 8) ^ @as(u32, try reader.readByte());
42 }
43 }
44
45 inline fn getBit(self: *RangeDecoder, reader: anytype) !bool {
46 self.range >>= 1;
47
48 const bit = self.code >= self.range;
49 if (bit)
50 self.code -= self.range;
51
52 try self.normalize(reader);
53 return bit;
54 }
55
56 pub fn get(self: *RangeDecoder, reader: anytype, count: usize) !u32 {
57 var result: u32 = 0;
58 var i: usize = 0;
59 while (i < count) : (i += 1)
60 result = (result << 1) ^ @intFromBool(try self.getBit(reader));
61 return result;
62 }
63
64 pub inline fn decodeBit(self: *RangeDecoder, reader: anytype, prob: *u16, update: bool) !bool {
65 const bound = (self.range >> 11) * prob.*;
66
67 if (self.code < bound) {
68 if (update)
69 prob.* += (0x800 - prob.*) >> 5;
70 self.range = bound;
71
72 try self.normalize(reader);
73 return false;
74 } else {
75 if (update)
76 prob.* -= prob.* >> 5;
77 self.code -= bound;
78 self.range -= bound;
79
80 try self.normalize(reader);
81 return true;
82 }
83 }
84
85 fn parseBitTree(
86 self: *RangeDecoder,
87 reader: anytype,
88 num_bits: u5,
89 probs: []u16,
90 update: bool,
91 ) !u32 {
92 var tmp: u32 = 1;
93 var i: @TypeOf(num_bits) = 0;
94 while (i < num_bits) : (i += 1) {
95 const bit = try self.decodeBit(reader, &probs[tmp], update);
96 tmp = (tmp << 1) ^ @intFromBool(bit);
97 }
98 return tmp - (@as(u32, 1) << num_bits);
99 }
100
101 pub fn parseReverseBitTree(
102 self: *RangeDecoder,
103 reader: anytype,
104 num_bits: u5,
105 probs: []u16,
106 offset: usize,
107 update: bool,
108 ) !u32 {
109 var result: u32 = 0;
110 var tmp: usize = 1;
111 var i: @TypeOf(num_bits) = 0;
112 while (i < num_bits) : (i += 1) {
113 const bit = @intFromBool(try self.decodeBit(reader, &probs[offset + tmp], update));
114 tmp = (tmp << 1) ^ bit;
115 result ^= @as(u32, bit) << i;
116 }
117 return result;
118 }
119};
120
121pub fn BitTree(comptime num_bits: usize) type {
122 return struct {
123 probs: [1 << num_bits]u16 = @splat(0x400),
124
125 const Self = @This();
126
127 pub fn parse(
128 self: *Self,
129 reader: anytype,
130 decoder: *RangeDecoder,
131 update: bool,
132 ) !u32 {
133 return decoder.parseBitTree(reader, num_bits, &self.probs, update);
134 }
135
136 pub fn parseReverse(
137 self: *Self,
138 reader: anytype,
139 decoder: *RangeDecoder,
140 update: bool,
141 ) !u32 {
142 return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, update);
143 }
144
145 pub fn reset(self: *Self) void {
146 @memset(&self.probs, 0x400);
147 }
148 };
149}
150
151pub const LenDecoder = struct {
152 choice: u16 = 0x400,
153 choice2: u16 = 0x400,
154 low_coder: [16]BitTree(3) = @splat(.{}),
155 mid_coder: [16]BitTree(3) = @splat(.{}),
156 high_coder: BitTree(8) = .{},
157
158 pub fn decode(
159 self: *LenDecoder,
160 reader: anytype,
161 decoder: *RangeDecoder,
162 pos_state: usize,
163 update: bool,
164 ) !usize {
165 if (!try decoder.decodeBit(reader, &self.choice, update)) {
166 return @as(usize, try self.low_coder[pos_state].parse(reader, decoder, update));
167 } else if (!try decoder.decodeBit(reader, &self.choice2, update)) {
168 return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder, update)) + 8;
169 } else {
170 return @as(usize, try self.high_coder.parse(reader, decoder, update)) + 16;
171 }
172 }
173
174 pub fn reset(self: *LenDecoder) void {
175 self.choice = 0x400;
176 self.choice2 = 0x400;
177 for (&self.low_coder) |*t| t.reset();
178 for (&self.mid_coder) |*t| t.reset();
179 self.high_coder.reset();
180 }
181};
lib/std/compress/lzma/vec2d.zig deleted-128
...@@ -1,128 +0,0 @@
1const std = @import("../../std.zig");
2const math = std.math;
3const mem = std.mem;
4const Allocator = std.mem.Allocator;
5
6pub fn Vec2D(comptime T: type) type {
7 return struct {
8 data: []T,
9 cols: usize,
10
11 const Self = @This();
12
13 pub fn init(allocator: Allocator, value: T, size: struct { usize, usize }) !Self {
14 const len = try math.mul(usize, size[0], size[1]);
15 const data = try allocator.alloc(T, len);
16 @memset(data, value);
17 return Self{
18 .data = data,
19 .cols = size[1],
20 };
21 }
22
23 pub fn deinit(self: *Self, allocator: Allocator) void {
24 allocator.free(self.data);
25 self.* = undefined;
26 }
27
28 pub fn fill(self: *Self, value: T) void {
29 @memset(self.data, value);
30 }
31
32 inline fn _get(self: Self, row: usize) ![]T {
33 const start_row = try math.mul(usize, row, self.cols);
34 const end_row = try math.add(usize, start_row, self.cols);
35 return self.data[start_row..end_row];
36 }
37
38 pub fn get(self: Self, row: usize) ![]const T {
39 return self._get(row);
40 }
41
42 pub fn getMut(self: *Self, row: usize) ![]T {
43 return self._get(row);
44 }
45 };
46}
47
48const testing = std.testing;
49const expectEqualSlices = std.testing.expectEqualSlices;
50const expectError = std.testing.expectError;
51
52test "init" {
53 const allocator = testing.allocator;
54 var vec2d = try Vec2D(i32).init(allocator, 1, .{ 2, 3 });
55 defer vec2d.deinit(allocator);
56
57 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(0));
58 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(1));
59}
60
61test "init overflow" {
62 const allocator = testing.allocator;
63 try expectError(
64 error.Overflow,
65 Vec2D(i32).init(allocator, 1, .{ math.maxInt(usize), math.maxInt(usize) }),
66 );
67}
68
69test "fill" {
70 const allocator = testing.allocator;
71 var vec2d = try Vec2D(i32).init(allocator, 0, .{ 2, 3 });
72 defer vec2d.deinit(allocator);
73
74 vec2d.fill(7);
75
76 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(0));
77 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(1));
78}
79
80test "get" {
81 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
82 const vec2d = Vec2D(i32){
83 .data = &data,
84 .cols = 2,
85 };
86
87 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
88 try expectEqualSlices(i32, &.{ 2, 3 }, try vec2d.get(1));
89 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
90 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
91}
92
93test "getMut" {
94 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
95 var vec2d = Vec2D(i32){
96 .data = &data,
97 .cols = 2,
98 };
99
100 const row = try vec2d.getMut(1);
101 row[1] = 9;
102
103 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
104 // (1, 1) should be 9.
105 try expectEqualSlices(i32, &.{ 2, 9 }, try vec2d.get(1));
106 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
107 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
108}
109
110test "get multiplication overflow" {
111 const allocator = testing.allocator;
112 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 4 });
113 defer matrix.deinit(allocator);
114
115 const row = (math.maxInt(usize) / 4) + 1;
116 try expectError(error.Overflow, matrix.get(row));
117 try expectError(error.Overflow, matrix.getMut(row));
118}
119
120test "get addition overflow" {
121 const allocator = testing.allocator;
122 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 5 });
123 defer matrix.deinit(allocator);
124
125 const row = math.maxInt(usize) / 5;
126 try expectError(error.Overflow, matrix.get(row));
127 try expectError(error.Overflow, matrix.getMut(row));
128}
lib/std/compress/lzma2.zig+266-2
...@@ -1,14 +1,278 @@...@@ -1,14 +1,278 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const Allocator = std.mem.Allocator;2const Allocator = std.mem.Allocator;
3const ArrayList = std.ArrayList;
4const lzma = std.compress.lzma;
35
4pub const decode = @import("lzma2/decode.zig");6/// An accumulating buffer for LZ sequences
7pub const LzAccumBuffer = struct {
8 /// Buffer
9 buf: ArrayList(u8),
10
11 /// Buffer memory limit
12 memlimit: usize,
13
14 /// Total number of bytes sent through the buffer
15 len: usize,
16
17 const Self = @This();
18
19 pub fn init(memlimit: usize) Self {
20 return Self{
21 .buf = .{},
22 .memlimit = memlimit,
23 .len = 0,
24 };
25 }
26
27 pub fn appendByte(self: *Self, allocator: Allocator, byte: u8) !void {
28 try self.buf.append(allocator, byte);
29 self.len += 1;
30 }
31
32 /// Reset the internal dictionary
33 pub fn reset(self: *Self, writer: anytype) !void {
34 try writer.writeAll(self.buf.items);
35 self.buf.clearRetainingCapacity();
36 self.len = 0;
37 }
38
39 /// Retrieve the last byte or return a default
40 pub fn lastOr(self: Self, lit: u8) u8 {
41 const buf_len = self.buf.items.len;
42 return if (buf_len == 0)
43 lit
44 else
45 self.buf.items[buf_len - 1];
46 }
47
48 /// Retrieve the n-th last byte
49 pub fn lastN(self: Self, dist: usize) !u8 {
50 const buf_len = self.buf.items.len;
51 if (dist > buf_len) {
52 return error.CorruptInput;
53 }
54
55 return self.buf.items[buf_len - dist];
56 }
57
58 /// Append a literal
59 pub fn appendLiteral(
60 self: *Self,
61 allocator: Allocator,
62 lit: u8,
63 writer: anytype,
64 ) !void {
65 _ = writer;
66 if (self.len >= self.memlimit) {
67 return error.CorruptInput;
68 }
69 try self.buf.append(allocator, lit);
70 self.len += 1;
71 }
72
73 /// Fetch an LZ sequence (length, distance) from inside the buffer
74 pub fn appendLz(
75 self: *Self,
76 allocator: Allocator,
77 len: usize,
78 dist: usize,
79 writer: anytype,
80 ) !void {
81 _ = writer;
82
83 const buf_len = self.buf.items.len;
84 if (dist > buf_len) {
85 return error.CorruptInput;
86 }
87
88 var offset = buf_len - dist;
89 var i: usize = 0;
90 while (i < len) : (i += 1) {
91 const x = self.buf.items[offset];
92 try self.buf.append(allocator, x);
93 offset += 1;
94 }
95 self.len += len;
96 }
97
98 pub fn finish(self: *Self, writer: anytype) !void {
99 try writer.writeAll(self.buf.items);
100 self.buf.clearRetainingCapacity();
101 }
102
103 pub fn deinit(self: *Self, allocator: Allocator) void {
104 self.buf.deinit(allocator);
105 self.* = undefined;
106 }
107};
108
109pub const Decode = struct {
110 lzma_state: lzma.Decode,
111
112 pub fn init(allocator: Allocator) !Decode {
113 return Decode{
114 .lzma_state = try lzma.Decode.init(
115 allocator,
116 .{
117 .lc = 0,
118 .lp = 0,
119 .pb = 0,
120 },
121 null,
122 ),
123 };
124 }
125
126 pub fn deinit(self: *Decode, allocator: Allocator) void {
127 self.lzma_state.deinit(allocator);
128 self.* = undefined;
129 }
130
131 pub fn decompress(
132 self: *Decode,
133 allocator: Allocator,
134 reader: anytype,
135 writer: anytype,
136 ) !void {
137 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
138 defer accum.deinit(allocator);
139
140 while (true) {
141 const status = try reader.readByte();
142
143 switch (status) {
144 0 => break,
145 1 => try parseUncompressed(allocator, reader, writer, &accum, true),
146 2 => try parseUncompressed(allocator, reader, writer, &accum, false),
147 else => try self.parseLzma(allocator, reader, writer, &accum, status),
148 }
149 }
150
151 try accum.finish(writer);
152 }
153
154 fn parseLzma(
155 self: *Decode,
156 allocator: Allocator,
157 reader: anytype,
158 writer: anytype,
159 accum: *LzAccumBuffer,
160 status: u8,
161 ) !void {
162 if (status & 0x80 == 0) {
163 return error.CorruptInput;
164 }
165
166 const Reset = struct {
167 dict: bool,
168 state: bool,
169 props: bool,
170 };
171
172 const reset = switch ((status >> 5) & 0x3) {
173 0 => Reset{
174 .dict = false,
175 .state = false,
176 .props = false,
177 },
178 1 => Reset{
179 .dict = false,
180 .state = true,
181 .props = false,
182 },
183 2 => Reset{
184 .dict = false,
185 .state = true,
186 .props = true,
187 },
188 3 => Reset{
189 .dict = true,
190 .state = true,
191 .props = true,
192 },
193 else => unreachable,
194 };
195
196 const unpacked_size = blk: {
197 var tmp: u64 = status & 0x1F;
198 tmp <<= 16;
199 tmp |= try reader.readInt(u16, .big);
200 break :blk tmp + 1;
201 };
202
203 const packed_size = blk: {
204 const tmp: u17 = try reader.readInt(u16, .big);
205 break :blk tmp + 1;
206 };
207
208 if (reset.dict) {
209 try accum.reset(writer);
210 }
211
212 if (reset.state) {
213 var new_props = self.lzma_state.lzma_props;
214
215 if (reset.props) {
216 var props = try reader.readByte();
217 if (props >= 225) {
218 return error.CorruptInput;
219 }
220
221 const lc = @as(u4, @intCast(props % 9));
222 props /= 9;
223 const lp = @as(u3, @intCast(props % 5));
224 props /= 5;
225 const pb = @as(u3, @intCast(props));
226
227 if (lc + lp > 4) {
228 return error.CorruptInput;
229 }
230
231 new_props = .{ .lc = lc, .lp = lp, .pb = pb };
232 }
233
234 try self.lzma_state.resetState(allocator, new_props);
235 }
236
237 self.lzma_state.unpacked_size = unpacked_size + accum.len;
238
239 var counter = std.io.countingReader(reader);
240 const counter_reader = counter.reader();
241
242 var rangecoder = try lzma.RangeDecoder.init(counter_reader);
243 while (try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
244
245 if (counter.bytes_read != packed_size) {
246 return error.CorruptInput;
247 }
248 }
249
250 fn parseUncompressed(
251 allocator: Allocator,
252 reader: anytype,
253 writer: anytype,
254 accum: *LzAccumBuffer,
255 reset_dict: bool,
256 ) !void {
257 const unpacked_size = @as(u17, try reader.readInt(u16, .big)) + 1;
258
259 if (reset_dict) {
260 try accum.reset(writer);
261 }
262
263 var i: @TypeOf(unpacked_size) = 0;
264 while (i < unpacked_size) : (i += 1) {
265 try accum.appendByte(allocator, try reader.readByte());
266 }
267 }
268};
5269
6pub fn decompress(270pub fn decompress(
7 allocator: Allocator,271 allocator: Allocator,
8 reader: anytype,272 reader: anytype,
9 writer: anytype,273 writer: anytype,
10) !void {274) !void {
11 var decoder = try decode.Decoder.init(allocator);275 var decoder = try Decode.init(allocator);
12 defer decoder.deinit(allocator);276 defer decoder.deinit(allocator);
13 return decoder.decompress(allocator, reader, writer);277 return decoder.decompress(allocator, reader, writer);
14}278}
lib/std/compress/lzma2/decode.zig deleted-169
...@@ -1,169 +0,0 @@
1const std = @import("../../std.zig");
2const Allocator = std.mem.Allocator;
3
4const lzma = @import("../lzma.zig");
5const DecoderState = lzma.decode.DecoderState;
6const LzAccumBuffer = lzma.decode.lzbuffer.LzAccumBuffer;
7const Properties = lzma.decode.Properties;
8const RangeDecoder = lzma.decode.rangecoder.RangeDecoder;
9
10pub const Decoder = struct {
11 lzma_state: DecoderState,
12
13 pub fn init(allocator: Allocator) !Decoder {
14 return Decoder{
15 .lzma_state = try DecoderState.init(
16 allocator,
17 Properties{
18 .lc = 0,
19 .lp = 0,
20 .pb = 0,
21 },
22 null,
23 ),
24 };
25 }
26
27 pub fn deinit(self: *Decoder, allocator: Allocator) void {
28 self.lzma_state.deinit(allocator);
29 self.* = undefined;
30 }
31
32 pub fn decompress(
33 self: *Decoder,
34 allocator: Allocator,
35 reader: anytype,
36 writer: anytype,
37 ) !void {
38 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
39 defer accum.deinit(allocator);
40
41 while (true) {
42 const status = try reader.readByte();
43
44 switch (status) {
45 0 => break,
46 1 => try parseUncompressed(allocator, reader, writer, &accum, true),
47 2 => try parseUncompressed(allocator, reader, writer, &accum, false),
48 else => try self.parseLzma(allocator, reader, writer, &accum, status),
49 }
50 }
51
52 try accum.finish(writer);
53 }
54
55 fn parseLzma(
56 self: *Decoder,
57 allocator: Allocator,
58 reader: anytype,
59 writer: anytype,
60 accum: *LzAccumBuffer,
61 status: u8,
62 ) !void {
63 if (status & 0x80 == 0) {
64 return error.CorruptInput;
65 }
66
67 const Reset = struct {
68 dict: bool,
69 state: bool,
70 props: bool,
71 };
72
73 const reset = switch ((status >> 5) & 0x3) {
74 0 => Reset{
75 .dict = false,
76 .state = false,
77 .props = false,
78 },
79 1 => Reset{
80 .dict = false,
81 .state = true,
82 .props = false,
83 },
84 2 => Reset{
85 .dict = false,
86 .state = true,
87 .props = true,
88 },
89 3 => Reset{
90 .dict = true,
91 .state = true,
92 .props = true,
93 },
94 else => unreachable,
95 };
96
97 const unpacked_size = blk: {
98 var tmp: u64 = status & 0x1F;
99 tmp <<= 16;
100 tmp |= try reader.readInt(u16, .big);
101 break :blk tmp + 1;
102 };
103
104 const packed_size = blk: {
105 const tmp: u17 = try reader.readInt(u16, .big);
106 break :blk tmp + 1;
107 };
108
109 if (reset.dict) {
110 try accum.reset(writer);
111 }
112
113 if (reset.state) {
114 var new_props = self.lzma_state.lzma_props;
115
116 if (reset.props) {
117 var props = try reader.readByte();
118 if (props >= 225) {
119 return error.CorruptInput;
120 }
121
122 const lc = @as(u4, @intCast(props % 9));
123 props /= 9;
124 const lp = @as(u3, @intCast(props % 5));
125 props /= 5;
126 const pb = @as(u3, @intCast(props));
127
128 if (lc + lp > 4) {
129 return error.CorruptInput;
130 }
131
132 new_props = Properties{ .lc = lc, .lp = lp, .pb = pb };
133 }
134
135 try self.lzma_state.resetState(allocator, new_props);
136 }
137
138 self.lzma_state.unpacked_size = unpacked_size + accum.len;
139
140 var counter = std.io.countingReader(reader);
141 const counter_reader = counter.reader();
142
143 var rangecoder = try RangeDecoder.init(counter_reader);
144 while (try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
145
146 if (counter.bytes_read != packed_size) {
147 return error.CorruptInput;
148 }
149 }
150
151 fn parseUncompressed(
152 allocator: Allocator,
153 reader: anytype,
154 writer: anytype,
155 accum: *LzAccumBuffer,
156 reset_dict: bool,
157 ) !void {
158 const unpacked_size = @as(u17, try reader.readInt(u16, .big)) + 1;
159
160 if (reset_dict) {
161 try accum.reset(writer);
162 }
163
164 var i: @TypeOf(unpacked_size) = 0;
165 while (i < unpacked_size) : (i += 1) {
166 try accum.appendByte(allocator, try reader.readByte());
167 }
168 }
169};