| ... | @@ -3,10 +3,10 @@ const assert = std.debug.assert; | ... | @@ -3,10 +3,10 @@ const assert = std.debug.assert; |
| 3 | | 3 | |
| 4 | const types = @import("types.zig"); | 4 | const types = @import("types.zig"); |
| 5 | const frame = types.frame; | 5 | const frame = types.frame; |
| 6 | const Literals = types.compressed_block.Literals; | 6 | const LiteralsSection = types.compressed_block.LiteralsSection; |
| 7 | const Sequences = types.compressed_block.Sequences; | 7 | const SequencesSection = types.compressed_block.SequencesSection; |
| 8 | const Table = types.compressed_block.Table; | 8 | const Table = types.compressed_block.Table; |
| 9 | const RingBuffer = @import("RingBuffer.zig"); | 9 | pub const RingBuffer = @import("RingBuffer.zig"); |
| 10 | | 10 | |
| 11 | const readInt = std.mem.readIntLittle; | 11 | const readInt = std.mem.readIntLittle; |
| 12 | const readIntSlice = std.mem.readIntSliceLittle; | 12 | const readIntSlice = std.mem.readIntSliceLittle; |
| ... | @@ -55,7 +55,7 @@ pub fn decodeFrame(dest: []u8, src: []const u8, verify_checksum: bool) !ReadWrit | ... | @@ -55,7 +55,7 @@ pub fn decodeFrame(dest: []u8, src: []const u8, verify_checksum: bool) !ReadWrit |
| 55 | }; | 55 | }; |
| 56 | } | 56 | } |
| 57 | | 57 | |
| 58 | const DecodeState = struct { | 58 | pub const DecodeState = struct { |
| 59 | repeat_offsets: [3]u32, | 59 | repeat_offsets: [3]u32, |
| 60 | | 60 | |
| 61 | offset: StateData(8), | 61 | offset: StateData(8), |
| ... | @@ -70,7 +70,7 @@ const DecodeState = struct { | ... | @@ -70,7 +70,7 @@ const DecodeState = struct { |
| 70 | | 70 | |
| 71 | literal_stream_reader: ReverseBitReader, | 71 | literal_stream_reader: ReverseBitReader, |
| 72 | literal_stream_index: usize, | 72 | literal_stream_index: usize, |
| 73 | huffman_tree: ?Literals.HuffmanTree, | 73 | huffman_tree: ?LiteralsSection.HuffmanTree, |
| 74 | | 74 | |
| 75 | literal_written_count: usize, | 75 | literal_written_count: usize, |
| 76 | | 76 | |
| ... | @@ -84,7 +84,55 @@ const DecodeState = struct { | ... | @@ -84,7 +84,55 @@ const DecodeState = struct { |
| 84 | }; | 84 | }; |
| 85 | } | 85 | } |
| 86 | | 86 | |
| 87 | fn readInitialState(self: *DecodeState, bit_reader: anytype) !void { | 87 | pub fn prepare( |
| | 88 | self: *DecodeState, |
| | 89 | src: []const u8, |
| | 90 | literals: LiteralsSection, |
| | 91 | sequences_header: SequencesSection.Header, |
| | 92 | ) !usize { |
| | 93 | if (literals.huffman_tree) |tree| { |
| | 94 | self.huffman_tree = tree; |
| | 95 | } else if (literals.header.block_type == .treeless and self.huffman_tree == null) { |
| | 96 | return error.TreelessLiteralsFirst; |
| | 97 | } |
| | 98 | |
| | 99 | switch (literals.header.block_type) { |
| | 100 | .raw, .rle => {}, |
| | 101 | .compressed, .treeless => { |
| | 102 | self.literal_stream_index = 0; |
| | 103 | switch (literals.streams) { |
| | 104 | .one => |slice| try self.initLiteralStream(slice), |
| | 105 | .four => |streams| try self.initLiteralStream(streams[0]), |
| | 106 | } |
| | 107 | }, |
| | 108 | } |
| | 109 | |
| | 110 | if (sequences_header.sequence_count > 0) { |
| | 111 | var bytes_read = try self.updateFseTable( |
| | 112 | src, |
| | 113 | .literal, |
| | 114 | sequences_header.literal_lengths, |
| | 115 | ); |
| | 116 | |
| | 117 | bytes_read += try self.updateFseTable( |
| | 118 | src[bytes_read..], |
| | 119 | .offset, |
| | 120 | sequences_header.offsets, |
| | 121 | ); |
| | 122 | |
| | 123 | bytes_read += try self.updateFseTable( |
| | 124 | src[bytes_read..], |
| | 125 | .match, |
| | 126 | sequences_header.match_lengths, |
| | 127 | ); |
| | 128 | self.fse_tables_undefined = false; |
| | 129 | |
| | 130 | return bytes_read; |
| | 131 | } |
| | 132 | return 0; |
| | 133 | } |
| | 134 | |
| | 135 | pub fn readInitialFseState(self: *DecodeState, bit_reader: anytype) !void { |
| 88 | self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log); | 136 | self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log); |
| 89 | self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log); | 137 | self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log); |
| 90 | self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log); | 138 | self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log); |
| ... | @@ -130,7 +178,7 @@ const DecodeState = struct { | ... | @@ -130,7 +178,7 @@ const DecodeState = struct { |
| 130 | self: *DecodeState, | 178 | self: *DecodeState, |
| 131 | src: []const u8, | 179 | src: []const u8, |
| 132 | comptime choice: DataType, | 180 | comptime choice: DataType, |
| 133 | mode: Sequences.Header.Mode, | 181 | mode: SequencesSection.Header.Mode, |
| 134 | ) !usize { | 182 | ) !usize { |
| 135 | const field_name = @tagName(choice); | 183 | const field_name = @tagName(choice); |
| 136 | switch (mode) { | 184 | switch (mode) { |
| ... | @@ -213,7 +261,13 @@ const DecodeState = struct { | ... | @@ -213,7 +261,13 @@ const DecodeState = struct { |
| 213 | }; | 261 | }; |
| 214 | } | 262 | } |
| 215 | | 263 | |
| 216 | fn executeSequenceSlice(self: *DecodeState, dest: []u8, write_pos: usize, literals: Literals, sequence: Sequence) !void { | 264 | fn executeSequenceSlice( |
| | 265 | self: *DecodeState, |
| | 266 | dest: []u8, |
| | 267 | write_pos: usize, |
| | 268 | literals: LiteralsSection, |
| | 269 | sequence: Sequence, |
| | 270 | ) !void { |
| 217 | try self.decodeLiteralsSlice(dest[write_pos..], literals, sequence.literal_length); | 271 | try self.decodeLiteralsSlice(dest[write_pos..], literals, sequence.literal_length); |
| 218 | | 272 | |
| 219 | // TODO: should we validate offset against max_window_size? | 273 | // TODO: should we validate offset against max_window_size? |
| ... | @@ -225,7 +279,12 @@ const DecodeState = struct { | ... | @@ -225,7 +279,12 @@ const DecodeState = struct { |
| 225 | std.mem.copy(u8, dest[write_pos + sequence.literal_length ..], dest[copy_start..copy_end]); | 279 | std.mem.copy(u8, dest[write_pos + sequence.literal_length ..], dest[copy_start..copy_end]); |
| 226 | } | 280 | } |
| 227 | | 281 | |
| 228 | fn executeSequenceRingBuffer(self: *DecodeState, dest: *RingBuffer, literals: Literals, sequence: Sequence) !void { | 282 | fn executeSequenceRingBuffer( |
| | 283 | self: *DecodeState, |
| | 284 | dest: *RingBuffer, |
| | 285 | literals: LiteralsSection, |
| | 286 | sequence: Sequence, |
| | 287 | ) !void { |
| 229 | try self.decodeLiteralsRingBuffer(dest, literals, sequence.literal_length); | 288 | try self.decodeLiteralsRingBuffer(dest, literals, sequence.literal_length); |
| 230 | // TODO: check that ring buffer window is full enough for match copies | 289 | // TODO: check that ring buffer window is full enough for match copies |
| 231 | const copy_slice = dest.sliceAt(dest.write_index + dest.data.len - sequence.offset, sequence.match_length); | 290 | const copy_slice = dest.sliceAt(dest.write_index + dest.data.len - sequence.offset, sequence.match_length); |
| ... | @@ -234,11 +293,11 @@ const DecodeState = struct { | ... | @@ -234,11 +293,11 @@ const DecodeState = struct { |
| 234 | for (copy_slice.second) |b| dest.writeAssumeCapacity(b); | 293 | for (copy_slice.second) |b| dest.writeAssumeCapacity(b); |
| 235 | } | 294 | } |
| 236 | | 295 | |
| 237 | fn decodeSequenceSlice( | 296 | pub fn decodeSequenceSlice( |
| 238 | self: *DecodeState, | 297 | self: *DecodeState, |
| 239 | dest: []u8, | 298 | dest: []u8, |
| 240 | write_pos: usize, | 299 | write_pos: usize, |
| 241 | literals: Literals, | 300 | literals: LiteralsSection, |
| 242 | bit_reader: anytype, | 301 | bit_reader: anytype, |
| 243 | last_sequence: bool, | 302 | last_sequence: bool, |
| 244 | ) !usize { | 303 | ) !usize { |
| ... | @@ -255,10 +314,10 @@ const DecodeState = struct { | ... | @@ -255,10 +314,10 @@ const DecodeState = struct { |
| 255 | return sequence.match_length + sequence.literal_length; | 314 | return sequence.match_length + sequence.literal_length; |
| 256 | } | 315 | } |
| 257 | | 316 | |
| 258 | fn decodeSequenceRingBuffer( | 317 | pub fn decodeSequenceRingBuffer( |
| 259 | self: *DecodeState, | 318 | self: *DecodeState, |
| 260 | dest: *RingBuffer, | 319 | dest: *RingBuffer, |
| 261 | literals: Literals, | 320 | literals: LiteralsSection, |
| 262 | bit_reader: anytype, | 321 | bit_reader: anytype, |
| 263 | last_sequence: bool, | 322 | last_sequence: bool, |
| 264 | ) !usize { | 323 | ) !usize { |
| ... | @@ -280,7 +339,7 @@ const DecodeState = struct { | ... | @@ -280,7 +339,7 @@ const DecodeState = struct { |
| 280 | return sequence.match_length + sequence.literal_length; | 339 | return sequence.match_length + sequence.literal_length; |
| 281 | } | 340 | } |
| 282 | | 341 | |
| 283 | fn nextLiteralMultiStream(self: *DecodeState, literals: Literals) !void { | 342 | fn nextLiteralMultiStream(self: *DecodeState, literals: LiteralsSection) !void { |
| 284 | self.literal_stream_index += 1; | 343 | self.literal_stream_index += 1; |
| 285 | try self.initLiteralStream(literals.streams.four[self.literal_stream_index]); | 344 | try self.initLiteralStream(literals.streams.four[self.literal_stream_index]); |
| 286 | } | 345 | } |
| ... | @@ -290,7 +349,7 @@ const DecodeState = struct { | ... | @@ -290,7 +349,7 @@ const DecodeState = struct { |
| 290 | try self.literal_stream_reader.init(bytes); | 349 | try self.literal_stream_reader.init(bytes); |
| 291 | } | 350 | } |
| 292 | | 351 | |
| 293 | fn decodeLiteralsSlice(self: *DecodeState, dest: []u8, literals: Literals, len: usize) !void { | 352 | pub fn decodeLiteralsSlice(self: *DecodeState, dest: []u8, literals: LiteralsSection, len: usize) !void { |
| 294 | if (self.literal_written_count + len > literals.header.regenerated_size) return error.MalformedLiteralsLength; | 353 | if (self.literal_written_count + len > literals.header.regenerated_size) return error.MalformedLiteralsLength; |
| 295 | switch (literals.header.block_type) { | 354 | switch (literals.header.block_type) { |
| 296 | .raw => { | 355 | .raw => { |
| ... | @@ -310,7 +369,7 @@ const DecodeState = struct { | ... | @@ -310,7 +369,7 @@ const DecodeState = struct { |
| 310 | // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4; | 369 | // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4; |
| 311 | const huffman_tree = self.huffman_tree orelse unreachable; | 370 | const huffman_tree = self.huffman_tree orelse unreachable; |
| 312 | const max_bit_count = huffman_tree.max_bit_count; | 371 | const max_bit_count = huffman_tree.max_bit_count; |
| 313 | const starting_bit_count = Literals.HuffmanTree.weightToBitCount( | 372 | const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount( |
| 314 | huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight, | 373 | huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight, |
| 315 | max_bit_count, | 374 | max_bit_count, |
| 316 | ); | 375 | ); |
| ... | @@ -345,7 +404,7 @@ const DecodeState = struct { | ... | @@ -345,7 +404,7 @@ const DecodeState = struct { |
| 345 | }, | 404 | }, |
| 346 | .index => |index| { | 405 | .index => |index| { |
| 347 | huffman_tree_index = index; | 406 | huffman_tree_index = index; |
| 348 | const bit_count = Literals.HuffmanTree.weightToBitCount( | 407 | const bit_count = LiteralsSection.HuffmanTree.weightToBitCount( |
| 349 | huffman_tree.nodes[index].weight, | 408 | huffman_tree.nodes[index].weight, |
| 350 | max_bit_count, | 409 | max_bit_count, |
| 351 | ); | 410 | ); |
| ... | @@ -359,7 +418,7 @@ const DecodeState = struct { | ... | @@ -359,7 +418,7 @@ const DecodeState = struct { |
| 359 | } | 418 | } |
| 360 | } | 419 | } |
| 361 | | 420 | |
| 362 | fn decodeLiteralsRingBuffer(self: *DecodeState, dest: *RingBuffer, literals: Literals, len: usize) !void { | 421 | pub fn decodeLiteralsRingBuffer(self: *DecodeState, dest: *RingBuffer, literals: LiteralsSection, len: usize) !void { |
| 363 | if (self.literal_written_count + len > literals.header.regenerated_size) return error.MalformedLiteralsLength; | 422 | if (self.literal_written_count + len > literals.header.regenerated_size) return error.MalformedLiteralsLength; |
| 364 | switch (literals.header.block_type) { | 423 | switch (literals.header.block_type) { |
| 365 | .raw => { | 424 | .raw => { |
| ... | @@ -378,7 +437,7 @@ const DecodeState = struct { | ... | @@ -378,7 +437,7 @@ const DecodeState = struct { |
| 378 | // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4; | 437 | // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4; |
| 379 | const huffman_tree = self.huffman_tree orelse unreachable; | 438 | const huffman_tree = self.huffman_tree orelse unreachable; |
| 380 | const max_bit_count = huffman_tree.max_bit_count; | 439 | const max_bit_count = huffman_tree.max_bit_count; |
| 381 | const starting_bit_count = Literals.HuffmanTree.weightToBitCount( | 440 | const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount( |
| 382 | huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight, | 441 | huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight, |
| 383 | max_bit_count, | 442 | max_bit_count, |
| 384 | ); | 443 | ); |
| ... | @@ -413,7 +472,7 @@ const DecodeState = struct { | ... | @@ -413,7 +472,7 @@ const DecodeState = struct { |
| 413 | }, | 472 | }, |
| 414 | .index => |index| { | 473 | .index => |index| { |
| 415 | huffman_tree_index = index; | 474 | huffman_tree_index = index; |
| 416 | const bit_count = Literals.HuffmanTree.weightToBitCount( | 475 | const bit_count = LiteralsSection.HuffmanTree.weightToBitCount( |
| 417 | huffman_tree.nodes[index].weight, | 476 | huffman_tree.nodes[index].weight, |
| 418 | max_bit_count, | 477 | max_bit_count, |
| 419 | ); | 478 | ); |
| ... | @@ -647,54 +706,6 @@ fn decodeRleBlockRingBuffer(dest: *RingBuffer, src: []const u8, block_size: u21, | ... | @@ -647,54 +706,6 @@ fn decodeRleBlockRingBuffer(dest: *RingBuffer, src: []const u8, block_size: u21, |
| 647 | return block_size; | 706 | return block_size; |
| 648 | } | 707 | } |
| 649 | | 708 | |
| 650 | fn prepareDecodeState( | | |
| 651 | decode_state: *DecodeState, | | |
| 652 | src: []const u8, | | |
| 653 | literals: Literals, | | |
| 654 | sequences_header: Sequences.Header, | | |
| 655 | ) !usize { | | |
| 656 | if (literals.huffman_tree) |tree| { | | |
| 657 | decode_state.huffman_tree = tree; | | |
| 658 | } else if (literals.header.block_type == .treeless and decode_state.huffman_tree == null) { | | |
| 659 | return error.TreelessLiteralsFirst; | | |
| 660 | } | | |
| 661 | | | |
| 662 | switch (literals.header.block_type) { | | |
| 663 | .raw, .rle => {}, | | |
| 664 | .compressed, .treeless => { | | |
| 665 | decode_state.literal_stream_index = 0; | | |
| 666 | switch (literals.streams) { | | |
| 667 | .one => |slice| try decode_state.initLiteralStream(slice), | | |
| 668 | .four => |streams| try decode_state.initLiteralStream(streams[0]), | | |
| 669 | } | | |
| 670 | }, | | |
| 671 | } | | |
| 672 | | | |
| 673 | if (sequences_header.sequence_count > 0) { | | |
| 674 | var bytes_read = try decode_state.updateFseTable( | | |
| 675 | src, | | |
| 676 | .literal, | | |
| 677 | sequences_header.literal_lengths, | | |
| 678 | ); | | |
| 679 | | | |
| 680 | bytes_read += try decode_state.updateFseTable( | | |
| 681 | src[bytes_read..], | | |
| 682 | .offset, | | |
| 683 | sequences_header.offsets, | | |
| 684 | ); | | |
| 685 | | | |
| 686 | bytes_read += try decode_state.updateFseTable( | | |
| 687 | src[bytes_read..], | | |
| 688 | .match, | | |
| 689 | sequences_header.match_lengths, | | |
| 690 | ); | | |
| 691 | decode_state.fse_tables_undefined = false; | | |
| 692 | | | |
| 693 | return bytes_read; | | |
| 694 | } | | |
| 695 | return 0; | | |
| 696 | } | | |
| 697 | | | |
| 698 | pub fn decodeBlock( | 709 | pub fn decodeBlock( |
| 699 | dest: []u8, | 710 | dest: []u8, |
| 700 | src: []const u8, | 711 | src: []const u8, |
| ... | @@ -715,7 +726,7 @@ pub fn decodeBlock( | ... | @@ -715,7 +726,7 @@ pub fn decodeBlock( |
| 715 | const literals = try decodeLiteralsSection(src, &bytes_read); | 726 | const literals = try decodeLiteralsSection(src, &bytes_read); |
| 716 | const sequences_header = try decodeSequencesHeader(src[bytes_read..], &bytes_read); | 727 | const sequences_header = try decodeSequencesHeader(src[bytes_read..], &bytes_read); |
| 717 | | 728 | |
| 718 | bytes_read += try prepareDecodeState(decode_state, src[bytes_read..], literals, sequences_header); | 729 | bytes_read += try decode_state.prepare(src[bytes_read..], literals, sequences_header); |
| 719 | | 730 | |
| 720 | var bytes_written: usize = 0; | 731 | var bytes_written: usize = 0; |
| 721 | if (sequences_header.sequence_count > 0) { | 732 | if (sequences_header.sequence_count > 0) { |
| ... | @@ -723,7 +734,7 @@ pub fn decodeBlock( | ... | @@ -723,7 +734,7 @@ pub fn decodeBlock( |
| 723 | var bit_stream: ReverseBitReader = undefined; | 734 | var bit_stream: ReverseBitReader = undefined; |
| 724 | try bit_stream.init(bit_stream_bytes); | 735 | try bit_stream.init(bit_stream_bytes); |
| 725 | | 736 | |
| 726 | try decode_state.readInitialState(&bit_stream); | 737 | try decode_state.readInitialFseState(&bit_stream); |
| 727 | | 738 | |
| 728 | var i: usize = 0; | 739 | var i: usize = 0; |
| 729 | while (i < sequences_header.sequence_count) : (i += 1) { | 740 | while (i < sequences_header.sequence_count) : (i += 1) { |
| ... | @@ -780,7 +791,7 @@ pub fn decodeBlockRingBuffer( | ... | @@ -780,7 +791,7 @@ pub fn decodeBlockRingBuffer( |
| 780 | const literals = try decodeLiteralsSection(src, &bytes_read); | 791 | const literals = try decodeLiteralsSection(src, &bytes_read); |
| 781 | const sequences_header = try decodeSequencesHeader(src[bytes_read..], &bytes_read); | 792 | const sequences_header = try decodeSequencesHeader(src[bytes_read..], &bytes_read); |
| 782 | | 793 | |
| 783 | bytes_read += try prepareDecodeState(decode_state, src[bytes_read..], literals, sequences_header); | 794 | bytes_read += try decode_state.prepare(src[bytes_read..], literals, sequences_header); |
| 784 | | 795 | |
| 785 | var bytes_written: usize = 0; | 796 | var bytes_written: usize = 0; |
| 786 | if (sequences_header.sequence_count > 0) { | 797 | if (sequences_header.sequence_count > 0) { |
| ... | @@ -788,7 +799,7 @@ pub fn decodeBlockRingBuffer( | ... | @@ -788,7 +799,7 @@ pub fn decodeBlockRingBuffer( |
| 788 | var bit_stream: ReverseBitReader = undefined; | 799 | var bit_stream: ReverseBitReader = undefined; |
| 789 | try bit_stream.init(bit_stream_bytes); | 800 | try bit_stream.init(bit_stream_bytes); |
| 790 | | 801 | |
| 791 | try decode_state.readInitialState(&bit_stream); | 802 | try decode_state.readInitialFseState(&bit_stream); |
| 792 | | 803 | |
| 793 | var i: usize = 0; | 804 | var i: usize = 0; |
| 794 | while (i < sequences_header.sequence_count) : (i += 1) { | 805 | while (i < sequences_header.sequence_count) : (i += 1) { |
| ... | @@ -928,7 +939,7 @@ pub fn decodeBlockHeader(src: *const [3]u8) frame.ZStandard.Block.Header { | ... | @@ -928,7 +939,7 @@ pub fn decodeBlockHeader(src: *const [3]u8) frame.ZStandard.Block.Header { |
| 928 | }; | 939 | }; |
| 929 | } | 940 | } |
| 930 | | 941 | |
| 931 | pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals { | 942 | pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !LiteralsSection { |
| 932 | // TODO: we probably want to enable safety for release-fast and release-small (or insert custom checks) | 943 | // TODO: we probably want to enable safety for release-fast and release-small (or insert custom checks) |
| 933 | var bytes_read: usize = 0; | 944 | var bytes_read: usize = 0; |
| 934 | const header = decodeLiteralsHeader(src, &bytes_read); | 945 | const header = decodeLiteralsHeader(src, &bytes_read); |
| ... | @@ -936,7 +947,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals | ... | @@ -936,7 +947,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals |
| 936 | .raw => { | 947 | .raw => { |
| 937 | const stream = src[bytes_read .. bytes_read + header.regenerated_size]; | 948 | const stream = src[bytes_read .. bytes_read + header.regenerated_size]; |
| 938 | consumed_count.* += header.regenerated_size + bytes_read; | 949 | consumed_count.* += header.regenerated_size + bytes_read; |
| 939 | return Literals{ | 950 | return LiteralsSection{ |
| 940 | .header = header, | 951 | .header = header, |
| 941 | .huffman_tree = null, | 952 | .huffman_tree = null, |
| 942 | .streams = .{ .one = stream }, | 953 | .streams = .{ .one = stream }, |
| ... | @@ -945,7 +956,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals | ... | @@ -945,7 +956,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals |
| 945 | .rle => { | 956 | .rle => { |
| 946 | const stream = src[bytes_read .. bytes_read + 1]; | 957 | const stream = src[bytes_read .. bytes_read + 1]; |
| 947 | consumed_count.* += 1 + bytes_read; | 958 | consumed_count.* += 1 + bytes_read; |
| 948 | return Literals{ | 959 | return LiteralsSection{ |
| 949 | .header = header, | 960 | .header = header, |
| 950 | .huffman_tree = null, | 961 | .huffman_tree = null, |
| 951 | .streams = .{ .one = stream }, | 962 | .streams = .{ .one = stream }, |
| ... | @@ -966,7 +977,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals | ... | @@ -966,7 +977,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals |
| 966 | const stream = src[bytes_read .. bytes_read + total_streams_size]; | 977 | const stream = src[bytes_read .. bytes_read + total_streams_size]; |
| 967 | bytes_read += total_streams_size; | 978 | bytes_read += total_streams_size; |
| 968 | consumed_count.* += bytes_read; | 979 | consumed_count.* += bytes_read; |
| 969 | return Literals{ | 980 | return LiteralsSection{ |
| 970 | .header = header, | 981 | .header = header, |
| 971 | .huffman_tree = huffman_tree, | 982 | .huffman_tree = huffman_tree, |
| 972 | .streams = .{ .one = stream }, | 983 | .streams = .{ .one = stream }, |
| ... | @@ -988,7 +999,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals | ... | @@ -988,7 +999,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals |
| 988 | | 999 | |
| 989 | consumed_count.* += total_streams_size + bytes_read; | 1000 | consumed_count.* += total_streams_size + bytes_read; |
| 990 | | 1001 | |
| 991 | return Literals{ | 1002 | return LiteralsSection{ |
| 992 | .header = header, | 1003 | .header = header, |
| 993 | .huffman_tree = huffman_tree, | 1004 | .huffman_tree = huffman_tree, |
| 994 | .streams = .{ .four = .{ | 1005 | .streams = .{ .four = .{ |
| ... | @@ -1002,7 +1013,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals | ... | @@ -1002,7 +1013,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals |
| 1002 | } | 1013 | } |
| 1003 | } | 1014 | } |
| 1004 | | 1015 | |
| 1005 | fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanTree { | 1016 | fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !LiteralsSection.HuffmanTree { |
| 1006 | var bytes_read: usize = 0; | 1017 | var bytes_read: usize = 0; |
| 1007 | bytes_read += 1; | 1018 | bytes_read += 1; |
| 1008 | const header = src[0]; | 1019 | const header = src[0]; |
| ... | @@ -1094,7 +1105,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT | ... | @@ -1094,7 +1105,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT |
| 1094 | weights[symbol_count - 1] = @intCast(u4, std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1); | 1105 | weights[symbol_count - 1] = @intCast(u4, std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1); |
| 1095 | log.debug("weights[{d}] = {d}", .{ symbol_count - 1, weights[symbol_count - 1] }); | 1106 | log.debug("weights[{d}] = {d}", .{ symbol_count - 1, weights[symbol_count - 1] }); |
| 1096 | | 1107 | |
| 1097 | var weight_sorted_prefixed_symbols: [256]Literals.HuffmanTree.PrefixedSymbol = undefined; | 1108 | var weight_sorted_prefixed_symbols: [256]LiteralsSection.HuffmanTree.PrefixedSymbol = undefined; |
| 1098 | for (weight_sorted_prefixed_symbols[0..symbol_count]) |_, i| { | 1109 | for (weight_sorted_prefixed_symbols[0..symbol_count]) |_, i| { |
| 1099 | weight_sorted_prefixed_symbols[i] = .{ | 1110 | weight_sorted_prefixed_symbols[i] = .{ |
| 1100 | .symbol = @intCast(u8, i), | 1111 | .symbol = @intCast(u8, i), |
| ... | @@ -1104,7 +1115,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT | ... | @@ -1104,7 +1115,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT |
| 1104 | } | 1115 | } |
| 1105 | | 1116 | |
| 1106 | std.sort.sort( | 1117 | std.sort.sort( |
| 1107 | Literals.HuffmanTree.PrefixedSymbol, | 1118 | LiteralsSection.HuffmanTree.PrefixedSymbol, |
| 1108 | weight_sorted_prefixed_symbols[0..symbol_count], | 1119 | weight_sorted_prefixed_symbols[0..symbol_count], |
| 1109 | weights, | 1120 | weights, |
| 1110 | lessThanByWeight, | 1121 | lessThanByWeight, |
| ... | @@ -1137,7 +1148,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT | ... | @@ -1137,7 +1148,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT |
| 1137 | } | 1148 | } |
| 1138 | } | 1149 | } |
| 1139 | consumed_count.* += bytes_read; | 1150 | consumed_count.* += bytes_read; |
| 1140 | const tree = Literals.HuffmanTree{ | 1151 | const tree = LiteralsSection.HuffmanTree{ |
| 1141 | .max_bit_count = max_number_of_bits, | 1152 | .max_bit_count = max_number_of_bits, |
| 1142 | .symbol_count_minus_one = @intCast(u8, prefixed_symbol_count - 1), | 1153 | .symbol_count_minus_one = @intCast(u8, prefixed_symbol_count - 1), |
| 1143 | .nodes = weight_sorted_prefixed_symbols, | 1154 | .nodes = weight_sorted_prefixed_symbols, |
| ... | @@ -1148,8 +1159,8 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT | ... | @@ -1148,8 +1159,8 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT |
| 1148 | | 1159 | |
| 1149 | fn lessThanByWeight( | 1160 | fn lessThanByWeight( |
| 1150 | weights: [256]u4, | 1161 | weights: [256]u4, |
| 1151 | lhs: Literals.HuffmanTree.PrefixedSymbol, | 1162 | lhs: LiteralsSection.HuffmanTree.PrefixedSymbol, |
| 1152 | rhs: Literals.HuffmanTree.PrefixedSymbol, | 1163 | rhs: LiteralsSection.HuffmanTree.PrefixedSymbol, |
| 1153 | ) bool { | 1164 | ) bool { |
| 1154 | // NOTE: this function relies on the use of a stable sorting algorithm, | 1165 | // NOTE: this function relies on the use of a stable sorting algorithm, |
| 1155 | // otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs; | 1166 | // otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs; |
| ... | @@ -1157,11 +1168,11 @@ fn lessThanByWeight( | ... | @@ -1157,11 +1168,11 @@ fn lessThanByWeight( |
| 1157 | return weights[lhs.symbol] < weights[rhs.symbol]; | 1168 | return weights[lhs.symbol] < weights[rhs.symbol]; |
| 1158 | } | 1169 | } |
| 1159 | | 1170 | |
| 1160 | pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) Literals.Header { | 1171 | pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) LiteralsSection.Header { |
| 1161 | // TODO: we probably want to enable safety for release-fast and release-small (or insert custom checks) | 1172 | // TODO: we probably want to enable safety for release-fast and release-small (or insert custom checks) |
| 1162 | const start = consumed_count.*; | 1173 | const start = consumed_count.*; |
| 1163 | const byte0 = src[0]; | 1174 | const byte0 = src[0]; |
| 1164 | const block_type = @intToEnum(Literals.BlockType, byte0 & 0b11); | 1175 | const block_type = @intToEnum(LiteralsSection.BlockType, byte0 & 0b11); |
| 1165 | const size_format = @intCast(u2, (byte0 & 0b1100) >> 2); | 1176 | const size_format = @intCast(u2, (byte0 & 0b1100) >> 2); |
| 1166 | var regenerated_size: u20 = undefined; | 1177 | var regenerated_size: u20 = undefined; |
| 1167 | var compressed_size: ?u18 = null; | 1178 | var compressed_size: ?u18 = null; |
| ... | @@ -1220,7 +1231,7 @@ pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) Literals.He | ... | @@ -1220,7 +1231,7 @@ pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) Literals.He |
| 1220 | compressed_size, | 1231 | compressed_size, |
| 1221 | }, | 1232 | }, |
| 1222 | ); | 1233 | ); |
| 1223 | return Literals.Header{ | 1234 | return LiteralsSection.Header{ |
| 1224 | .block_type = block_type, | 1235 | .block_type = block_type, |
| 1225 | .size_format = size_format, | 1236 | .size_format = size_format, |
| 1226 | .regenerated_size = regenerated_size, | 1237 | .regenerated_size = regenerated_size, |
| ... | @@ -1228,7 +1239,7 @@ pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) Literals.He | ... | @@ -1228,7 +1239,7 @@ pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) Literals.He |
| 1228 | }; | 1239 | }; |
| 1229 | } | 1240 | } |
| 1230 | | 1241 | |
| 1231 | fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Header { | 1242 | pub fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !SequencesSection.Header { |
| 1232 | var sequence_count: u24 = undefined; | 1243 | var sequence_count: u24 = undefined; |
| 1233 | | 1244 | |
| 1234 | var bytes_read: usize = 0; | 1245 | var bytes_read: usize = 0; |
| ... | @@ -1237,7 +1248,7 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea | ... | @@ -1237,7 +1248,7 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea |
| 1237 | bytes_read += 1; | 1248 | bytes_read += 1; |
| 1238 | log.debug("decoded sequences header '{}': sequence count = 0", .{std.fmt.fmtSliceHexUpper(src[0..bytes_read])}); | 1249 | log.debug("decoded sequences header '{}': sequence count = 0", .{std.fmt.fmtSliceHexUpper(src[0..bytes_read])}); |
| 1239 | consumed_count.* += bytes_read; | 1250 | consumed_count.* += bytes_read; |
| 1240 | return Sequences.Header{ | 1251 | return SequencesSection.Header{ |
| 1241 | .sequence_count = 0, | 1252 | .sequence_count = 0, |
| 1242 | .offsets = undefined, | 1253 | .offsets = undefined, |
| 1243 | .match_lengths = undefined, | 1254 | .match_lengths = undefined, |
| ... | @@ -1258,9 +1269,9 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea | ... | @@ -1258,9 +1269,9 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea |
| 1258 | bytes_read += 1; | 1269 | bytes_read += 1; |
| 1259 | | 1270 | |
| 1260 | consumed_count.* += bytes_read; | 1271 | consumed_count.* += bytes_read; |
| 1261 | const matches_mode = @intToEnum(Sequences.Header.Mode, (compression_modes & 0b00001100) >> 2); | 1272 | const matches_mode = @intToEnum(SequencesSection.Header.Mode, (compression_modes & 0b00001100) >> 2); |
| 1262 | const offsets_mode = @intToEnum(Sequences.Header.Mode, (compression_modes & 0b00110000) >> 4); | 1273 | const offsets_mode = @intToEnum(SequencesSection.Header.Mode, (compression_modes & 0b00110000) >> 4); |
| 1263 | const literal_mode = @intToEnum(Sequences.Header.Mode, (compression_modes & 0b11000000) >> 6); | 1274 | const literal_mode = @intToEnum(SequencesSection.Header.Mode, (compression_modes & 0b11000000) >> 6); |
| 1264 | log.debug("decoded sequences header '{}': (sc={d},o={s},m={s},l={s})", .{ | 1275 | log.debug("decoded sequences header '{}': (sc={d},o={s},m={s},l={s})", .{ |
| 1265 | std.fmt.fmtSliceHexUpper(src[0..bytes_read]), | 1276 | std.fmt.fmtSliceHexUpper(src[0..bytes_read]), |
| 1266 | sequence_count, | 1277 | sequence_count, |
| ... | @@ -1270,7 +1281,7 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea | ... | @@ -1270,7 +1281,7 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea |
| 1270 | }); | 1281 | }); |
| 1271 | if (compression_modes & 0b11 != 0) return error.ReservedBitSet; | 1282 | if (compression_modes & 0b11 != 0) return error.ReservedBitSet; |
| 1272 | | 1283 | |
| 1273 | return Sequences.Header{ | 1284 | return SequencesSection.Header{ |
| 1274 | .sequence_count = sequence_count, | 1285 | .sequence_count = sequence_count, |
| 1275 | .offsets = offsets_mode, | 1286 | .offsets = offsets_mode, |
| 1276 | .match_lengths = matches_mode, | 1287 | .match_lengths = matches_mode, |
| ... | @@ -1428,25 +1439,25 @@ const ReversedByteReader = struct { | ... | @@ -1428,25 +1439,25 @@ const ReversedByteReader = struct { |
| 1428 | } | 1439 | } |
| 1429 | }; | 1440 | }; |
| 1430 | | 1441 | |
| 1431 | const ReverseBitReader = struct { | 1442 | pub const ReverseBitReader = struct { |
| 1432 | byte_reader: ReversedByteReader, | 1443 | byte_reader: ReversedByteReader, |
| 1433 | bit_reader: std.io.BitReader(.Big, ReversedByteReader.Reader), | 1444 | bit_reader: std.io.BitReader(.Big, ReversedByteReader.Reader), |
| 1434 | | 1445 | |
| 1435 | fn init(self: *ReverseBitReader, bytes: []const u8) !void { | 1446 | pub fn init(self: *ReverseBitReader, bytes: []const u8) !void { |
| 1436 | self.byte_reader = ReversedByteReader.init(bytes); | 1447 | self.byte_reader = ReversedByteReader.init(bytes); |
| 1437 | self.bit_reader = std.io.bitReader(.Big, self.byte_reader.reader()); | 1448 | self.bit_reader = std.io.bitReader(.Big, self.byte_reader.reader()); |
| 1438 | while (0 == self.readBitsNoEof(u1, 1) catch return error.BitStreamHasNoStartBit) {} | 1449 | while (0 == self.readBitsNoEof(u1, 1) catch return error.BitStreamHasNoStartBit) {} |
| 1439 | } | 1450 | } |
| 1440 | | 1451 | |
| 1441 | fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: usize) !U { | 1452 | pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: usize) !U { |
| 1442 | return self.bit_reader.readBitsNoEof(U, num_bits); | 1453 | return self.bit_reader.readBitsNoEof(U, num_bits); |
| 1443 | } | 1454 | } |
| 1444 | | 1455 | |
| 1445 | fn readBits(self: *@This(), comptime U: type, num_bits: usize, out_bits: *usize) !U { | 1456 | pub fn readBits(self: *@This(), comptime U: type, num_bits: usize, out_bits: *usize) !U { |
| 1446 | return try self.bit_reader.readBits(U, num_bits, out_bits); | 1457 | return try self.bit_reader.readBits(U, num_bits, out_bits); |
| 1447 | } | 1458 | } |
| 1448 | | 1459 | |
| 1449 | fn alignToByte(self: *@This()) void { | 1460 | pub fn alignToByte(self: *@This()) void { |
| 1450 | self.bit_reader.alignToByte(); | 1461 | self.bit_reader.alignToByte(); |
| 1451 | } | 1462 | } |
| 1452 | }; | 1463 | }; |
| ... | @@ -1514,7 +1525,7 @@ fn dumpFseTable(prefix: []const u8, table: []const Table.Fse) void { | ... | @@ -1514,7 +1525,7 @@ fn dumpFseTable(prefix: []const u8, table: []const Table.Fse) void { |
| 1514 | } | 1525 | } |
| 1515 | } | 1526 | } |
| 1516 | | 1527 | |
| 1517 | fn dumpHuffmanTree(tree: Literals.HuffmanTree) void { | 1528 | fn dumpHuffmanTree(tree: LiteralsSection.HuffmanTree) void { |
| 1518 | log.debug("Huffman tree: max bit count = {}, symbol count = {}", .{ tree.max_bit_count, tree.symbol_count_minus_one + 1 }); | 1529 | log.debug("Huffman tree: max bit count = {}, symbol count = {}", .{ tree.max_bit_count, tree.symbol_count_minus_one + 1 }); |
| 1519 | for (tree.nodes[0 .. tree.symbol_count_minus_one + 1]) |node| { | 1530 | for (tree.nodes[0 .. tree.symbol_count_minus_one + 1]) |node| { |
| 1520 | log.debug("symbol = {[symbol]d}, prefix = {[prefix]d}, weight = {[weight]d}", node); | 1531 | log.debug("symbol = {[symbol]d}, prefix = {[prefix]d}, weight = {[weight]d}", node); |