| ... | @@ -1,6 +1,5 @@ | ... | @@ -1,6 +1,5 @@ |
| 1 | const std = @import("std"); | 1 | const std = @import("std"); |
| 2 | const assert = std.debug.assert; | 2 | const assert = std.debug.assert; |
| 3 | const Allocator = std.mem.Allocator; | | |
| 4 | const RingBuffer = std.RingBuffer; | 3 | const RingBuffer = std.RingBuffer; |
| 5 | | 4 | |
| 6 | const types = @import("types.zig"); | 5 | const types = @import("types.zig"); |
| ... | @@ -117,41 +116,6 @@ pub fn decode(dest: []u8, src: []const u8, verify_checksum: bool) error{ | ... | @@ -117,41 +116,6 @@ pub fn decode(dest: []u8, src: []const u8, verify_checksum: bool) error{ |
| 117 | return write_count; | 116 | return write_count; |
| 118 | } | 117 | } |
| 119 | | 118 | |
| 120 | /// Decodes a stream of frames from `src`; returns the decoded bytes. The stream | | |
| 121 | /// should not have extra trailing bytes - either all bytes in `src` will be | | |
| 122 | /// decoded, or an error will be returned. | | |
| 123 | /// | | |
| 124 | /// Errors returned: | | |
| 125 | /// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that | | |
| 126 | /// uses a dictionary | | |
| 127 | /// - `error.MalformedFrame` if a frame in `src` is invalid | | |
| 128 | /// - `error.OutOfMemory` if `allocator` cannot allocate enough memory | | |
| 129 | pub fn decodeAlloc( | | |
| 130 | allocator: Allocator, | | |
| 131 | src: []const u8, | | |
| 132 | verify_checksum: bool, | | |
| 133 | window_size_max: usize, | | |
| 134 | ) error{ DictionaryIdFlagUnsupported, MalformedFrame, OutOfMemory }![]u8 { | | |
| 135 | var result = std.ArrayList(u8).init(allocator); | | |
| 136 | errdefer result.deinit(); | | |
| 137 | | | |
| 138 | var read_count: usize = 0; | | |
| 139 | while (read_count < src.len) { | | |
| 140 | read_count += decodeFrameArrayList( | | |
| 141 | allocator, | | |
| 142 | &result, | | |
| 143 | src[read_count..], | | |
| 144 | verify_checksum, | | |
| 145 | window_size_max, | | |
| 146 | ) catch |err| switch (err) { | | |
| 147 | error.OutOfMemory => return error.OutOfMemory, | | |
| 148 | error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported, | | |
| 149 | else => return error.MalformedFrame, | | |
| 150 | }; | | |
| 151 | } | | |
| 152 | return result.toOwnedSlice(); | | |
| 153 | } | | |
| 154 | | | |
| 155 | /// Decodes the frame at the start of `src` into `dest`. Returns the number of | 119 | /// Decodes the frame at the start of `src` into `dest`. Returns the number of |
| 156 | /// bytes read from `src` and written to `dest`. This function can only decode | 120 | /// bytes read from `src` and written to `dest`. This function can only decode |
| 157 | /// frames that declare the decompressed content size. | 121 | /// frames that declare the decompressed content size. |
| ... | @@ -207,58 +171,6 @@ pub fn decodeFrame( | ... | @@ -207,58 +171,6 @@ pub fn decodeFrame( |
| 207 | } | 171 | } |
| 208 | } | 172 | } |
| 209 | | 173 | |
| 210 | /// Decodes the frame at the start of `src` into `dest`. Returns the number of | | |
| 211 | /// bytes read from `src`. | | |
| 212 | /// | | |
| 213 | /// Errors returned: | | |
| 214 | /// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic | | |
| 215 | /// number for a Zstandard or skippable frame | | |
| 216 | /// - `error.WindowSizeUnknown` if the frame does not have a valid window size | | |
| 217 | /// - `error.WindowTooLarge` if the window size is larger than | | |
| 218 | /// `window_size_max` | | |
| 219 | /// - `error.ContentSizeTooLarge` if the frame header indicates a content size | | |
| 220 | /// that is larger than `std.math.maxInt(usize)` | | |
| 221 | /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary | | |
| 222 | /// - `error.ChecksumFailure` if `verify_checksum` is true and the frame | | |
| 223 | /// contains a checksum that does not match the checksum of the decompressed | | |
| 224 | /// data | | |
| 225 | /// - `error.ReservedBitSet` if any of the reserved bits of the frame header | | |
| 226 | /// are set | | |
| 227 | /// - `error.EndOfStream` if `src` does not contain a complete frame | | |
| 228 | /// - `error.BadContentSize` if the content size declared by the frame does | | |
| 229 | /// not equal the actual size of decompressed data | | |
| 230 | /// - `error.OutOfMemory` if `allocator` cannot allocate enough memory | | |
| 231 | /// - an error in `block.Error` if there are errors decoding a block | | |
| 232 | /// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a | | |
| 233 | /// size greater than `src.len` | | |
| 234 | pub fn decodeFrameArrayList( | | |
| 235 | allocator: Allocator, | | |
| 236 | dest: *std.ArrayList(u8), | | |
| 237 | src: []const u8, | | |
| 238 | verify_checksum: bool, | | |
| 239 | window_size_max: usize, | | |
| 240 | ) (error{ BadMagic, OutOfMemory, SkippableSizeTooLarge } || FrameContext.Error || FrameError)!usize { | | |
| 241 | var fbs: std.io.FixedBufferStream = .{ .buffer = src }; | | |
| 242 | const reader = fbs.reader(); | | |
| 243 | const magic = try reader.readInt(u32, .little); | | |
| 244 | switch (try frameType(magic)) { | | |
| 245 | .zstandard => return decodeZstandardFrameArrayList( | | |
| 246 | allocator, | | |
| 247 | dest, | | |
| 248 | src, | | |
| 249 | verify_checksum, | | |
| 250 | window_size_max, | | |
| 251 | ), | | |
| 252 | .skippable => { | | |
| 253 | const content_size = try fbs.reader().readInt(u32, .little); | | |
| 254 | if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge; | | |
| 255 | const read_count = @as(usize, content_size) + 8; | | |
| 256 | if (read_count > src.len) return error.SkippableSizeTooLarge; | | |
| 257 | return read_count; | | |
| 258 | }, | | |
| 259 | } | | |
| 260 | } | | |
| 261 | | | |
| 262 | /// Returns the frame checksum corresponding to the data fed into `hasher` | 174 | /// Returns the frame checksum corresponding to the data fed into `hasher` |
| 263 | pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 { | 175 | pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 { |
| 264 | const hash = hasher.final(); | 176 | const hash = hasher.final(); |
| ... | @@ -420,117 +332,6 @@ pub const FrameContext = struct { | ... | @@ -420,117 +332,6 @@ pub const FrameContext = struct { |
| 420 | } | 332 | } |
| 421 | }; | 333 | }; |
| 422 | | 334 | |
| 423 | /// Decode a Zstandard from from `src` and return number of bytes read; see | | |
| 424 | /// `decodeZstandardFrame()`. The first four bytes of `src` must be the magic | | |
| 425 | /// number for a Zstandard frame. | | |
| 426 | /// | | |
| 427 | /// Errors returned: | | |
| 428 | /// - `error.WindowSizeUnknown` if the frame does not have a valid window size | | |
| 429 | /// - `error.WindowTooLarge` if the window size is larger than | | |
| 430 | /// `window_size_max` | | |
| 431 | /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary | | |
| 432 | /// - `error.ContentSizeTooLarge` if the frame header indicates a content size | | |
| 433 | /// that is larger than `std.math.maxInt(usize)` | | |
| 434 | /// - `error.ChecksumFailure` if `verify_checksum` is true and the frame | | |
| 435 | /// contains a checksum that does not match the checksum of the decompressed | | |
| 436 | /// data | | |
| 437 | /// - `error.ReservedBitSet` if the reserved bit of the frame header is set | | |
| 438 | /// - `error.EndOfStream` if `src` does not contain a complete frame | | |
| 439 | /// - `error.OutOfMemory` if `allocator` cannot allocate enough memory | | |
| 440 | /// - an error in `block.Error` if there are errors decoding a block | | |
| 441 | /// - `error.BadContentSize` if the content size declared by the frame does | | |
| 442 | /// not equal the size of decompressed data | | |
| 443 | pub fn decodeZstandardFrameArrayList( | | |
| 444 | allocator: Allocator, | | |
| 445 | dest: *std.ArrayList(u8), | | |
| 446 | src: []const u8, | | |
| 447 | verify_checksum: bool, | | |
| 448 | window_size_max: usize, | | |
| 449 | ) (error{OutOfMemory} || FrameContext.Error || FrameError)!usize { | | |
| 450 | assert(std.mem.readInt(u32, src[0..4], .little) == frame.Zstandard.magic_number); | | |
| 451 | var consumed_count: usize = 4; | | |
| 452 | | | |
| 453 | var frame_context = context: { | | |
| 454 | var fbs: std.io.FixedBufferStream = .{ .buffer = src[consumed_count..] }; | | |
| 455 | const source = fbs.reader(); | | |
| 456 | const frame_header = try decodeZstandardHeader(source); | | |
| 457 | consumed_count += fbs.pos; | | |
| 458 | break :context try FrameContext.init(frame_header, window_size_max, verify_checksum); | | |
| 459 | }; | | |
| 460 | | | |
| 461 | consumed_count += try decodeZstandardFrameBlocksArrayList( | | |
| 462 | allocator, | | |
| 463 | dest, | | |
| 464 | src[consumed_count..], | | |
| 465 | &frame_context, | | |
| 466 | ); | | |
| 467 | return consumed_count; | | |
| 468 | } | | |
| 469 | | | |
| 470 | pub fn decodeZstandardFrameBlocksArrayList( | | |
| 471 | allocator: Allocator, | | |
| 472 | dest: *std.ArrayList(u8), | | |
| 473 | src: []const u8, | | |
| 474 | frame_context: *FrameContext, | | |
| 475 | ) (error{OutOfMemory} || FrameError)!usize { | | |
| 476 | const initial_len = dest.items.len; | | |
| 477 | | | |
| 478 | var ring_buffer = try RingBuffer.init(allocator, frame_context.window_size); | | |
| 479 | defer ring_buffer.deinit(allocator); | | |
| 480 | | | |
| 481 | // These tables take 7680 bytes | | |
| 482 | var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined; | | |
| 483 | var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined; | | |
| 484 | var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined; | | |
| 485 | | | |
| 486 | var block_header = try block.decodeBlockHeaderSlice(src); | | |
| 487 | var consumed_count: usize = 3; | | |
| 488 | var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data); | | |
| 489 | while (true) : ({ | | |
| 490 | block_header = try block.decodeBlockHeaderSlice(src[consumed_count..]); | | |
| 491 | consumed_count += 3; | | |
| 492 | }) { | | |
| 493 | const written_size = try block.decodeBlockRingBuffer( | | |
| 494 | &ring_buffer, | | |
| 495 | src[consumed_count..], | | |
| 496 | block_header, | | |
| 497 | &decode_state, | | |
| 498 | &consumed_count, | | |
| 499 | frame_context.block_size_max, | | |
| 500 | ); | | |
| 501 | if (frame_context.content_size) |size| { | | |
| 502 | if (dest.items.len - initial_len > size) { | | |
| 503 | return error.BadContentSize; | | |
| 504 | } | | |
| 505 | } | | |
| 506 | if (written_size > 0) { | | |
| 507 | const written_slice = ring_buffer.sliceLast(written_size); | | |
| 508 | try dest.appendSlice(written_slice.first); | | |
| 509 | try dest.appendSlice(written_slice.second); | | |
| 510 | if (frame_context.hasher_opt) |*hasher| { | | |
| 511 | hasher.update(written_slice.first); | | |
| 512 | hasher.update(written_slice.second); | | |
| 513 | } | | |
| 514 | } | | |
| 515 | if (block_header.last_block) break; | | |
| 516 | } | | |
| 517 | if (frame_context.content_size) |size| { | | |
| 518 | if (dest.items.len - initial_len != size) { | | |
| 519 | return error.BadContentSize; | | |
| 520 | } | | |
| 521 | } | | |
| 522 | | | |
| 523 | if (frame_context.has_checksum) { | | |
| 524 | if (src.len < consumed_count + 4) return error.EndOfStream; | | |
| 525 | const checksum = std.mem.readInt(u32, src[consumed_count..][0..4], .little); | | |
| 526 | consumed_count += 4; | | |
| 527 | if (frame_context.hasher_opt) |*hasher| { | | |
| 528 | if (checksum != computeChecksum(hasher)) return error.ChecksumFailure; | | |
| 529 | } | | |
| 530 | } | | |
| 531 | return consumed_count; | | |
| 532 | } | | |
| 533 | | | |
| 534 | fn decodeFrameBlocksInner( | 335 | fn decodeFrameBlocksInner( |
| 535 | dest: []u8, | 336 | dest: []u8, |
| 536 | src: []const u8, | 337 | src: []const u8, |
Andrew Kelley <andrew@ziglang.org> 2025-04-30 14:22:05-07:00
lib