| ... | ... | @@ -1,6 +1,5 @@ |
| 1 | 1 | const std = @import("std"); |
| 2 | 2 | const assert = std.debug.assert; |
| 3 | | const Allocator = std.mem.Allocator; |
| 4 | 3 | const RingBuffer = std.RingBuffer; |
| 5 | 4 | |
| 6 | 5 | const types = @import("types.zig"); |
| ... | ... | @@ -117,41 +116,6 @@ pub fn decode(dest: []u8, src: []const u8, verify_checksum: bool) error{ |
| 117 | 116 | return write_count; |
| 118 | 117 | } |
| 119 | 118 | |
| 120 | | /// Decodes a stream of frames from `src`; returns the decoded bytes. The stream |
| 121 | | /// should not have extra trailing bytes - either all bytes in `src` will be |
| 122 | | /// decoded, or an error will be returned. |
| 123 | | /// |
| 124 | | /// Errors returned: |
| 125 | | /// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that |
| 126 | | /// uses a dictionary |
| 127 | | /// - `error.MalformedFrame` if a frame in `src` is invalid |
| 128 | | /// - `error.OutOfMemory` if `allocator` cannot allocate enough memory |
| 129 | | pub fn decodeAlloc( |
| 130 | | allocator: Allocator, |
| 131 | | src: []const u8, |
| 132 | | verify_checksum: bool, |
| 133 | | window_size_max: usize, |
| 134 | | ) error{ DictionaryIdFlagUnsupported, MalformedFrame, OutOfMemory }![]u8 { |
| 135 | | var result = std.ArrayList(u8).init(allocator); |
| 136 | | errdefer result.deinit(); |
| 137 | | |
| 138 | | var read_count: usize = 0; |
| 139 | | while (read_count < src.len) { |
| 140 | | read_count += decodeFrameArrayList( |
| 141 | | allocator, |
| 142 | | &result, |
| 143 | | src[read_count..], |
| 144 | | verify_checksum, |
| 145 | | window_size_max, |
| 146 | | ) catch |err| switch (err) { |
| 147 | | error.OutOfMemory => return error.OutOfMemory, |
| 148 | | error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported, |
| 149 | | else => return error.MalformedFrame, |
| 150 | | }; |
| 151 | | } |
| 152 | | return result.toOwnedSlice(); |
| 153 | | } |
| 154 | | |
| 155 | 119 | /// Decodes the frame at the start of `src` into `dest`. Returns the number of |
| 156 | 120 | /// bytes read from `src` and written to `dest`. This function can only decode |
| 157 | 121 | /// frames that declare the decompressed content size. |
| ... | ... | @@ -207,58 +171,6 @@ pub fn decodeFrame( |
| 207 | 171 | } |
| 208 | 172 | } |
| 209 | 173 | |
| 210 | | /// Decodes the frame at the start of `src` into `dest`. Returns the number of |
| 211 | | /// bytes read from `src`. |
| 212 | | /// |
| 213 | | /// Errors returned: |
| 214 | | /// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic |
| 215 | | /// number for a Zstandard or skippable frame |
| 216 | | /// - `error.WindowSizeUnknown` if the frame does not have a valid window size |
| 217 | | /// - `error.WindowTooLarge` if the window size is larger than |
| 218 | | /// `window_size_max` |
| 219 | | /// - `error.ContentSizeTooLarge` if the frame header indicates a content size |
| 220 | | /// that is larger than `std.math.maxInt(usize)` |
| 221 | | /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary |
| 222 | | /// - `error.ChecksumFailure` if `verify_checksum` is true and the frame |
| 223 | | /// contains a checksum that does not match the checksum of the decompressed |
| 224 | | /// data |
| 225 | | /// - `error.ReservedBitSet` if any of the reserved bits of the frame header |
| 226 | | /// are set |
| 227 | | /// - `error.EndOfStream` if `src` does not contain a complete frame |
| 228 | | /// - `error.BadContentSize` if the content size declared by the frame does |
| 229 | | /// not equal the actual size of decompressed data |
| 230 | | /// - `error.OutOfMemory` if `allocator` cannot allocate enough memory |
| 231 | | /// - an error in `block.Error` if there are errors decoding a block |
| 232 | | /// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a |
| 233 | | /// size greater than `src.len` |
| 234 | | pub fn decodeFrameArrayList( |
| 235 | | allocator: Allocator, |
| 236 | | dest: *std.ArrayList(u8), |
| 237 | | src: []const u8, |
| 238 | | verify_checksum: bool, |
| 239 | | window_size_max: usize, |
| 240 | | ) (error{ BadMagic, OutOfMemory, SkippableSizeTooLarge } || FrameContext.Error || FrameError)!usize { |
| 241 | | var fbs: std.io.FixedBufferStream = .{ .buffer = src }; |
| 242 | | const reader = fbs.reader(); |
| 243 | | const magic = try reader.readInt(u32, .little); |
| 244 | | switch (try frameType(magic)) { |
| 245 | | .zstandard => return decodeZstandardFrameArrayList( |
| 246 | | allocator, |
| 247 | | dest, |
| 248 | | src, |
| 249 | | verify_checksum, |
| 250 | | window_size_max, |
| 251 | | ), |
| 252 | | .skippable => { |
| 253 | | const content_size = try fbs.reader().readInt(u32, .little); |
| 254 | | if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge; |
| 255 | | const read_count = @as(usize, content_size) + 8; |
| 256 | | if (read_count > src.len) return error.SkippableSizeTooLarge; |
| 257 | | return read_count; |
| 258 | | }, |
| 259 | | } |
| 260 | | } |
| 261 | | |
| 262 | 174 | /// Returns the frame checksum corresponding to the data fed into `hasher` |
| 263 | 175 | pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 { |
| 264 | 176 | const hash = hasher.final(); |
| ... | ... | @@ -420,117 +332,6 @@ pub const FrameContext = struct { |
| 420 | 332 | } |
| 421 | 333 | }; |
| 422 | 334 | |
| 423 | | /// Decode a Zstandard from from `src` and return number of bytes read; see |
| 424 | | /// `decodeZstandardFrame()`. The first four bytes of `src` must be the magic |
| 425 | | /// number for a Zstandard frame. |
| 426 | | /// |
| 427 | | /// Errors returned: |
| 428 | | /// - `error.WindowSizeUnknown` if the frame does not have a valid window size |
| 429 | | /// - `error.WindowTooLarge` if the window size is larger than |
| 430 | | /// `window_size_max` |
| 431 | | /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary |
| 432 | | /// - `error.ContentSizeTooLarge` if the frame header indicates a content size |
| 433 | | /// that is larger than `std.math.maxInt(usize)` |
| 434 | | /// - `error.ChecksumFailure` if `verify_checksum` is true and the frame |
| 435 | | /// contains a checksum that does not match the checksum of the decompressed |
| 436 | | /// data |
| 437 | | /// - `error.ReservedBitSet` if the reserved bit of the frame header is set |
| 438 | | /// - `error.EndOfStream` if `src` does not contain a complete frame |
| 439 | | /// - `error.OutOfMemory` if `allocator` cannot allocate enough memory |
| 440 | | /// - an error in `block.Error` if there are errors decoding a block |
| 441 | | /// - `error.BadContentSize` if the content size declared by the frame does |
| 442 | | /// not equal the size of decompressed data |
| 443 | | pub fn decodeZstandardFrameArrayList( |
| 444 | | allocator: Allocator, |
| 445 | | dest: *std.ArrayList(u8), |
| 446 | | src: []const u8, |
| 447 | | verify_checksum: bool, |
| 448 | | window_size_max: usize, |
| 449 | | ) (error{OutOfMemory} || FrameContext.Error || FrameError)!usize { |
| 450 | | assert(std.mem.readInt(u32, src[0..4], .little) == frame.Zstandard.magic_number); |
| 451 | | var consumed_count: usize = 4; |
| 452 | | |
| 453 | | var frame_context = context: { |
| 454 | | var fbs: std.io.FixedBufferStream = .{ .buffer = src[consumed_count..] }; |
| 455 | | const source = fbs.reader(); |
| 456 | | const frame_header = try decodeZstandardHeader(source); |
| 457 | | consumed_count += fbs.pos; |
| 458 | | break :context try FrameContext.init(frame_header, window_size_max, verify_checksum); |
| 459 | | }; |
| 460 | | |
| 461 | | consumed_count += try decodeZstandardFrameBlocksArrayList( |
| 462 | | allocator, |
| 463 | | dest, |
| 464 | | src[consumed_count..], |
| 465 | | &frame_context, |
| 466 | | ); |
| 467 | | return consumed_count; |
| 468 | | } |
| 469 | | |
| 470 | | pub fn decodeZstandardFrameBlocksArrayList( |
| 471 | | allocator: Allocator, |
| 472 | | dest: *std.ArrayList(u8), |
| 473 | | src: []const u8, |
| 474 | | frame_context: *FrameContext, |
| 475 | | ) (error{OutOfMemory} || FrameError)!usize { |
| 476 | | const initial_len = dest.items.len; |
| 477 | | |
| 478 | | var ring_buffer = try RingBuffer.init(allocator, frame_context.window_size); |
| 479 | | defer ring_buffer.deinit(allocator); |
| 480 | | |
| 481 | | // These tables take 7680 bytes |
| 482 | | var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined; |
| 483 | | var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined; |
| 484 | | var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined; |
| 485 | | |
| 486 | | var block_header = try block.decodeBlockHeaderSlice(src); |
| 487 | | var consumed_count: usize = 3; |
| 488 | | var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data); |
| 489 | | while (true) : ({ |
| 490 | | block_header = try block.decodeBlockHeaderSlice(src[consumed_count..]); |
| 491 | | consumed_count += 3; |
| 492 | | }) { |
| 493 | | const written_size = try block.decodeBlockRingBuffer( |
| 494 | | &ring_buffer, |
| 495 | | src[consumed_count..], |
| 496 | | block_header, |
| 497 | | &decode_state, |
| 498 | | &consumed_count, |
| 499 | | frame_context.block_size_max, |
| 500 | | ); |
| 501 | | if (frame_context.content_size) |size| { |
| 502 | | if (dest.items.len - initial_len > size) { |
| 503 | | return error.BadContentSize; |
| 504 | | } |
| 505 | | } |
| 506 | | if (written_size > 0) { |
| 507 | | const written_slice = ring_buffer.sliceLast(written_size); |
| 508 | | try dest.appendSlice(written_slice.first); |
| 509 | | try dest.appendSlice(written_slice.second); |
| 510 | | if (frame_context.hasher_opt) |*hasher| { |
| 511 | | hasher.update(written_slice.first); |
| 512 | | hasher.update(written_slice.second); |
| 513 | | } |
| 514 | | } |
| 515 | | if (block_header.last_block) break; |
| 516 | | } |
| 517 | | if (frame_context.content_size) |size| { |
| 518 | | if (dest.items.len - initial_len != size) { |
| 519 | | return error.BadContentSize; |
| 520 | | } |
| 521 | | } |
| 522 | | |
| 523 | | if (frame_context.has_checksum) { |
| 524 | | if (src.len < consumed_count + 4) return error.EndOfStream; |
| 525 | | const checksum = std.mem.readInt(u32, src[consumed_count..][0..4], .little); |
| 526 | | consumed_count += 4; |
| 527 | | if (frame_context.hasher_opt) |*hasher| { |
| 528 | | if (checksum != computeChecksum(hasher)) return error.ChecksumFailure; |
| 529 | | } |
| 530 | | } |
| 531 | | return consumed_count; |
| 532 | | } |
| 533 | | |
| 534 | 335 | fn decodeFrameBlocksInner( |
| 535 | 336 | dest: []u8, |
| 536 | 337 | src: []const u8, |
Andrew Kelley <andrew@ziglang.org> 2025-04-30 14:22:05-07:00
lib