authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-05 22:27:00+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-20 09:09:06+11:00
logece52e0771560726a72a11cfafb59bb1dc9ad221
treec2d65a9ef75acb932e4bcfc15fedc35f3374ecda
parenta9c8376305d4c99591432e0ed267fad665bb4f5f

std.compress.zstandard: verify content size and fix crash

2 files changed, 70 insertions(+), 23 deletions(-)

lib/std/compress/zstandard/decode/block.zig+14-3
......@@ -334,6 +334,8 @@ pub const DecodeState = struct {
334334 /// mean the literal stream or the sequence is malformed).
335335 /// - `error.InvalidBitStream` if the FSE sequence bitstream is malformed
336336 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits
337 /// - `error.DestTooSmall` if `dest` is not large enough to holde the
338 /// decompressed sequence
337339 pub fn decodeSequenceSlice(
338340 self: *DecodeState,
339341 dest: []u8,
......@@ -341,10 +343,11 @@ pub const DecodeState = struct {
341343 bit_reader: *readers.ReverseBitReader,
342344 sequence_size_limit: usize,
343345 last_sequence: bool,
344 ) DecodeSequenceError!usize {
346 ) (error{DestTooSmall} || DecodeSequenceError)!usize {
345347 const sequence = try self.nextSequence(bit_reader);
346348 const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
347349 if (sequence_length > sequence_size_limit) return error.MalformedSequence;
350 if (sequence_length > dest[write_pos..].len) return error.DestTooSmall;
348351
349352 try self.executeSequenceSlice(dest, write_pos, sequence);
350353 if (!last_sequence) {
......@@ -583,6 +586,8 @@ pub const DecodeState = struct {
583586/// - `error.MalformedRleBlock` if the block is an RLE block and `src.len < 1`
584587/// - `error.MalformedCompressedBlock` if there are errors decoding a
585588/// compressed block
589/// - `error.DestTooSmall` is `dest` is not large enough to hold the
590/// decompressed block
586591pub fn decodeBlock(
587592 dest: []u8,
588593 src: []const u8,
......@@ -590,13 +595,14 @@ pub fn decodeBlock(
590595 decode_state: *DecodeState,
591596 consumed_count: *usize,
592597 written_count: usize,
593) Error!usize {
598) (error{DestTooSmall} || Error)!usize {
594599 const block_size_max = @min(1 << 17, dest[written_count..].len); // 128KiB
595600 const block_size = block_header.block_size;
596601 if (block_size_max < block_size) return error.BlockSizeOverMaximum;
597602 switch (block_header.block_type) {
598603 .raw => {
599604 if (src.len < block_size) return error.MalformedBlockSize;
605 if (dest[written_count..].len < block_size) return error.DestTooSmall;
600606 const data = src[0..block_size];
601607 std.mem.copy(u8, dest[written_count..], data);
602608 consumed_count.* += block_size;
......@@ -604,6 +610,7 @@ pub fn decodeBlock(
604610 },
605611 .rle => {
606612 if (src.len < 1) return error.MalformedRleBlock;
613 if (dest[written_count..].len < block_size) return error.DestTooSmall;
607614 var write_pos: usize = written_count;
608615 while (write_pos < block_size + written_count) : (write_pos += 1) {
609616 dest[write_pos] = src[0];
......@@ -644,7 +651,10 @@ pub fn decodeBlock(
644651 &bit_stream,
645652 sequence_size_limit,
646653 i == sequences_header.sequence_count - 1,
647 ) catch return error.MalformedCompressedBlock;
654 ) catch |err| switch (err) {
655 error.DestTooSmall => return error.DestTooSmall,
656 else => return error.MalformedCompressedBlock,
657 };
648658 bytes_written += decompressed_size;
649659 sequence_size_limit -= decompressed_size;
650660 }
......@@ -655,6 +665,7 @@ pub fn decodeBlock(
655665
656666 if (decode_state.literal_written_count < literals.header.regenerated_size) {
657667 const len = literals.header.regenerated_size - decode_state.literal_written_count;
668 if (len > dest[written_count + bytes_written ..].len) return error.DestTooSmall;
658669 decode_state.decodeLiteralsSlice(dest[written_count + bytes_written ..], len) catch
659670 return error.MalformedCompressedBlock;
660671 bytes_written += len;
lib/std/compress/zstandard/decompress.zig+56-20
......@@ -96,6 +96,7 @@ pub fn decodeAlloc(
9696/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
9797/// uncompressed content size
9898/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
99/// size declared by the frame header
99100/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
100101/// number for a Zstandard or Skippable frame
101102/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
......@@ -180,6 +181,7 @@ pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 {
180181const FrameError = error{
181182 DictionaryIdFlagUnsupported,
182183 ChecksumFailure,
184 BadContentSize,
183185 EndOfStream,
184186} || InvalidBit || block.Error;
185187
......@@ -191,7 +193,7 @@ const FrameError = error{
191193/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
192194/// uncompressed content size
193195/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
194/// number for a Zstandard or Skippable frame
196/// size declared by the frame header
195197/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
196198/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
197199/// contains a checksum that does not match the checksum of the decompressed
......@@ -200,39 +202,51 @@ const FrameError = error{
200202/// - `error.UnusedBitSet` if the unused bit of the frame header is set
201203/// - `error.EndOfStream` if `src` does not contain a complete frame
202204/// - an error in `block.Error` if there are errors decoding a block
205/// - `error.BadContentSize` if the content size declared by the frame does
206/// not equal the actual size of decompressed data
203207pub fn decodeZstandardFrame(
204208 dest: []u8,
205209 src: []const u8,
206210 verify_checksum: bool,
207) (error{ UnknownContentSizeUnsupported, ContentTooLarge } || FrameError)!ReadWriteCount {
211) (error{
212 UnknownContentSizeUnsupported,
213 ContentTooLarge,
214 ContentSizeTooLarge,
215 WindowSizeUnknown,
216} || FrameError)!ReadWriteCount {
208217 assert(readInt(u32, src[0..4]) == frame.Zstandard.magic_number);
209218 var consumed_count: usize = 4;
210219
211 var fbs = std.io.fixedBufferStream(src[consumed_count..]);
212 var source = fbs.reader();
213 const frame_header = try decodeZstandardHeader(source);
214 consumed_count += fbs.pos;
215
216 if (frame_header.descriptor.dictionary_id_flag != 0) return error.DictionaryIdFlagUnsupported;
220 var frame_context = context: {
221 var fbs = std.io.fixedBufferStream(src[consumed_count..]);
222 var source = fbs.reader();
223 const frame_header = try decodeZstandardHeader(source);
224 consumed_count += fbs.pos;
225 break :context FrameContext.init(frame_header, std.math.maxInt(usize), verify_checksum) catch |err| switch (err) {
226 error.WindowTooLarge => unreachable,
227 inline else => |e| return e,
228 };
229 };
217230
218 const content_size = frame_header.content_size orelse return error.UnknownContentSizeUnsupported;
231 const content_size = frame_context.content_size orelse return error.UnknownContentSizeUnsupported;
219232 if (dest.len < content_size) return error.ContentTooLarge;
220233
221 const should_compute_checksum = frame_header.descriptor.content_checksum_flag and verify_checksum;
222 var hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null;
223
224 const written_count = try decodeFrameBlocks(
225 dest,
234 const written_count = decodeFrameBlocks(
235 dest[0..content_size],
226236 src[consumed_count..],
227237 &consumed_count,
228 if (hasher_opt) |*hasher| hasher else null,
229 );
238 if (frame_context.hasher_opt) |*hasher| hasher else null,
239 ) catch |err| switch (err) {
240 error.DestTooSmall => return error.BadContentSize,
241 inline else => |e| return e,
242 };
230243
231 if (frame_header.descriptor.content_checksum_flag) {
244 if (written_count != content_size) return error.BadContentSize;
245 if (frame_context.has_checksum) {
232246 if (src.len < consumed_count + 4) return error.EndOfStream;
233247 const checksum = readIntSlice(u32, src[consumed_count .. consumed_count + 4]);
234248 consumed_count += 4;
235 if (hasher_opt) |*hasher| {
249 if (frame_context.hasher_opt) |*hasher| {
236250 if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
237251 }
238252 }
......@@ -244,8 +258,14 @@ pub const FrameContext = struct {
244258 window_size: usize,
245259 has_checksum: bool,
246260 block_size_max: usize,
261 content_size: ?usize,
247262
248 const Error = error{ DictionaryIdFlagUnsupported, WindowSizeUnknown, WindowTooLarge };
263 const Error = error{
264 DictionaryIdFlagUnsupported,
265 WindowSizeUnknown,
266 WindowTooLarge,
267 ContentSizeTooLarge,
268 };
249269 /// Validates `frame_header` and returns the associated `FrameContext`.
250270 ///
251271 /// Errors returned:
......@@ -266,11 +286,18 @@ pub const FrameContext = struct {
266286 @intCast(usize, window_size_raw);
267287
268288 const should_compute_checksum = frame_header.descriptor.content_checksum_flag and verify_checksum;
289
290 const content_size = if (frame_header.content_size) |size|
291 std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
292 else
293 null;
294
269295 return .{
270296 .hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
271297 .window_size = window_size,
272298 .has_checksum = frame_header.descriptor.content_checksum_flag,
273299 .block_size_max = @min(1 << 17, window_size),
300 .content_size = content_size,
274301 };
275302 }
276303};
......@@ -294,6 +321,8 @@ pub const FrameContext = struct {
294321/// - `error.EndOfStream` if `src` does not contain a complete frame
295322/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
296323/// - an error in `block.Error` if there are errors decoding a block
324/// - `error.BadContentSize` if the content size declared by the frame does
325/// not equal the size of decompressed data
297326pub fn decodeZstandardFrameAlloc(
298327 allocator: Allocator,
299328 src: []const u8,
......@@ -321,6 +350,7 @@ pub fn decodeZstandardFrameArrayList(
321350 window_size_max: usize,
322351) (error{OutOfMemory} || FrameContext.Error || FrameError)!usize {
323352 assert(readInt(u32, src[0..4]) == frame.Zstandard.magic_number);
353 const initial_len = dest.items.len;
324354 var consumed_count: usize = 4;
325355
326356 var frame_context = context: {
......@@ -364,6 +394,12 @@ pub fn decodeZstandardFrameArrayList(
364394 hasher.update(written_slice.second);
365395 }
366396 }
397 const added_len = dest.items.len - initial_len;
398 if (frame_context.content_size) |size| {
399 if (added_len != size) {
400 return error.BadContentSize;
401 }
402 }
367403 if (block_header.last_block) break;
368404 }
369405
......@@ -384,7 +420,7 @@ fn decodeFrameBlocks(
384420 src: []const u8,
385421 consumed_count: *usize,
386422 hash: ?*std.hash.XxHash64,
387) (error{EndOfStream} || block.Error)!usize {
423) (error{ EndOfStream, DestTooSmall } || block.Error)!usize {
388424 // These tables take 7680 bytes
389425 var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
390426 var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;