authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-05 22:27:00+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-20 09:09:06+11:00
logece52e0771560726a72a11cfafb59bb1dc9ad221
treec2d65a9ef75acb932e4bcfc15fedc35f3374ecda
parenta9c8376305d4c99591432e0ed267fad665bb4f5f

std.compress.zstandard: verify content size and fix crash

2 files changed, 70 insertions(+), 23 deletions(-)

lib/std/compress/zstandard/decode/block.zig+14-3
...@@ -334,6 +334,8 @@ pub const DecodeState = struct {...@@ -334,6 +334,8 @@ pub const DecodeState = struct {
334 /// mean the literal stream or the sequence is malformed).334 /// mean the literal stream or the sequence is malformed).
335 /// - `error.InvalidBitStream` if the FSE sequence bitstream is malformed335 /// - `error.InvalidBitStream` if the FSE sequence bitstream is malformed
336 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits336 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits
337 /// - `error.DestTooSmall` if `dest` is not large enough to holde the
338 /// decompressed sequence
337 pub fn decodeSequenceSlice(339 pub fn decodeSequenceSlice(
338 self: *DecodeState,340 self: *DecodeState,
339 dest: []u8,341 dest: []u8,
...@@ -341,10 +343,11 @@ pub const DecodeState = struct {...@@ -341,10 +343,11 @@ pub const DecodeState = struct {
341 bit_reader: *readers.ReverseBitReader,343 bit_reader: *readers.ReverseBitReader,
342 sequence_size_limit: usize,344 sequence_size_limit: usize,
343 last_sequence: bool,345 last_sequence: bool,
344 ) DecodeSequenceError!usize {346 ) (error{DestTooSmall} || DecodeSequenceError)!usize {
345 const sequence = try self.nextSequence(bit_reader);347 const sequence = try self.nextSequence(bit_reader);
346 const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;348 const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
347 if (sequence_length > sequence_size_limit) return error.MalformedSequence;349 if (sequence_length > sequence_size_limit) return error.MalformedSequence;
350 if (sequence_length > dest[write_pos..].len) return error.DestTooSmall;
348351
349 try self.executeSequenceSlice(dest, write_pos, sequence);352 try self.executeSequenceSlice(dest, write_pos, sequence);
350 if (!last_sequence) {353 if (!last_sequence) {
...@@ -583,6 +586,8 @@ pub const DecodeState = struct {...@@ -583,6 +586,8 @@ pub const DecodeState = struct {
583/// - `error.MalformedRleBlock` if the block is an RLE block and `src.len < 1`586/// - `error.MalformedRleBlock` if the block is an RLE block and `src.len < 1`
584/// - `error.MalformedCompressedBlock` if there are errors decoding a587/// - `error.MalformedCompressedBlock` if there are errors decoding a
585/// compressed block588/// compressed block
589/// - `error.DestTooSmall` is `dest` is not large enough to hold the
590/// decompressed block
586pub fn decodeBlock(591pub fn decodeBlock(
587 dest: []u8,592 dest: []u8,
588 src: []const u8,593 src: []const u8,
...@@ -590,13 +595,14 @@ pub fn decodeBlock(...@@ -590,13 +595,14 @@ pub fn decodeBlock(
590 decode_state: *DecodeState,595 decode_state: *DecodeState,
591 consumed_count: *usize,596 consumed_count: *usize,
592 written_count: usize,597 written_count: usize,
593) Error!usize {598) (error{DestTooSmall} || Error)!usize {
594 const block_size_max = @min(1 << 17, dest[written_count..].len); // 128KiB599 const block_size_max = @min(1 << 17, dest[written_count..].len); // 128KiB
595 const block_size = block_header.block_size;600 const block_size = block_header.block_size;
596 if (block_size_max < block_size) return error.BlockSizeOverMaximum;601 if (block_size_max < block_size) return error.BlockSizeOverMaximum;
597 switch (block_header.block_type) {602 switch (block_header.block_type) {
598 .raw => {603 .raw => {
599 if (src.len < block_size) return error.MalformedBlockSize;604 if (src.len < block_size) return error.MalformedBlockSize;
605 if (dest[written_count..].len < block_size) return error.DestTooSmall;
600 const data = src[0..block_size];606 const data = src[0..block_size];
601 std.mem.copy(u8, dest[written_count..], data);607 std.mem.copy(u8, dest[written_count..], data);
602 consumed_count.* += block_size;608 consumed_count.* += block_size;
...@@ -604,6 +610,7 @@ pub fn decodeBlock(...@@ -604,6 +610,7 @@ pub fn decodeBlock(
604 },610 },
605 .rle => {611 .rle => {
606 if (src.len < 1) return error.MalformedRleBlock;612 if (src.len < 1) return error.MalformedRleBlock;
613 if (dest[written_count..].len < block_size) return error.DestTooSmall;
607 var write_pos: usize = written_count;614 var write_pos: usize = written_count;
608 while (write_pos < block_size + written_count) : (write_pos += 1) {615 while (write_pos < block_size + written_count) : (write_pos += 1) {
609 dest[write_pos] = src[0];616 dest[write_pos] = src[0];
...@@ -644,7 +651,10 @@ pub fn decodeBlock(...@@ -644,7 +651,10 @@ pub fn decodeBlock(
644 &bit_stream,651 &bit_stream,
645 sequence_size_limit,652 sequence_size_limit,
646 i == sequences_header.sequence_count - 1,653 i == sequences_header.sequence_count - 1,
647 ) catch return error.MalformedCompressedBlock;654 ) catch |err| switch (err) {
655 error.DestTooSmall => return error.DestTooSmall,
656 else => return error.MalformedCompressedBlock,
657 };
648 bytes_written += decompressed_size;658 bytes_written += decompressed_size;
649 sequence_size_limit -= decompressed_size;659 sequence_size_limit -= decompressed_size;
650 }660 }
...@@ -655,6 +665,7 @@ pub fn decodeBlock(...@@ -655,6 +665,7 @@ pub fn decodeBlock(
655665
656 if (decode_state.literal_written_count < literals.header.regenerated_size) {666 if (decode_state.literal_written_count < literals.header.regenerated_size) {
657 const len = literals.header.regenerated_size - decode_state.literal_written_count;667 const len = literals.header.regenerated_size - decode_state.literal_written_count;
668 if (len > dest[written_count + bytes_written ..].len) return error.DestTooSmall;
658 decode_state.decodeLiteralsSlice(dest[written_count + bytes_written ..], len) catch669 decode_state.decodeLiteralsSlice(dest[written_count + bytes_written ..], len) catch
659 return error.MalformedCompressedBlock;670 return error.MalformedCompressedBlock;
660 bytes_written += len;671 bytes_written += len;
lib/std/compress/zstandard/decompress.zig+56-20
...@@ -96,6 +96,7 @@ pub fn decodeAlloc(...@@ -96,6 +96,7 @@ pub fn decodeAlloc(
96/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the96/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
97/// uncompressed content size97/// uncompressed content size
98/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data98/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
99/// size declared by the frame header
99/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic100/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
100/// number for a Zstandard or Skippable frame101/// number for a Zstandard or Skippable frame
101/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary102/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
...@@ -180,6 +181,7 @@ pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 {...@@ -180,6 +181,7 @@ pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 {
180const FrameError = error{181const FrameError = error{
181 DictionaryIdFlagUnsupported,182 DictionaryIdFlagUnsupported,
182 ChecksumFailure,183 ChecksumFailure,
184 BadContentSize,
183 EndOfStream,185 EndOfStream,
184} || InvalidBit || block.Error;186} || InvalidBit || block.Error;
185187
...@@ -191,7 +193,7 @@ const FrameError = error{...@@ -191,7 +193,7 @@ const FrameError = error{
191/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the193/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
192/// uncompressed content size194/// uncompressed content size
193/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data195/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
194/// number for a Zstandard or Skippable frame196/// size declared by the frame header
195/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary197/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
196/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame198/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
197/// contains a checksum that does not match the checksum of the decompressed199/// contains a checksum that does not match the checksum of the decompressed
...@@ -200,39 +202,51 @@ const FrameError = error{...@@ -200,39 +202,51 @@ const FrameError = error{
200/// - `error.UnusedBitSet` if the unused bit of the frame header is set202/// - `error.UnusedBitSet` if the unused bit of the frame header is set
201/// - `error.EndOfStream` if `src` does not contain a complete frame203/// - `error.EndOfStream` if `src` does not contain a complete frame
202/// - an error in `block.Error` if there are errors decoding a block204/// - an error in `block.Error` if there are errors decoding a block
205/// - `error.BadContentSize` if the content size declared by the frame does
206/// not equal the actual size of decompressed data
203pub fn decodeZstandardFrame(207pub fn decodeZstandardFrame(
204 dest: []u8,208 dest: []u8,
205 src: []const u8,209 src: []const u8,
206 verify_checksum: bool,210 verify_checksum: bool,
207) (error{ UnknownContentSizeUnsupported, ContentTooLarge } || FrameError)!ReadWriteCount {211) (error{
212 UnknownContentSizeUnsupported,
213 ContentTooLarge,
214 ContentSizeTooLarge,
215 WindowSizeUnknown,
216} || FrameError)!ReadWriteCount {
208 assert(readInt(u32, src[0..4]) == frame.Zstandard.magic_number);217 assert(readInt(u32, src[0..4]) == frame.Zstandard.magic_number);
209 var consumed_count: usize = 4;218 var consumed_count: usize = 4;
210219
211 var fbs = std.io.fixedBufferStream(src[consumed_count..]);220 var frame_context = context: {
212 var source = fbs.reader();221 var fbs = std.io.fixedBufferStream(src[consumed_count..]);
213 const frame_header = try decodeZstandardHeader(source);222 var source = fbs.reader();
214 consumed_count += fbs.pos;223 const frame_header = try decodeZstandardHeader(source);
215224 consumed_count += fbs.pos;
216 if (frame_header.descriptor.dictionary_id_flag != 0) return error.DictionaryIdFlagUnsupported;225 break :context FrameContext.init(frame_header, std.math.maxInt(usize), verify_checksum) catch |err| switch (err) {
226 error.WindowTooLarge => unreachable,
227 inline else => |e| return e,
228 };
229 };
217230
218 const content_size = frame_header.content_size orelse return error.UnknownContentSizeUnsupported;231 const content_size = frame_context.content_size orelse return error.UnknownContentSizeUnsupported;
219 if (dest.len < content_size) return error.ContentTooLarge;232 if (dest.len < content_size) return error.ContentTooLarge;
220233
221 const should_compute_checksum = frame_header.descriptor.content_checksum_flag and verify_checksum;234 const written_count = decodeFrameBlocks(
222 var hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null;235 dest[0..content_size],
223
224 const written_count = try decodeFrameBlocks(
225 dest,
226 src[consumed_count..],236 src[consumed_count..],
227 &consumed_count,237 &consumed_count,
228 if (hasher_opt) |*hasher| hasher else null,238 if (frame_context.hasher_opt) |*hasher| hasher else null,
229 );239 ) catch |err| switch (err) {
240 error.DestTooSmall => return error.BadContentSize,
241 inline else => |e| return e,
242 };
230243
231 if (frame_header.descriptor.content_checksum_flag) {244 if (written_count != content_size) return error.BadContentSize;
245 if (frame_context.has_checksum) {
232 if (src.len < consumed_count + 4) return error.EndOfStream;246 if (src.len < consumed_count + 4) return error.EndOfStream;
233 const checksum = readIntSlice(u32, src[consumed_count .. consumed_count + 4]);247 const checksum = readIntSlice(u32, src[consumed_count .. consumed_count + 4]);
234 consumed_count += 4;248 consumed_count += 4;
235 if (hasher_opt) |*hasher| {249 if (frame_context.hasher_opt) |*hasher| {
236 if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;250 if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
237 }251 }
238 }252 }
...@@ -244,8 +258,14 @@ pub const FrameContext = struct {...@@ -244,8 +258,14 @@ pub const FrameContext = struct {
244 window_size: usize,258 window_size: usize,
245 has_checksum: bool,259 has_checksum: bool,
246 block_size_max: usize,260 block_size_max: usize,
261 content_size: ?usize,
247262
248 const Error = error{ DictionaryIdFlagUnsupported, WindowSizeUnknown, WindowTooLarge };263 const Error = error{
264 DictionaryIdFlagUnsupported,
265 WindowSizeUnknown,
266 WindowTooLarge,
267 ContentSizeTooLarge,
268 };
249 /// Validates `frame_header` and returns the associated `FrameContext`.269 /// Validates `frame_header` and returns the associated `FrameContext`.
250 ///270 ///
251 /// Errors returned:271 /// Errors returned:
...@@ -266,11 +286,18 @@ pub const FrameContext = struct {...@@ -266,11 +286,18 @@ pub const FrameContext = struct {
266 @intCast(usize, window_size_raw);286 @intCast(usize, window_size_raw);
267287
268 const should_compute_checksum = frame_header.descriptor.content_checksum_flag and verify_checksum;288 const should_compute_checksum = frame_header.descriptor.content_checksum_flag and verify_checksum;
289
290 const content_size = if (frame_header.content_size) |size|
291 std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
292 else
293 null;
294
269 return .{295 return .{
270 .hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,296 .hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
271 .window_size = window_size,297 .window_size = window_size,
272 .has_checksum = frame_header.descriptor.content_checksum_flag,298 .has_checksum = frame_header.descriptor.content_checksum_flag,
273 .block_size_max = @min(1 << 17, window_size),299 .block_size_max = @min(1 << 17, window_size),
300 .content_size = content_size,
274 };301 };
275 }302 }
276};303};
...@@ -294,6 +321,8 @@ pub const FrameContext = struct {...@@ -294,6 +321,8 @@ pub const FrameContext = struct {
294/// - `error.EndOfStream` if `src` does not contain a complete frame321/// - `error.EndOfStream` if `src` does not contain a complete frame
295/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory322/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
296/// - an error in `block.Error` if there are errors decoding a block323/// - an error in `block.Error` if there are errors decoding a block
324/// - `error.BadContentSize` if the content size declared by the frame does
325/// not equal the size of decompressed data
297pub fn decodeZstandardFrameAlloc(326pub fn decodeZstandardFrameAlloc(
298 allocator: Allocator,327 allocator: Allocator,
299 src: []const u8,328 src: []const u8,
...@@ -321,6 +350,7 @@ pub fn decodeZstandardFrameArrayList(...@@ -321,6 +350,7 @@ pub fn decodeZstandardFrameArrayList(
321 window_size_max: usize,350 window_size_max: usize,
322) (error{OutOfMemory} || FrameContext.Error || FrameError)!usize {351) (error{OutOfMemory} || FrameContext.Error || FrameError)!usize {
323 assert(readInt(u32, src[0..4]) == frame.Zstandard.magic_number);352 assert(readInt(u32, src[0..4]) == frame.Zstandard.magic_number);
353 const initial_len = dest.items.len;
324 var consumed_count: usize = 4;354 var consumed_count: usize = 4;
325355
326 var frame_context = context: {356 var frame_context = context: {
...@@ -364,6 +394,12 @@ pub fn decodeZstandardFrameArrayList(...@@ -364,6 +394,12 @@ pub fn decodeZstandardFrameArrayList(
364 hasher.update(written_slice.second);394 hasher.update(written_slice.second);
365 }395 }
366 }396 }
397 const added_len = dest.items.len - initial_len;
398 if (frame_context.content_size) |size| {
399 if (added_len != size) {
400 return error.BadContentSize;
401 }
402 }
367 if (block_header.last_block) break;403 if (block_header.last_block) break;
368 }404 }
369405
...@@ -384,7 +420,7 @@ fn decodeFrameBlocks(...@@ -384,7 +420,7 @@ fn decodeFrameBlocks(
384 src: []const u8,420 src: []const u8,
385 consumed_count: *usize,421 consumed_count: *usize,
386 hash: ?*std.hash.XxHash64,422 hash: ?*std.hash.XxHash64,
387) (error{EndOfStream} || block.Error)!usize {423) (error{ EndOfStream, DestTooSmall } || block.Error)!usize {
388 // These tables take 7680 bytes424 // These tables take 7680 bytes
389 var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;425 var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
390 var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;426 var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;