authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-05-01 15:38:05-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:28-07:00
log2ff5ea223150ff4d77b094812e4bd184eaecb0c6
treed1221034cfccb5f14558b63dca4f115d287dbf9a
parent439117d0d75f28d8d555fceacc42315ec1dd8e34

std.compress.zstd: fix raw/rle blocks not doing frame accounting

3 files changed, 63 insertions(+), 65 deletions(-)

lib/std/compress/zstd.zig+4-3
...@@ -123,19 +123,20 @@ test "decompression" {...@@ -123,19 +123,20 @@ test "decompression" {
123 try testExpectDecompress(uncompressed, compressed19);123 try testExpectDecompress(uncompressed, compressed19);
124}124}
125125
126test "zero sized block" {126test "zero sized raw block" {
127 const input_raw =127 const input_raw =
128 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number128 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
129 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero129 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
130 "\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero130 "\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
131 try testExpectDecompress("", input_raw);
132}
131133
134test "zero sized rle block" {
132 const input_rle =135 const input_rle =
133 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number136 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
134 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero137 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
135 "\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero138 "\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
136 "\xaa"; // block_content139 "\xaa"; // block_content
137
138 try testExpectDecompress("", input_raw);
139 try testExpectDecompress("", input_rle);140 try testExpectDecompress("", input_rle);
140}141}
141142
lib/std/compress/zstd/Decompress.zig+58-61
...@@ -147,88 +147,85 @@ fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {...@@ -147,88 +147,85 @@ fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {
147fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state: *State.InFrame) !usize {147fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state: *State.InFrame) !usize {
148 const in = d.input;148 const in = d.input;
149149
150 var literal_fse_buffer: [zstd.table_size_max.literal]Table.Fse = undefined;
151 var match_fse_buffer: [zstd.table_size_max.match]Table.Fse = undefined;
152 var offset_fse_buffer: [zstd.table_size_max.offset]Table.Fse = undefined;
153 var literals_buffer: [zstd.block_size_max]u8 = undefined;
154 var sequence_buffer: [zstd.block_size_max]u8 = undefined;
155
156 var decode: Frame.Zstandard.Decode = .init(&literal_fse_buffer, &match_fse_buffer, &offset_fse_buffer);
157
158 const header_bytes = try in.takeArray(3);150 const header_bytes = try in.takeArray(3);
159 const block_header: Frame.Zstandard.Block.Header = @bitCast(header_bytes.*);151 const block_header: Frame.Zstandard.Block.Header = @bitCast(header_bytes.*);
160 const block_size = block_header.size;152 const block_size = block_header.size;
161 if (state.frame.block_size_max < block_size) return error.BlockOversize;153 if (state.frame.block_size_max < block_size) return error.BlockOversize;
162 if (@intFromEnum(limit) < block_size) return error.OutputBufferUndersize;154 if (@intFromEnum(limit) < block_size) return error.OutputBufferUndersize;
155 var bytes_written: usize = 0;
163 switch (block_header.type) {156 switch (block_header.type) {
164 .raw => {157 .raw => {
165 try in.readAll(bw, .limited(block_size));158 try in.readAll(bw, .limited(block_size));
166 return block_size;159 bytes_written = block_size;
167 },160 },
168 .rle => {161 .rle => {
169 const byte = try in.takeByte();162 const byte = try in.takeByte();
170 try bw.splatByteAll(byte, block_size);163 try bw.splatByteAll(byte, block_size);
171 return block_size;164 bytes_written = block_size;
172 },165 },
173 .compressed => {},166 .compressed => {
174 .reserved => return error.ReservedBlock,167 var literal_fse_buffer: [zstd.table_size_max.literal]Table.Fse = undefined;
175 }168 var match_fse_buffer: [zstd.table_size_max.match]Table.Fse = undefined;
169 var offset_fse_buffer: [zstd.table_size_max.offset]Table.Fse = undefined;
170 var literals_buffer: [zstd.block_size_max]u8 = undefined;
171 var sequence_buffer: [zstd.block_size_max]u8 = undefined;
172 var decode: Frame.Zstandard.Decode = .init(&literal_fse_buffer, &match_fse_buffer, &offset_fse_buffer);
173 var remaining: Reader.Limit = .limited(block_size);
174 const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);
175 const sequences_header = try SequencesSection.Header.decode(in, &remaining);
176
177 try decode.prepare(in, &remaining, literals, sequences_header);
178
179 {
180 if (sequence_buffer.len < @intFromEnum(remaining))
181 return error.SequenceBufferUndersize;
182 const seq_slice = remaining.slice(&sequence_buffer);
183 try in.readSlice(seq_slice);
184 var bit_stream = try ReverseBitReader.init(seq_slice);
176185
177 var remaining: Reader.Limit = .limited(block_size);186 if (sequences_header.sequence_count > 0) {
178 const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);187 try decode.readInitialFseState(&bit_stream);
179 const sequences_header = try SequencesSection.Header.decode(in, &remaining);188
189 var sequence_size_limit = state.frame.block_size_max;
190 for (0..sequences_header.sequence_count) |i| {
191 const decompressed_size = try decode.decodeSequence(
192 bw,
193 &bit_stream,
194 sequence_size_limit,
195 i == sequences_header.sequence_count - 1,
196 );
197 sequence_size_limit -= decompressed_size;
198 bytes_written += decompressed_size;
199 }
200 }
180201
181 try decode.prepare(in, &remaining, literals, sequences_header);202 if (!bit_stream.isEmpty()) {
203 return error.MalformedCompressedBlock;
204 }
205 }
182206
183 var bytes_written: usize = 0;207 if (decode.literal_written_count < literals.header.regenerated_size) {
184 {208 const len = literals.header.regenerated_size - decode.literal_written_count;
185 if (sequence_buffer.len < @intFromEnum(remaining))209 try decode.decodeLiterals(bw, len);
186 return error.SequenceBufferUndersize;210 bytes_written += len;
187 const seq_slice = remaining.slice(&sequence_buffer);
188 try in.readSlice(seq_slice);
189 var bit_stream = try ReverseBitReader.init(seq_slice);
190
191 if (sequences_header.sequence_count > 0) {
192 try decode.readInitialFseState(&bit_stream);
193
194 var sequence_size_limit = state.frame.block_size_max;
195 for (0..sequences_header.sequence_count) |i| {
196 const decompressed_size = try decode.decodeSequence(
197 bw,
198 &bit_stream,
199 sequence_size_limit,
200 i == sequences_header.sequence_count - 1,
201 );
202 sequence_size_limit -= decompressed_size;
203 bytes_written += decompressed_size;
204 }211 }
205 }
206212
207 if (!bit_stream.isEmpty()) {213 switch (decode.literal_header.block_type) {
208 return error.MalformedCompressedBlock;214 .treeless, .compressed => {
209 }215 if (!decode.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
210 }216 },
217 .raw, .rle => {},
218 }
211219
212 if (decode.literal_written_count < literals.header.regenerated_size) {220 if (bytes_written > state.frame.block_size_max) return error.BlockOversize;
213 const len = literals.header.regenerated_size - decode.literal_written_count;221 if (remaining.nonzero()) return error.MalformedCompressedBlock;
214 try decode.decodeLiterals(bw, len);
215 bytes_written += len;
216 }
217222
218 switch (decode.literal_header.block_type) {223 state.decompressed_size += bytes_written;
219 .treeless, .compressed => {224 if (state.frame.content_size) |size| {
220 if (!decode.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;225 if (state.decompressed_size > size) return error.MalformedFrame;
226 }
221 },227 },
222 .raw, .rle => {},228 .reserved => return error.ReservedBlock,
223 }
224
225 if (bytes_written > state.frame.block_size_max) return error.BlockOversize;
226 if (remaining.nonzero()) return error.MalformedCompressedBlock;
227
228 state.decompressed_size += bytes_written;
229
230 if (state.frame.content_size) |size| {
231 if (state.decompressed_size > size) return error.MalformedFrame;
232 }229 }
233230
234 if (state.frame.hasher_opt) |*hasher| {231 if (state.frame.hasher_opt) |*hasher| {
lib/std/io/BufferedReader.zig+1-1
...@@ -730,7 +730,7 @@ pub fn fill(br: *BufferedReader, n: usize) Reader.Error!void {...@@ -730,7 +730,7 @@ pub fn fill(br: *BufferedReader, n: usize) Reader.Error!void {
730 }730 }
731 if (seek > 0) {731 if (seek > 0) {
732 const remainder = buffer[seek..];732 const remainder = buffer[seek..];
733 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);733 @memmove(buffer[0..remainder.len], remainder);
734 br.end = remainder.len;734 br.end = remainder.len;
735 br.seek = 0;735 br.seek = 0;
736 }736 }