authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-05-01 15:38:05-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:28-07:00
log2ff5ea223150ff4d77b094812e4bd184eaecb0c6
treed1221034cfccb5f14558b63dca4f115d287dbf9a
parent439117d0d75f28d8d555fceacc42315ec1dd8e34

std.compress.zstd: fix raw/rle blocks not doing frame accounting

3 files changed, 63 insertions(+), 65 deletions(-)

lib/std/compress/zstd.zig+4-3
......@@ -123,19 +123,20 @@ test "decompression" {
123123 try testExpectDecompress(uncompressed, compressed19);
124124}
125125
126test "zero sized block" {
126test "zero sized raw block" {
127127 const input_raw =
128128 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
129129 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
130130 "\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
131 try testExpectDecompress("", input_raw);
132}
131133
134test "zero sized rle block" {
132135 const input_rle =
133136 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
134137 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
135138 "\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
136139 "\xaa"; // block_content
137
138 try testExpectDecompress("", input_raw);
139140 try testExpectDecompress("", input_rle);
140141}
141142
lib/std/compress/zstd/Decompress.zig+58-61
......@@ -147,88 +147,85 @@ fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {
147147fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state: *State.InFrame) !usize {
148148 const in = d.input;
149149
150 var literal_fse_buffer: [zstd.table_size_max.literal]Table.Fse = undefined;
151 var match_fse_buffer: [zstd.table_size_max.match]Table.Fse = undefined;
152 var offset_fse_buffer: [zstd.table_size_max.offset]Table.Fse = undefined;
153 var literals_buffer: [zstd.block_size_max]u8 = undefined;
154 var sequence_buffer: [zstd.block_size_max]u8 = undefined;
155
156 var decode: Frame.Zstandard.Decode = .init(&literal_fse_buffer, &match_fse_buffer, &offset_fse_buffer);
157
158150 const header_bytes = try in.takeArray(3);
159151 const block_header: Frame.Zstandard.Block.Header = @bitCast(header_bytes.*);
160152 const block_size = block_header.size;
161153 if (state.frame.block_size_max < block_size) return error.BlockOversize;
162154 if (@intFromEnum(limit) < block_size) return error.OutputBufferUndersize;
155 var bytes_written: usize = 0;
163156 switch (block_header.type) {
164157 .raw => {
165158 try in.readAll(bw, .limited(block_size));
166 return block_size;
159 bytes_written = block_size;
167160 },
168161 .rle => {
169162 const byte = try in.takeByte();
170163 try bw.splatByteAll(byte, block_size);
171 return block_size;
164 bytes_written = block_size;
172165 },
173 .compressed => {},
174 .reserved => return error.ReservedBlock,
175 }
166 .compressed => {
167 var literal_fse_buffer: [zstd.table_size_max.literal]Table.Fse = undefined;
168 var match_fse_buffer: [zstd.table_size_max.match]Table.Fse = undefined;
169 var offset_fse_buffer: [zstd.table_size_max.offset]Table.Fse = undefined;
170 var literals_buffer: [zstd.block_size_max]u8 = undefined;
171 var sequence_buffer: [zstd.block_size_max]u8 = undefined;
172 var decode: Frame.Zstandard.Decode = .init(&literal_fse_buffer, &match_fse_buffer, &offset_fse_buffer);
173 var remaining: Reader.Limit = .limited(block_size);
174 const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);
175 const sequences_header = try SequencesSection.Header.decode(in, &remaining);
176
177 try decode.prepare(in, &remaining, literals, sequences_header);
178
179 {
180 if (sequence_buffer.len < @intFromEnum(remaining))
181 return error.SequenceBufferUndersize;
182 const seq_slice = remaining.slice(&sequence_buffer);
183 try in.readSlice(seq_slice);
184 var bit_stream = try ReverseBitReader.init(seq_slice);
176185
177 var remaining: Reader.Limit = .limited(block_size);
178 const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);
179 const sequences_header = try SequencesSection.Header.decode(in, &remaining);
186 if (sequences_header.sequence_count > 0) {
187 try decode.readInitialFseState(&bit_stream);
188
189 var sequence_size_limit = state.frame.block_size_max;
190 for (0..sequences_header.sequence_count) |i| {
191 const decompressed_size = try decode.decodeSequence(
192 bw,
193 &bit_stream,
194 sequence_size_limit,
195 i == sequences_header.sequence_count - 1,
196 );
197 sequence_size_limit -= decompressed_size;
198 bytes_written += decompressed_size;
199 }
200 }
180201
181 try decode.prepare(in, &remaining, literals, sequences_header);
202 if (!bit_stream.isEmpty()) {
203 return error.MalformedCompressedBlock;
204 }
205 }
182206
183 var bytes_written: usize = 0;
184 {
185 if (sequence_buffer.len < @intFromEnum(remaining))
186 return error.SequenceBufferUndersize;
187 const seq_slice = remaining.slice(&sequence_buffer);
188 try in.readSlice(seq_slice);
189 var bit_stream = try ReverseBitReader.init(seq_slice);
190
191 if (sequences_header.sequence_count > 0) {
192 try decode.readInitialFseState(&bit_stream);
193
194 var sequence_size_limit = state.frame.block_size_max;
195 for (0..sequences_header.sequence_count) |i| {
196 const decompressed_size = try decode.decodeSequence(
197 bw,
198 &bit_stream,
199 sequence_size_limit,
200 i == sequences_header.sequence_count - 1,
201 );
202 sequence_size_limit -= decompressed_size;
203 bytes_written += decompressed_size;
207 if (decode.literal_written_count < literals.header.regenerated_size) {
208 const len = literals.header.regenerated_size - decode.literal_written_count;
209 try decode.decodeLiterals(bw, len);
210 bytes_written += len;
204211 }
205 }
206212
207 if (!bit_stream.isEmpty()) {
208 return error.MalformedCompressedBlock;
209 }
210 }
213 switch (decode.literal_header.block_type) {
214 .treeless, .compressed => {
215 if (!decode.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
216 },
217 .raw, .rle => {},
218 }
211219
212 if (decode.literal_written_count < literals.header.regenerated_size) {
213 const len = literals.header.regenerated_size - decode.literal_written_count;
214 try decode.decodeLiterals(bw, len);
215 bytes_written += len;
216 }
220 if (bytes_written > state.frame.block_size_max) return error.BlockOversize;
221 if (remaining.nonzero()) return error.MalformedCompressedBlock;
217222
218 switch (decode.literal_header.block_type) {
219 .treeless, .compressed => {
220 if (!decode.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
223 state.decompressed_size += bytes_written;
224 if (state.frame.content_size) |size| {
225 if (state.decompressed_size > size) return error.MalformedFrame;
226 }
221227 },
222 .raw, .rle => {},
223 }
224
225 if (bytes_written > state.frame.block_size_max) return error.BlockOversize;
226 if (remaining.nonzero()) return error.MalformedCompressedBlock;
227
228 state.decompressed_size += bytes_written;
229
230 if (state.frame.content_size) |size| {
231 if (state.decompressed_size > size) return error.MalformedFrame;
228 .reserved => return error.ReservedBlock,
232229 }
233230
234231 if (state.frame.hasher_opt) |*hasher| {
lib/std/io/BufferedReader.zig+1-1
......@@ -730,7 +730,7 @@ pub fn fill(br: *BufferedReader, n: usize) Reader.Error!void {
730730 }
731731 if (seek > 0) {
732732 const remainder = buffer[seek..];
733 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
733 @memmove(buffer[0..remainder.len], remainder);
734734 br.end = remainder.len;
735735 br.seek = 0;
736736 }