authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2023-12-01 18:26:31+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-01-13 19:37:33-07:00
log6bfa7bf197634272f30d864a4563f7cddbaf55c0
tree0e6f0998ee2fe2d774b7344a648a6bc896e82b64
parent6e7a39c935b13dddc9153e534e5af8fe12bc5cac

tar: use scratch buffer for file names

That makes names strings stable during the iteration. Otherwise string buffers can be overwritten while reading file content.

1 files changed, 130 insertions(+), 159 deletions(-)

lib/std/tar.zig+130-159
......@@ -66,6 +66,7 @@ pub const Options = struct {
6666};
6767
6868const BLOCK_SIZE = 512;
69const MAX_HEADER_NAME_SIZE = 100 + 1 + 155; // name(100) + separator(1) + prefix(155)
6970
7071pub const Header = struct {
7172 bytes: *const [BLOCK_SIZE]u8,
......@@ -90,16 +91,14 @@ pub const Header = struct {
9091 };
9192
9293 /// Includes prefix concatenated, if any.
93 /// Return value may point into Header buffer, or might point into the
94 /// argument buffer.
9594 /// TODO: check against "../" and other nefarious things
96 pub fn fullFileName(header: Header, buffer: *[std.fs.MAX_PATH_BYTES]u8) ![]const u8 {
95 pub fn fullName(header: Header, buffer: *[MAX_HEADER_NAME_SIZE]u8) ![]const u8 {
9796 const n = name(header);
98 if (!is_ustar(header))
99 return n;
10097 const p = prefix(header);
101 if (p.len == 0)
102 return n;
98 if (!is_ustar(header) or p.len == 0) {
99 @memcpy(buffer[0..n.len], n);
100 return buffer[0..n.len];
101 }
103102 @memcpy(buffer[0..p.len], p);
104103 buffer[p.len] = '/';
105104 @memcpy(buffer[p.len + 1 ..][0..n.len], n);
......@@ -180,7 +179,7 @@ pub const Header = struct {
180179 }
181180
182181 // Checks calculated chksum with value of chksum field.
183 // Returns error or chksum value.
182 // Returns error or valid chksum value.
184183 // Zero value indicates empty block.
185184 pub fn checkChksum(header: Header) !u64 {
186185 const field = try header.chksum();
......@@ -190,7 +189,7 @@ pub const Header = struct {
190189 }
191190};
192191
193// break string on first null char
192// Breaks string on first null char.
194193fn nullStr(str: []const u8) []const u8 {
195194 for (str, 0..) |c, i| {
196195 if (c == 0) return str[0..i];
......@@ -198,14 +197,10 @@ fn nullStr(str: []const u8) []const u8 {
198197 return str;
199198}
200199
201// File size rounded to te block boundary.
202inline fn roundedFileSize(file_size: usize) usize {
203 return std.mem.alignForward(usize, file_size, BLOCK_SIZE);
204}
205
206200// Number of padding bytes in the last file block.
207inline fn filePadding(file_size: usize) usize {
208 return roundedFileSize(file_size) - file_size;
201inline fn blockPadding(size: usize) usize {
202 const block_rounded = std.mem.alignForward(usize, size, BLOCK_SIZE); // size rounded to te block boundary
203 return block_rounded - size;
209204}
210205
211206fn BufferedReader(comptime ReaderType: type) type {
......@@ -217,44 +212,38 @@ fn BufferedReader(comptime ReaderType: type) type {
217212
218213 const Self = @This();
219214
220 fn readChunk(self: *Self, count: usize) ![]const u8 {
221 self.ensureCapacity(BLOCK_SIZE * 2);
222 const ask = @min(self.buffer.len - self.end, count -| (self.end - self.start));
223 self.end += try self.unbuffered_reader.readAtLeast(self.buffer[self.end..], ask);
224 return self.buffer[self.start..self.end];
215 // Fills buffer from underlaying reader.
216 fn fillBuffer(self: *Self) !void {
217 self.removeUsed();
218 self.end += try self.unbuffered_reader.read(self.buffer[self.end..]);
225219 }
226220
227 // Returns slice of size count or part of it.
221 // Returns slice of size count or how much fits into buffer.
228222 pub fn readSlice(self: *Self, count: usize) ![]const u8 {
229223 if (count <= self.end - self.start) {
230 // fastpath, we have enough bytes in buffer
231224 return self.buffer[self.start .. self.start + count];
232225 }
233
234 const chunk_size = roundedFileSize(count) + BLOCK_SIZE;
235 const temp = try self.readChunk(chunk_size);
236 if (temp.len == 0) return error.UnexpectedEndOfStream;
237 return temp[0..@min(count, temp.len)];
226 try self.fillBuffer();
227 const buf = self.buffer[self.start..self.end];
228 if (buf.len == 0) return error.UnexpectedEndOfStream;
229 return buf[0..@min(count, buf.len)];
238230 }
239231
240 // Returns tar header block, 512 bytes. Before reading advances buffer
241 // for padding of the previous block, to position reader at the start of
242 // new block. After reading advances for block size, to position reader
243 // at the start of the file body.
244 pub fn readBlock(self: *Self, padding: usize) !?[]const u8 {
232 // Returns tar header block, 512 bytes, or null if eof. Before reading
233 // advances buffer for padding of the previous block, to position reader
234 // at the start of new block. After reading advances for block size, to
235 // position reader at the start of the file content.
236 pub fn readHeader(self: *Self, padding: usize) !?[]const u8 {
245237 try self.skip(padding);
246 const block_bytes = try self.readChunk(BLOCK_SIZE * 2);
247 switch (block_bytes.len) {
248 0 => return null,
249 1...(BLOCK_SIZE - 1) => return error.UnexpectedEndOfStream,
250 else => {},
251 }
238 const buf = self.readSlice(BLOCK_SIZE) catch return null;
239 if (buf.len < BLOCK_SIZE) return error.UnexpectedEndOfStream;
252240 self.advance(BLOCK_SIZE);
253 return block_bytes[0..BLOCK_SIZE];
241 return buf[0..BLOCK_SIZE];
254242 }
255243
256 // Retruns byte at current position in buffer.
244 // Returns byte at current position in buffer.
257245 pub fn readByte(self: *@This()) u8 {
246 assert(self.start < self.end);
258247 return self.buffer[self.start];
259248 }
260249
......@@ -275,78 +264,36 @@ fn BufferedReader(comptime ReaderType: type) type {
275264 }
276265 }
277266
278 inline fn ensureCapacity(self: *Self, count: usize) void {
279 if (self.buffer.len - self.start < count) {
280 const dest_end = self.end - self.start;
281 @memcpy(self.buffer[0..dest_end], self.buffer[self.start..self.end]);
282 self.end = dest_end;
283 self.start = 0;
284 }
267 // Removes used part of the buffer.
268 inline fn removeUsed(self: *Self) void {
269 const dest_end = self.end - self.start;
270 if (self.start == 0 or dest_end > self.start) return;
271 @memcpy(self.buffer[0..dest_end], self.buffer[self.start..self.end]);
272 self.end = dest_end;
273 self.start = 0;
285274 }
286275
287 // Write count bytes to the writer.
276 // Writes count bytes to the writer. Advances reader.
288277 pub fn write(self: *Self, writer: anytype, count: usize) !void {
289 if (self.read(count)) |buf| {
290 try writer.writeAll(buf);
291 return;
292 }
293 var rdr = self.sliceReader(count);
294 while (try rdr.next()) |slice| {
278 var pos: usize = 0;
279 while (pos < count) {
280 const slice = try self.readSlice(count - pos);
295281 try writer.writeAll(slice);
282 self.advance(slice.len);
283 pos += slice.len;
296284 }
297285 }
298286
299 // Copy dst.len bytes into dst buffer.
287 // Copies dst.len bytes into dst buffer. Advances reader.
300288 pub fn copy(self: *Self, dst: []u8) ![]const u8 {
301 if (self.read(dst.len)) |buf| {
302 // fastpath we already have enough bytes in buffer
303 @memcpy(dst, buf);
304 return dst;
305 }
306 var rdr = self.sliceReader(dst.len);
307289 var pos: usize = 0;
308 while (try rdr.next()) |slice| : (pos += slice.len) {
290 while (pos < dst.len) {
291 const slice = try self.readSlice(dst.len - pos);
309292 @memcpy(dst[pos .. pos + slice.len], slice);
310 }
311 return dst;
312 }
313
314 // Retruns count bytes from buffer and advances for that number of
315 // bytes. If we don't have that much bytes buffered returns null.
316 fn read(self: *Self, count: usize) ?[]const u8 {
317 if (count <= self.end - self.start) {
318 const buf = self.buffer[self.start .. self.start + count];
319 self.advance(count);
320 return buf;
321 }
322 return null;
323 }
324
325 const SliceReader = struct {
326 size: usize,
327 offset: usize,
328 reader: *Self,
329
330 pub fn next(self: *SliceReader) !?[]const u8 {
331 const remaining_size = self.size - self.offset;
332 if (remaining_size == 0) return null;
333 const slice = try self.reader.readSlice(remaining_size);
334293 self.advance(slice.len);
335 return slice;
336 }
337
338 fn advance(self: *SliceReader, len: usize) void {
339 self.offset += len;
340 self.reader.advance(len);
294 pos += slice.len;
341295 }
342 };
343
344 pub fn sliceReader(self: *Self, size: usize) SliceReader {
345 return .{
346 .size = size,
347 .reader = self,
348 .offset = 0,
349 };
296 return dst;
350297 }
351298
352299 pub fn paxFileReader(self: *Self, size: usize) PaxFileReader {
......@@ -388,9 +335,6 @@ fn BufferedReader(comptime ReaderType: type) type {
388335 // Caller of the next has to call value in PaxAttribute, to advance
389336 // reader across value.
390337 pub fn next(self: *PaxFileReader) !?PaxAttribute {
391 const rdr = self.reader;
392 _ = rdr;
393
394338 while (true) {
395339 const remaining_size = self.size - self.offset;
396340 if (remaining_size == 0) return null;
......@@ -433,10 +377,14 @@ fn Iterator(comptime ReaderType: type) type {
433377 return struct {
434378 // scratch buffer for file attributes
435379 scratch: struct {
436 // size: two paths (name and link_name) and size (24 in pax attribute)
380 // size: two paths (name and link_name) and files size bytes (24 in pax attribute)
437381 buffer: [std.fs.MAX_PATH_BYTES * 2 + 24]u8 = undefined,
438382 tail: usize = 0,
439383
384 name: []const u8 = undefined,
385 link_name: []const u8 = undefined,
386 size: usize = 0,
387
440388 // Allocate size of the buffer for some attribute.
441389 fn alloc(self: *@This(), size: usize) ![]u8 {
442390 const free_size = self.buffer.len - self.tail;
......@@ -447,45 +395,53 @@ fn Iterator(comptime ReaderType: type) type {
447395 return self.buffer[head..self.tail];
448396 }
449397
450 // Free whole buffer.
451 fn free(self: *@This()) void {
398 // Reset buffer and all fields.
399 fn reset(self: *@This()) void {
452400 self.tail = 0;
401 self.name = self.buffer[0..0];
402 self.link_name = self.buffer[0..0];
403 self.size = 0;
404 }
405
406 fn append(self: *@This(), header: Header) !void {
407 if (self.size == 0) self.size = try header.fileSize();
408 if (self.link_name.len == 0) {
409 const link_name = header.linkName();
410 if (link_name.len > 0) {
411 const buf = try self.alloc(link_name.len);
412 @memcpy(buf, link_name);
413 self.link_name = buf;
414 }
415 }
416 if (self.name.len == 0) {
417 self.name = try header.fullName((try self.alloc(MAX_HEADER_NAME_SIZE))[0..MAX_HEADER_NAME_SIZE]);
418 }
453419 }
454420 } = .{},
455421
456422 reader: BufferedReaderType,
457423 diagnostics: ?*Options.Diagnostics,
458 padding: usize = 0, // bytes of file padding
424 padding: usize = 0, // bytes of padding to the end of the block
459425
460426 const Self = @This();
461427
462 const File = struct {
463 name: []const u8 = &[_]u8{},
464 link_name: []const u8 = &[_]u8{},
465 size: usize = 0,
466 file_type: Header.FileType = .normal,
428 pub const File = struct {
429 name: []const u8, // name of file, symlink or directory
430 link_name: []const u8, // target name of symlink
431 size: usize, // size of the file in bytes
432 file_type: Header.FileType,
433
467434 reader: *BufferedReaderType,
468435
436 // Writes file content to writer.
469437 pub fn write(self: File, writer: anytype) !void {
470438 try self.reader.write(writer, self.size);
471439 }
472440
441 // Skips file content. Advances reader.
473442 pub fn skip(self: File) !void {
474443 try self.reader.skip(self.size);
475444 }
476
477 fn chksum(self: File) ![16]u8 {
478 var sum = [_]u8{0} ** 16;
479 if (self.size == 0) return sum;
480
481 var rdr = self.reader.sliceReader(self.size);
482 var h = std.crypto.hash.Md5.init(.{});
483 while (try rdr.next()) |slice| {
484 h.update(slice);
485 }
486 h.final(&sum);
487 return sum;
488 }
489445 };
490446
491447 // Externally, `next` iterates through the tar archive as if it is a
......@@ -495,62 +451,62 @@ fn Iterator(comptime ReaderType: type) type {
495451 // loop iterates through one or more "header files" until it finds a
496452 // "normal file".
497453 pub fn next(self: *Self) !?File {
498 var file: File = .{ .reader = &self.reader };
499 self.scratch.free();
454 self.scratch.reset();
500455
501 while (try self.reader.readBlock(self.padding)) |block_bytes| {
456 while (try self.reader.readHeader(self.padding)) |block_bytes| {
502457 const header = Header{ .bytes = block_bytes[0..BLOCK_SIZE] };
503458 if (try header.checkChksum() == 0) return null; // zero block found
504459
505460 const file_type = header.fileType();
506 const file_size = try header.fileSize();
507 self.padding = filePadding(file_size);
461 const size: usize = @intCast(try header.fileSize());
462 self.padding = blockPadding(size);
508463
509464 switch (file_type) {
510 // file types to retrun from next
465 // File types to retrun upstream
511466 .directory, .normal, .symbolic_link => {
512 if (file.size == 0) file.size = file_size;
513 self.padding = filePadding(file.size);
514
515 if (file.name.len == 0)
516 file.name = try header.fullFileName((try self.scratch.alloc(std.fs.MAX_PATH_BYTES))[0..std.fs.MAX_PATH_BYTES]);
517 if (file.link_name.len == 0) file.link_name = header.linkName();
518 file.file_type = file_type;
467 try self.scratch.append(header);
468 const file = File{
469 .file_type = file_type,
470 .name = self.scratch.name,
471 .link_name = self.scratch.link_name,
472 .size = self.scratch.size,
473 .reader = &self.reader,
474 };
475 self.padding = blockPadding(file.size);
519476 return file;
520477 },
521 // prefix header types
478 // Prefix header types
522479 .gnu_long_name => {
523 file.name = nullStr(try self.reader.copy(try self.scratch.alloc(file_size)));
480 self.scratch.name = nullStr(try self.reader.copy(try self.scratch.alloc(size)));
524481 },
525482 .gnu_long_link => {
526 file.link_name = nullStr(try self.reader.copy(try self.scratch.alloc(file_size)));
483 self.scratch.link_name = nullStr(try self.reader.copy(try self.scratch.alloc(size)));
527484 },
528485 .extended_header => {
529 if (file_size == 0) continue;
530 // use just last extended header data
531 self.scratch.free();
532 file = File{ .reader = &self.reader };
486 if (size == 0) continue;
487 // Use just attributes from last extended header.
488 self.scratch.reset();
533489
534 var rdr = self.reader.paxFileReader(file_size);
490 var rdr = self.reader.paxFileReader(size);
535491 while (try rdr.next()) |attr| {
536492 switch (attr.key) {
537493 .path => {
538 file.name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));
494 self.scratch.name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));
539495 },
540496 .linkpath => {
541 file.link_name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));
497 self.scratch.link_name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));
542498 },
543499 .size => {
544 file.size = try std.fmt.parseInt(usize, try attr.value(try self.scratch.alloc(attr.value_len)), 10);
500 self.scratch.size = try std.fmt.parseInt(usize, try attr.value(try self.scratch.alloc(attr.value_len)), 10);
545501 },
546502 }
547503 }
548504 },
549 // ignored header types
505 // Ignored header type
550506 .global_extended_header => {
551 self.reader.skip(file_size) catch return error.TarHeadersTooBig;
507 self.reader.skip(size) catch return error.TarHeadersTooBig;
552508 },
553 // unsupported header types
509 // All other are unsupported header types
554510 else => {
555511 const d = self.diagnostics orelse return error.TarUnsupportedFileType;
556512 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
......@@ -1053,16 +1009,31 @@ test "tar: Go test cases" {
10531009 try std.testing.expectEqualStrings(expected.link_name, actual.link_name);
10541010
10551011 if (case.chksums.len > i) {
1056 var actual_chksum = try actual.chksum();
1057 var hex_to_bytes_buffer: [16]u8 = undefined;
1058 const expected_chksum = try std.fmt.hexToBytes(&hex_to_bytes_buffer, case.chksums[i]);
1059 // std.debug.print("actual chksum: {s}\n", .{std.fmt.fmtSliceHexLower(&actual_chksum)});
1060 try std.testing.expectEqualStrings(expected_chksum, &actual_chksum);
1012 var md5writer = Md5Writer{};
1013 try actual.write(&md5writer);
1014 const chksum = md5writer.chksum();
1015 // std.debug.print("actual chksum: {s}\n", .{chksum});
1016 try std.testing.expectEqualStrings(case.chksums[i], &chksum);
10611017 } else {
10621018 if (!expected.truncated) try actual.skip(); // skip file content
10631019 }
1064 i += 1;
10651020 }
10661021 try std.testing.expectEqual(case.files.len, i);
10671022 }
10681023}
1024
1025// used in test to calculate file chksum
1026const Md5Writer = struct {
1027 h: std.crypto.hash.Md5 = std.crypto.hash.Md5.init(.{}),
1028
1029 pub fn writeAll(self: *Md5Writer, buf: []const u8) !void {
1030 self.h.update(buf);
1031 }
1032
1033 pub fn chksum(self: *Md5Writer) [32]u8 {
1034 var s = [_]u8{0} ** 16;
1035 self.h.final(&s);
1036 return std.fmt.bytesToHex(s, .lower);
1037 }
1038};
1039