authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2023-12-01 18:26:31+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-01-13 19:37:33-07:00
log6bfa7bf197634272f30d864a4563f7cddbaf55c0
tree0e6f0998ee2fe2d774b7344a648a6bc896e82b64
parent6e7a39c935b13dddc9153e534e5af8fe12bc5cac

tar: use scratch buffer for file names

That makes names strings stable during the iteration. Otherwise string buffers can be overwritten while reading file content.

1 files changed, 130 insertions(+), 159 deletions(-)

lib/std/tar.zig+130-159
...@@ -66,6 +66,7 @@ pub const Options = struct {...@@ -66,6 +66,7 @@ pub const Options = struct {
66};66};
6767
68const BLOCK_SIZE = 512;68const BLOCK_SIZE = 512;
69const MAX_HEADER_NAME_SIZE = 100 + 1 + 155; // name(100) + separator(1) + prefix(155)
6970
70pub const Header = struct {71pub const Header = struct {
71 bytes: *const [BLOCK_SIZE]u8,72 bytes: *const [BLOCK_SIZE]u8,
...@@ -90,16 +91,14 @@ pub const Header = struct {...@@ -90,16 +91,14 @@ pub const Header = struct {
90 };91 };
9192
92 /// Includes prefix concatenated, if any.93 /// Includes prefix concatenated, if any.
93 /// Return value may point into Header buffer, or might point into the
94 /// argument buffer.
95 /// TODO: check against "../" and other nefarious things94 /// TODO: check against "../" and other nefarious things
96 pub fn fullFileName(header: Header, buffer: *[std.fs.MAX_PATH_BYTES]u8) ![]const u8 {95 pub fn fullName(header: Header, buffer: *[MAX_HEADER_NAME_SIZE]u8) ![]const u8 {
97 const n = name(header);96 const n = name(header);
98 if (!is_ustar(header))
99 return n;
100 const p = prefix(header);97 const p = prefix(header);
101 if (p.len == 0)98 if (!is_ustar(header) or p.len == 0) {
102 return n;99 @memcpy(buffer[0..n.len], n);
100 return buffer[0..n.len];
101 }
103 @memcpy(buffer[0..p.len], p);102 @memcpy(buffer[0..p.len], p);
104 buffer[p.len] = '/';103 buffer[p.len] = '/';
105 @memcpy(buffer[p.len + 1 ..][0..n.len], n);104 @memcpy(buffer[p.len + 1 ..][0..n.len], n);
...@@ -180,7 +179,7 @@ pub const Header = struct {...@@ -180,7 +179,7 @@ pub const Header = struct {
180 }179 }
181180
182 // Checks calculated chksum with value of chksum field.181 // Checks calculated chksum with value of chksum field.
183 // Returns error or chksum value.182 // Returns error or valid chksum value.
184 // Zero value indicates empty block.183 // Zero value indicates empty block.
185 pub fn checkChksum(header: Header) !u64 {184 pub fn checkChksum(header: Header) !u64 {
186 const field = try header.chksum();185 const field = try header.chksum();
...@@ -190,7 +189,7 @@ pub const Header = struct {...@@ -190,7 +189,7 @@ pub const Header = struct {
190 }189 }
191};190};
192191
193// break string on first null char192// Breaks string on first null char.
194fn nullStr(str: []const u8) []const u8 {193fn nullStr(str: []const u8) []const u8 {
195 for (str, 0..) |c, i| {194 for (str, 0..) |c, i| {
196 if (c == 0) return str[0..i];195 if (c == 0) return str[0..i];
...@@ -198,14 +197,10 @@ fn nullStr(str: []const u8) []const u8 {...@@ -198,14 +197,10 @@ fn nullStr(str: []const u8) []const u8 {
198 return str;197 return str;
199}198}
200199
201// File size rounded to te block boundary.
202inline fn roundedFileSize(file_size: usize) usize {
203 return std.mem.alignForward(usize, file_size, BLOCK_SIZE);
204}
205
206// Number of padding bytes in the last file block.200// Number of padding bytes in the last file block.
207inline fn filePadding(file_size: usize) usize {201inline fn blockPadding(size: usize) usize {
208 return roundedFileSize(file_size) - file_size;202 const block_rounded = std.mem.alignForward(usize, size, BLOCK_SIZE); // size rounded to te block boundary
203 return block_rounded - size;
209}204}
210205
211fn BufferedReader(comptime ReaderType: type) type {206fn BufferedReader(comptime ReaderType: type) type {
...@@ -217,44 +212,38 @@ fn BufferedReader(comptime ReaderType: type) type {...@@ -217,44 +212,38 @@ fn BufferedReader(comptime ReaderType: type) type {
217212
218 const Self = @This();213 const Self = @This();
219214
220 fn readChunk(self: *Self, count: usize) ![]const u8 {215 // Fills buffer from underlaying reader.
221 self.ensureCapacity(BLOCK_SIZE * 2);216 fn fillBuffer(self: *Self) !void {
222 const ask = @min(self.buffer.len - self.end, count -| (self.end - self.start));217 self.removeUsed();
223 self.end += try self.unbuffered_reader.readAtLeast(self.buffer[self.end..], ask);218 self.end += try self.unbuffered_reader.read(self.buffer[self.end..]);
224 return self.buffer[self.start..self.end];
225 }219 }
226220
227 // Returns slice of size count or part of it.221 // Returns slice of size count or how much fits into buffer.
228 pub fn readSlice(self: *Self, count: usize) ![]const u8 {222 pub fn readSlice(self: *Self, count: usize) ![]const u8 {
229 if (count <= self.end - self.start) {223 if (count <= self.end - self.start) {
230 // fastpath, we have enough bytes in buffer
231 return self.buffer[self.start .. self.start + count];224 return self.buffer[self.start .. self.start + count];
232 }225 }
233226 try self.fillBuffer();
234 const chunk_size = roundedFileSize(count) + BLOCK_SIZE;227 const buf = self.buffer[self.start..self.end];
235 const temp = try self.readChunk(chunk_size);228 if (buf.len == 0) return error.UnexpectedEndOfStream;
236 if (temp.len == 0) return error.UnexpectedEndOfStream;229 return buf[0..@min(count, buf.len)];
237 return temp[0..@min(count, temp.len)];
238 }230 }
239231
240 // Returns tar header block, 512 bytes. Before reading advances buffer232 // Returns tar header block, 512 bytes, or null if eof. Before reading
241 // for padding of the previous block, to position reader at the start of233 // advances buffer for padding of the previous block, to position reader
242 // new block. After reading advances for block size, to position reader234 // at the start of new block. After reading advances for block size, to
243 // at the start of the file body.235 // position reader at the start of the file content.
244 pub fn readBlock(self: *Self, padding: usize) !?[]const u8 {236 pub fn readHeader(self: *Self, padding: usize) !?[]const u8 {
245 try self.skip(padding);237 try self.skip(padding);
246 const block_bytes = try self.readChunk(BLOCK_SIZE * 2);238 const buf = self.readSlice(BLOCK_SIZE) catch return null;
247 switch (block_bytes.len) {239 if (buf.len < BLOCK_SIZE) return error.UnexpectedEndOfStream;
248 0 => return null,
249 1...(BLOCK_SIZE - 1) => return error.UnexpectedEndOfStream,
250 else => {},
251 }
252 self.advance(BLOCK_SIZE);240 self.advance(BLOCK_SIZE);
253 return block_bytes[0..BLOCK_SIZE];241 return buf[0..BLOCK_SIZE];
254 }242 }
255243
256 // Retruns byte at current position in buffer.244 // Returns byte at current position in buffer.
257 pub fn readByte(self: *@This()) u8 {245 pub fn readByte(self: *@This()) u8 {
246 assert(self.start < self.end);
258 return self.buffer[self.start];247 return self.buffer[self.start];
259 }248 }
260249
...@@ -275,78 +264,36 @@ fn BufferedReader(comptime ReaderType: type) type {...@@ -275,78 +264,36 @@ fn BufferedReader(comptime ReaderType: type) type {
275 }264 }
276 }265 }
277266
278 inline fn ensureCapacity(self: *Self, count: usize) void {267 // Removes used part of the buffer.
279 if (self.buffer.len - self.start < count) {268 inline fn removeUsed(self: *Self) void {
280 const dest_end = self.end - self.start;269 const dest_end = self.end - self.start;
281 @memcpy(self.buffer[0..dest_end], self.buffer[self.start..self.end]);270 if (self.start == 0 or dest_end > self.start) return;
282 self.end = dest_end;271 @memcpy(self.buffer[0..dest_end], self.buffer[self.start..self.end]);
283 self.start = 0;272 self.end = dest_end;
284 }273 self.start = 0;
285 }274 }
286275
287 // Write count bytes to the writer.276 // Writes count bytes to the writer. Advances reader.
288 pub fn write(self: *Self, writer: anytype, count: usize) !void {277 pub fn write(self: *Self, writer: anytype, count: usize) !void {
289 if (self.read(count)) |buf| {278 var pos: usize = 0;
290 try writer.writeAll(buf);279 while (pos < count) {
291 return;280 const slice = try self.readSlice(count - pos);
292 }
293 var rdr = self.sliceReader(count);
294 while (try rdr.next()) |slice| {
295 try writer.writeAll(slice);281 try writer.writeAll(slice);
282 self.advance(slice.len);
283 pos += slice.len;
296 }284 }
297 }285 }
298286
299 // Copy dst.len bytes into dst buffer.287 // Copies dst.len bytes into dst buffer. Advances reader.
300 pub fn copy(self: *Self, dst: []u8) ![]const u8 {288 pub fn copy(self: *Self, dst: []u8) ![]const u8 {
301 if (self.read(dst.len)) |buf| {
302 // fastpath we already have enough bytes in buffer
303 @memcpy(dst, buf);
304 return dst;
305 }
306 var rdr = self.sliceReader(dst.len);
307 var pos: usize = 0;289 var pos: usize = 0;
308 while (try rdr.next()) |slice| : (pos += slice.len) {290 while (pos < dst.len) {
291 const slice = try self.readSlice(dst.len - pos);
309 @memcpy(dst[pos .. pos + slice.len], slice);292 @memcpy(dst[pos .. pos + slice.len], slice);
310 }
311 return dst;
312 }
313
314 // Retruns count bytes from buffer and advances for that number of
315 // bytes. If we don't have that much bytes buffered returns null.
316 fn read(self: *Self, count: usize) ?[]const u8 {
317 if (count <= self.end - self.start) {
318 const buf = self.buffer[self.start .. self.start + count];
319 self.advance(count);
320 return buf;
321 }
322 return null;
323 }
324
325 const SliceReader = struct {
326 size: usize,
327 offset: usize,
328 reader: *Self,
329
330 pub fn next(self: *SliceReader) !?[]const u8 {
331 const remaining_size = self.size - self.offset;
332 if (remaining_size == 0) return null;
333 const slice = try self.reader.readSlice(remaining_size);
334 self.advance(slice.len);293 self.advance(slice.len);
335 return slice;294 pos += slice.len;
336 }
337
338 fn advance(self: *SliceReader, len: usize) void {
339 self.offset += len;
340 self.reader.advance(len);
341 }295 }
342 };296 return dst;
343
344 pub fn sliceReader(self: *Self, size: usize) SliceReader {
345 return .{
346 .size = size,
347 .reader = self,
348 .offset = 0,
349 };
350 }297 }
351298
352 pub fn paxFileReader(self: *Self, size: usize) PaxFileReader {299 pub fn paxFileReader(self: *Self, size: usize) PaxFileReader {
...@@ -388,9 +335,6 @@ fn BufferedReader(comptime ReaderType: type) type {...@@ -388,9 +335,6 @@ fn BufferedReader(comptime ReaderType: type) type {
388 // Caller of the next has to call value in PaxAttribute, to advance335 // Caller of the next has to call value in PaxAttribute, to advance
389 // reader across value.336 // reader across value.
390 pub fn next(self: *PaxFileReader) !?PaxAttribute {337 pub fn next(self: *PaxFileReader) !?PaxAttribute {
391 const rdr = self.reader;
392 _ = rdr;
393
394 while (true) {338 while (true) {
395 const remaining_size = self.size - self.offset;339 const remaining_size = self.size - self.offset;
396 if (remaining_size == 0) return null;340 if (remaining_size == 0) return null;
...@@ -433,10 +377,14 @@ fn Iterator(comptime ReaderType: type) type {...@@ -433,10 +377,14 @@ fn Iterator(comptime ReaderType: type) type {
433 return struct {377 return struct {
434 // scratch buffer for file attributes378 // scratch buffer for file attributes
435 scratch: struct {379 scratch: struct {
436 // size: two paths (name and link_name) and size (24 in pax attribute)380 // size: two paths (name and link_name) and files size bytes (24 in pax attribute)
437 buffer: [std.fs.MAX_PATH_BYTES * 2 + 24]u8 = undefined,381 buffer: [std.fs.MAX_PATH_BYTES * 2 + 24]u8 = undefined,
438 tail: usize = 0,382 tail: usize = 0,
439383
384 name: []const u8 = undefined,
385 link_name: []const u8 = undefined,
386 size: usize = 0,
387
440 // Allocate size of the buffer for some attribute.388 // Allocate size of the buffer for some attribute.
441 fn alloc(self: *@This(), size: usize) ![]u8 {389 fn alloc(self: *@This(), size: usize) ![]u8 {
442 const free_size = self.buffer.len - self.tail;390 const free_size = self.buffer.len - self.tail;
...@@ -447,45 +395,53 @@ fn Iterator(comptime ReaderType: type) type {...@@ -447,45 +395,53 @@ fn Iterator(comptime ReaderType: type) type {
447 return self.buffer[head..self.tail];395 return self.buffer[head..self.tail];
448 }396 }
449397
450 // Free whole buffer.398 // Reset buffer and all fields.
451 fn free(self: *@This()) void {399 fn reset(self: *@This()) void {
452 self.tail = 0;400 self.tail = 0;
401 self.name = self.buffer[0..0];
402 self.link_name = self.buffer[0..0];
403 self.size = 0;
404 }
405
406 fn append(self: *@This(), header: Header) !void {
407 if (self.size == 0) self.size = try header.fileSize();
408 if (self.link_name.len == 0) {
409 const link_name = header.linkName();
410 if (link_name.len > 0) {
411 const buf = try self.alloc(link_name.len);
412 @memcpy(buf, link_name);
413 self.link_name = buf;
414 }
415 }
416 if (self.name.len == 0) {
417 self.name = try header.fullName((try self.alloc(MAX_HEADER_NAME_SIZE))[0..MAX_HEADER_NAME_SIZE]);
418 }
453 }419 }
454 } = .{},420 } = .{},
455421
456 reader: BufferedReaderType,422 reader: BufferedReaderType,
457 diagnostics: ?*Options.Diagnostics,423 diagnostics: ?*Options.Diagnostics,
458 padding: usize = 0, // bytes of file padding424 padding: usize = 0, // bytes of padding to the end of the block
459425
460 const Self = @This();426 const Self = @This();
461427
462 const File = struct {428 pub const File = struct {
463 name: []const u8 = &[_]u8{},429 name: []const u8, // name of file, symlink or directory
464 link_name: []const u8 = &[_]u8{},430 link_name: []const u8, // target name of symlink
465 size: usize = 0,431 size: usize, // size of the file in bytes
466 file_type: Header.FileType = .normal,432 file_type: Header.FileType,
433
467 reader: *BufferedReaderType,434 reader: *BufferedReaderType,
468435
436 // Writes file content to writer.
469 pub fn write(self: File, writer: anytype) !void {437 pub fn write(self: File, writer: anytype) !void {
470 try self.reader.write(writer, self.size);438 try self.reader.write(writer, self.size);
471 }439 }
472440
441 // Skips file content. Advances reader.
473 pub fn skip(self: File) !void {442 pub fn skip(self: File) !void {
474 try self.reader.skip(self.size);443 try self.reader.skip(self.size);
475 }444 }
476
477 fn chksum(self: File) ![16]u8 {
478 var sum = [_]u8{0} ** 16;
479 if (self.size == 0) return sum;
480
481 var rdr = self.reader.sliceReader(self.size);
482 var h = std.crypto.hash.Md5.init(.{});
483 while (try rdr.next()) |slice| {
484 h.update(slice);
485 }
486 h.final(&sum);
487 return sum;
488 }
489 };445 };
490446
491 // Externally, `next` iterates through the tar archive as if it is a447 // Externally, `next` iterates through the tar archive as if it is a
...@@ -495,62 +451,62 @@ fn Iterator(comptime ReaderType: type) type {...@@ -495,62 +451,62 @@ fn Iterator(comptime ReaderType: type) type {
495 // loop iterates through one or more "header files" until it finds a451 // loop iterates through one or more "header files" until it finds a
496 // "normal file".452 // "normal file".
497 pub fn next(self: *Self) !?File {453 pub fn next(self: *Self) !?File {
498 var file: File = .{ .reader = &self.reader };454 self.scratch.reset();
499 self.scratch.free();
500455
501 while (try self.reader.readBlock(self.padding)) |block_bytes| {456 while (try self.reader.readHeader(self.padding)) |block_bytes| {
502 const header = Header{ .bytes = block_bytes[0..BLOCK_SIZE] };457 const header = Header{ .bytes = block_bytes[0..BLOCK_SIZE] };
503 if (try header.checkChksum() == 0) return null; // zero block found458 if (try header.checkChksum() == 0) return null; // zero block found
504459
505 const file_type = header.fileType();460 const file_type = header.fileType();
506 const file_size = try header.fileSize();461 const size: usize = @intCast(try header.fileSize());
507 self.padding = filePadding(file_size);462 self.padding = blockPadding(size);
508463
509 switch (file_type) {464 switch (file_type) {
510 // file types to retrun from next465 // File types to retrun upstream
511 .directory, .normal, .symbolic_link => {466 .directory, .normal, .symbolic_link => {
512 if (file.size == 0) file.size = file_size;467 try self.scratch.append(header);
513 self.padding = filePadding(file.size);468 const file = File{
514469 .file_type = file_type,
515 if (file.name.len == 0)470 .name = self.scratch.name,
516 file.name = try header.fullFileName((try self.scratch.alloc(std.fs.MAX_PATH_BYTES))[0..std.fs.MAX_PATH_BYTES]);471 .link_name = self.scratch.link_name,
517 if (file.link_name.len == 0) file.link_name = header.linkName();472 .size = self.scratch.size,
518 file.file_type = file_type;473 .reader = &self.reader,
474 };
475 self.padding = blockPadding(file.size);
519 return file;476 return file;
520 },477 },
521 // prefix header types478 // Prefix header types
522 .gnu_long_name => {479 .gnu_long_name => {
523 file.name = nullStr(try self.reader.copy(try self.scratch.alloc(file_size)));480 self.scratch.name = nullStr(try self.reader.copy(try self.scratch.alloc(size)));
524 },481 },
525 .gnu_long_link => {482 .gnu_long_link => {
526 file.link_name = nullStr(try self.reader.copy(try self.scratch.alloc(file_size)));483 self.scratch.link_name = nullStr(try self.reader.copy(try self.scratch.alloc(size)));
527 },484 },
528 .extended_header => {485 .extended_header => {
529 if (file_size == 0) continue;486 if (size == 0) continue;
530 // use just last extended header data487 // Use just attributes from last extended header.
531 self.scratch.free();488 self.scratch.reset();
532 file = File{ .reader = &self.reader };
533489
534 var rdr = self.reader.paxFileReader(file_size);490 var rdr = self.reader.paxFileReader(size);
535 while (try rdr.next()) |attr| {491 while (try rdr.next()) |attr| {
536 switch (attr.key) {492 switch (attr.key) {
537 .path => {493 .path => {
538 file.name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));494 self.scratch.name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));
539 },495 },
540 .linkpath => {496 .linkpath => {
541 file.link_name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));497 self.scratch.link_name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));
542 },498 },
543 .size => {499 .size => {
544 file.size = try std.fmt.parseInt(usize, try attr.value(try self.scratch.alloc(attr.value_len)), 10);500 self.scratch.size = try std.fmt.parseInt(usize, try attr.value(try self.scratch.alloc(attr.value_len)), 10);
545 },501 },
546 }502 }
547 }503 }
548 },504 },
549 // ignored header types505 // Ignored header type
550 .global_extended_header => {506 .global_extended_header => {
551 self.reader.skip(file_size) catch return error.TarHeadersTooBig;507 self.reader.skip(size) catch return error.TarHeadersTooBig;
552 },508 },
553 // unsupported header types509 // All other are unsupported header types
554 else => {510 else => {
555 const d = self.diagnostics orelse return error.TarUnsupportedFileType;511 const d = self.diagnostics orelse return error.TarUnsupportedFileType;
556 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{512 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
...@@ -1053,16 +1009,31 @@ test "tar: Go test cases" {...@@ -1053,16 +1009,31 @@ test "tar: Go test cases" {
1053 try std.testing.expectEqualStrings(expected.link_name, actual.link_name);1009 try std.testing.expectEqualStrings(expected.link_name, actual.link_name);
10541010
1055 if (case.chksums.len > i) {1011 if (case.chksums.len > i) {
1056 var actual_chksum = try actual.chksum();1012 var md5writer = Md5Writer{};
1057 var hex_to_bytes_buffer: [16]u8 = undefined;1013 try actual.write(&md5writer);
1058 const expected_chksum = try std.fmt.hexToBytes(&hex_to_bytes_buffer, case.chksums[i]);1014 const chksum = md5writer.chksum();
1059 // std.debug.print("actual chksum: {s}\n", .{std.fmt.fmtSliceHexLower(&actual_chksum)});1015 // std.debug.print("actual chksum: {s}\n", .{chksum});
1060 try std.testing.expectEqualStrings(expected_chksum, &actual_chksum);1016 try std.testing.expectEqualStrings(case.chksums[i], &chksum);
1061 } else {1017 } else {
1062 if (!expected.truncated) try actual.skip(); // skip file content1018 if (!expected.truncated) try actual.skip(); // skip file content
1063 }1019 }
1064 i += 1;
1065 }1020 }
1066 try std.testing.expectEqual(case.files.len, i);1021 try std.testing.expectEqual(case.files.len, i);
1067 }1022 }
1068}1023}
1024
1025// used in test to calculate file chksum
1026const Md5Writer = struct {
1027 h: std.crypto.hash.Md5 = std.crypto.hash.Md5.init(.{}),
1028
1029 pub fn writeAll(self: *Md5Writer, buf: []const u8) !void {
1030 self.h.update(buf);
1031 }
1032
1033 pub fn chksum(self: *Md5Writer) [32]u8 {
1034 var s = [_]u8{0} ** 16;
1035 self.h.final(&s);
1036 return std.fmt.bytesToHex(s, .lower);
1037 }
1038};
1039