authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2023-11-30 21:28:10+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-01-13 19:37:33-07:00
log6e7a39c935b13dddc9153e534e5af8fe12bc5cac
tree11ec9502b31a164b87ce614e2c2ab91721f40ed3
parentc761dfc1761b38be8d1dc72dd4c0cbf07d2c0eed

tar: refactor reading pax attributes

1 files changed, 197 insertions(+), 106 deletions(-)

lib/std/tar.zig+197-106
...@@ -198,6 +198,16 @@ fn nullStr(str: []const u8) []const u8 {...@@ -198,6 +198,16 @@ fn nullStr(str: []const u8) []const u8 {
198 return str;198 return str;
199}199}
200200
201// File size rounded to te block boundary.
202inline fn roundedFileSize(file_size: usize) usize {
203 return std.mem.alignForward(usize, file_size, BLOCK_SIZE);
204}
205
206// Number of padding bytes in the last file block.
207inline fn filePadding(file_size: usize) usize {
208 return roundedFileSize(file_size) - file_size;
209}
210
201fn BufferedReader(comptime ReaderType: type) type {211fn BufferedReader(comptime ReaderType: type) type {
202 return struct {212 return struct {
203 unbuffered_reader: ReaderType,213 unbuffered_reader: ReaderType,
...@@ -207,16 +217,32 @@ fn BufferedReader(comptime ReaderType: type) type {...@@ -207,16 +217,32 @@ fn BufferedReader(comptime ReaderType: type) type {
207217
208 const Self = @This();218 const Self = @This();
209219
210 pub fn readChunk(self: *Self, count: usize) ![]const u8 {220 fn readChunk(self: *Self, count: usize) ![]const u8 {
211 self.ensureCapacity(1024);221 self.ensureCapacity(BLOCK_SIZE * 2);
212
213 const ask = @min(self.buffer.len - self.end, count -| (self.end - self.start));222 const ask = @min(self.buffer.len - self.end, count -| (self.end - self.start));
214 self.end += try self.unbuffered_reader.readAtLeast(self.buffer[self.end..], ask);223 self.end += try self.unbuffered_reader.readAtLeast(self.buffer[self.end..], ask);
215
216 return self.buffer[self.start..self.end];224 return self.buffer[self.start..self.end];
217 }225 }
218226
219 pub fn readBlock(self: *Self) !?[]const u8 {227 // Returns slice of size count or part of it.
228 pub fn readSlice(self: *Self, count: usize) ![]const u8 {
229 if (count <= self.end - self.start) {
230 // fastpath, we have enough bytes in buffer
231 return self.buffer[self.start .. self.start + count];
232 }
233
234 const chunk_size = roundedFileSize(count) + BLOCK_SIZE;
235 const temp = try self.readChunk(chunk_size);
236 if (temp.len == 0) return error.UnexpectedEndOfStream;
237 return temp[0..@min(count, temp.len)];
238 }
239
240 // Returns tar header block, 512 bytes. Before reading advances buffer
241 // for padding of the previous block, to position reader at the start of
242 // new block. After reading advances for block size, to position reader
243 // at the start of the file body.
244 pub fn readBlock(self: *Self, padding: usize) !?[]const u8 {
245 try self.skip(padding);
220 const block_bytes = try self.readChunk(BLOCK_SIZE * 2);246 const block_bytes = try self.readChunk(BLOCK_SIZE * 2);
221 switch (block_bytes.len) {247 switch (block_bytes.len) {
222 0 => return null,248 0 => return null,
...@@ -227,11 +253,19 @@ fn BufferedReader(comptime ReaderType: type) type {...@@ -227,11 +253,19 @@ fn BufferedReader(comptime ReaderType: type) type {
227 return block_bytes[0..BLOCK_SIZE];253 return block_bytes[0..BLOCK_SIZE];
228 }254 }
229255
256 // Retruns byte at current position in buffer.
257 pub fn readByte(self: *@This()) u8 {
258 return self.buffer[self.start];
259 }
260
261 // Advances reader for count bytes, assumes that we have that number of
262 // bytes in buffer.
230 pub fn advance(self: *Self, count: usize) void {263 pub fn advance(self: *Self, count: usize) void {
231 self.start += count;264 self.start += count;
232 assert(self.start <= self.end);265 assert(self.start <= self.end);
233 }266 }
234267
268 // Advances reader without assuming that count bytes are in the buffer.
235 pub fn skip(self: *Self, count: usize) !void {269 pub fn skip(self: *Self, count: usize) !void {
236 if (self.start + count > self.end) {270 if (self.start + count > self.end) {
237 try self.unbuffered_reader.skipBytes(self.start + count - self.end, .{});271 try self.unbuffered_reader.skipBytes(self.start + count - self.end, .{});
...@@ -241,14 +275,6 @@ fn BufferedReader(comptime ReaderType: type) type {...@@ -241,14 +275,6 @@ fn BufferedReader(comptime ReaderType: type) type {
241 }275 }
242 }276 }
243277
244 pub fn skipPadding(self: *Self, file_size: usize) !void {
245 return self.skip(filePadding(file_size));
246 }
247
248 pub fn skipFile(self: *Self, file_size: usize) !void {
249 return self.skip(roundedFileSize(file_size));
250 }
251
252 inline fn ensureCapacity(self: *Self, count: usize) void {278 inline fn ensureCapacity(self: *Self, count: usize) void {
253 if (self.buffer.len - self.start < count) {279 if (self.buffer.len - self.start < count) {
254 const dest_end = self.end - self.start;280 const dest_end = self.end - self.start;
...@@ -258,16 +284,26 @@ fn BufferedReader(comptime ReaderType: type) type {...@@ -258,16 +284,26 @@ fn BufferedReader(comptime ReaderType: type) type {
258 }284 }
259 }285 }
260286
261 pub fn write(self: *Self, writer: anytype, size: usize) !void {287 // Write count bytes to the writer.
262 var rdr = self.sliceReader(size, true);288 pub fn write(self: *Self, writer: anytype, count: usize) !void {
289 if (self.read(count)) |buf| {
290 try writer.writeAll(buf);
291 return;
292 }
293 var rdr = self.sliceReader(count);
263 while (try rdr.next()) |slice| {294 while (try rdr.next()) |slice| {
264 try writer.writeAll(slice);295 try writer.writeAll(slice);
265 }296 }
266 }297 }
267298
268 // copy dst.len bytes into dst299 // Copy dst.len bytes into dst buffer.
269 pub fn copy(self: *Self, dst: []u8) ![]const u8 {300 pub fn copy(self: *Self, dst: []u8) ![]const u8 {
270 var rdr = self.sliceReader(dst.len, true);301 if (self.read(dst.len)) |buf| {
302 // fastpath we already have enough bytes in buffer
303 @memcpy(dst, buf);
304 return dst;
305 }
306 var rdr = self.sliceReader(dst.len);
271 var pos: usize = 0;307 var pos: usize = 0;
272 while (try rdr.next()) |slice| : (pos += slice.len) {308 while (try rdr.next()) |slice| : (pos += slice.len) {
273 @memcpy(dst[pos .. pos + slice.len], slice);309 @memcpy(dst[pos .. pos + slice.len], slice);
...@@ -275,91 +311,151 @@ fn BufferedReader(comptime ReaderType: type) type {...@@ -275,91 +311,151 @@ fn BufferedReader(comptime ReaderType: type) type {
275 return dst;311 return dst;
276 }312 }
277313
314 // Retruns count bytes from buffer and advances for that number of
315 // bytes. If we don't have that much bytes buffered returns null.
316 fn read(self: *Self, count: usize) ?[]const u8 {
317 if (count <= self.end - self.start) {
318 const buf = self.buffer[self.start .. self.start + count];
319 self.advance(count);
320 return buf;
321 }
322 return null;
323 }
324
278 const SliceReader = struct {325 const SliceReader = struct {
279 size: usize,326 size: usize,
280 chunk_size: usize,
281 offset: usize,327 offset: usize,
282 reader: *Self,328 reader: *Self,
283 auto_advance: bool,
284
285 pub fn next(self: *@This()) !?[]const u8 {
286 if (self.offset >= self.size) return null;
287329
288 const temp = try self.reader.readChunk(self.chunk_size - self.offset);330 pub fn next(self: *SliceReader) !?[]const u8 {
289 if (temp.len == 0) return error.UnexpectedEndOfStream;331 const remaining_size = self.size - self.offset;
290 const slice = temp[0..@min(self.remainingSize(), temp.len)];332 if (remaining_size == 0) return null;
291 if (self.auto_advance) try self.advance(slice.len);333 const slice = try self.reader.readSlice(remaining_size);
334 self.advance(slice.len);
292 return slice;335 return slice;
293 }336 }
294337
295 pub fn advance(self: *@This(), len: usize) !void {338 fn advance(self: *SliceReader, len: usize) void {
296 self.offset += len;339 self.offset += len;
297 try self.reader.skip(len);340 self.reader.advance(len);
298 }
299
300 pub fn byte(self: *@This()) u8 {
301 return self.reader.buffer[self.reader.start];
302 }
303
304 pub fn copy(self: *@This(), dst: []u8) ![]const u8 {
305 _ = try self.reader.copy(dst);
306 self.offset += dst.len;
307 return dst;
308 }
309
310 pub fn remainingSize(self: *@This()) usize {
311 return self.size - self.offset;
312 }341 }
313 };342 };
314343
315 pub fn sliceReader(self: *Self, size: usize, auto_advance: bool) Self.SliceReader {344 pub fn sliceReader(self: *Self, size: usize) SliceReader {
316 return .{345 return .{
317 .size = size,346 .size = size,
318 .chunk_size = roundedFileSize(size) + BLOCK_SIZE,347 .reader = self,
319 .offset = 0,348 .offset = 0,
349 };
350 }
351
352 pub fn paxFileReader(self: *Self, size: usize) PaxFileReader {
353 return .{
354 .size = size,
320 .reader = self,355 .reader = self,
321 .auto_advance = auto_advance,356 .offset = 0,
322 };357 };
323 }358 }
324 };
325}
326359
327// File size rounded to te block boundary.360 const PaxFileReader = struct {
328inline fn roundedFileSize(file_size: usize) usize {361 size: usize,
329 return std.mem.alignForward(usize, file_size, BLOCK_SIZE);362 offset: usize = 0,
330}363 reader: *Self,
331364
332// Number of padding bytes in the last file block.365 const PaxKey = enum {
333inline fn filePadding(file_size: usize) usize {366 path,
334 return roundedFileSize(file_size) - file_size;367 linkpath,
368 size,
369 };
370
371 const PaxAttribute = struct {
372 key: PaxKey,
373 value_len: usize,
374 parent: *PaxFileReader,
375
376 // Copies pax attribute value into destination buffer.
377 // Must be called with destination buffer of size at least value_len.
378 pub fn value(self: PaxAttribute, dst: []u8) ![]u8 {
379 assert(dst.len >= self.value_len);
380 const buf = dst[0..self.value_len];
381 _ = try self.parent.reader.copy(buf);
382 self.parent.offset += buf.len;
383 try self.parent.checkAttributeEnding();
384 return buf;
385 }
386 };
387
388 // Caller of the next has to call value in PaxAttribute, to advance
389 // reader across value.
390 pub fn next(self: *PaxFileReader) !?PaxAttribute {
391 const rdr = self.reader;
392 _ = rdr;
393
394 while (true) {
395 const remaining_size = self.size - self.offset;
396 if (remaining_size == 0) return null;
397
398 const inf = try parsePaxAttribute(
399 try self.reader.readSlice(remaining_size),
400 remaining_size,
401 );
402 const key: PaxKey = if (inf.is("path"))
403 .path
404 else if (inf.is("linkpath"))
405 .linkpath
406 else if (inf.is("size"))
407 .size
408 else {
409 try self.advance(inf.value_off + inf.value_len);
410 try self.checkAttributeEnding();
411 continue;
412 };
413 try self.advance(inf.value_off); // position reader at the start of the value
414 return PaxAttribute{ .key = key, .value_len = inf.value_len, .parent = self };
415 }
416 }
417
418 fn checkAttributeEnding(self: *PaxFileReader) !void {
419 if (self.reader.readByte() != '\n') return error.InvalidPaxAttribute;
420 try self.advance(1);
421 }
422
423 fn advance(self: *PaxFileReader, len: usize) !void {
424 self.offset += len;
425 try self.reader.skip(len);
426 }
427 };
428 };
335}429}
336430
337fn Iterator(comptime ReaderType: type) type {431fn Iterator(comptime ReaderType: type) type {
338 const BufferedReaderType = BufferedReader(ReaderType);432 const BufferedReaderType = BufferedReader(ReaderType);
339 return struct {433 return struct {
340 attrs: struct {434 // scratch buffer for file attributes
341 buffer: [std.fs.MAX_PATH_BYTES * 2]u8 = undefined,435 scratch: struct {
436 // size: two paths (name and link_name) and size (24 in pax attribute)
437 buffer: [std.fs.MAX_PATH_BYTES * 2 + 24]u8 = undefined,
342 tail: usize = 0,438 tail: usize = 0,
343439
440 // Allocate size of the buffer for some attribute.
344 fn alloc(self: *@This(), size: usize) ![]u8 {441 fn alloc(self: *@This(), size: usize) ![]u8 {
345 if (size > self.len()) return error.NameTooLong;442 const free_size = self.buffer.len - self.tail;
443 if (size > free_size) return error.TarScratchBufferOverflow;
346 const head = self.tail;444 const head = self.tail;
347 self.tail += size;445 self.tail += size;
348 assert(self.tail <= self.buffer.len);446 assert(self.tail <= self.buffer.len);
349 return self.buffer[head..self.tail];447 return self.buffer[head..self.tail];
350 }448 }
351449
450 // Free whole buffer.
352 fn free(self: *@This()) void {451 fn free(self: *@This()) void {
353 self.tail = 0;452 self.tail = 0;
354 }453 }
355
356 fn len(self: *@This()) usize {
357 return self.buffer.len - self.tail;
358 }
359 } = .{},454 } = .{},
360455
361 reader: BufferedReaderType,456 reader: BufferedReaderType,
362 diagnostics: ?*Options.Diagnostics,457 diagnostics: ?*Options.Diagnostics,
458 padding: usize = 0, // bytes of file padding
363459
364 const Self = @This();460 const Self = @This();
365461
...@@ -372,28 +468,22 @@ fn Iterator(comptime ReaderType: type) type {...@@ -372,28 +468,22 @@ fn Iterator(comptime ReaderType: type) type {
372468
373 pub fn write(self: File, writer: anytype) !void {469 pub fn write(self: File, writer: anytype) !void {
374 try self.reader.write(writer, self.size);470 try self.reader.write(writer, self.size);
375 try self.skipPadding();
376 }471 }
377472
378 pub fn skip(self: File) !void {473 pub fn skip(self: File) !void {
379 try self.reader.skip(roundedFileSize(self.size));474 try self.reader.skip(self.size);
380 }
381
382 fn skipPadding(self: File) !void {
383 try self.reader.skip(filePadding(self.size));
384 }475 }
385476
386 fn chksum(self: File) ![16]u8 {477 fn chksum(self: File) ![16]u8 {
387 var sum = [_]u8{0} ** 16;478 var sum = [_]u8{0} ** 16;
388 if (self.size == 0) return sum;479 if (self.size == 0) return sum;
389480
390 var rdr = self.reader.sliceReader(self.size, true);481 var rdr = self.reader.sliceReader(self.size);
391 var h = std.crypto.hash.Md5.init(.{});482 var h = std.crypto.hash.Md5.init(.{});
392 while (try rdr.next()) |slice| {483 while (try rdr.next()) |slice| {
393 h.update(slice);484 h.update(slice);
394 }485 }
395 h.final(&sum);486 h.final(&sum);
396 try self.skipPadding();
397 return sum;487 return sum;
398 }488 }
399 };489 };
...@@ -406,64 +496,65 @@ fn Iterator(comptime ReaderType: type) type {...@@ -406,64 +496,65 @@ fn Iterator(comptime ReaderType: type) type {
406 // "normal file".496 // "normal file".
407 pub fn next(self: *Self) !?File {497 pub fn next(self: *Self) !?File {
408 var file: File = .{ .reader = &self.reader };498 var file: File = .{ .reader = &self.reader };
409 self.attrs.free();499 self.scratch.free();
410500
411 while (try self.reader.readBlock()) |block_bytes| {501 while (try self.reader.readBlock(self.padding)) |block_bytes| {
412 const block = Header{ .bytes = block_bytes[0..BLOCK_SIZE] };502 const header = Header{ .bytes = block_bytes[0..BLOCK_SIZE] };
413 if (try block.checkChksum() == 0) return null; // zero block found503 if (try header.checkChksum() == 0) return null; // zero block found
414 const file_type = block.fileType();504
415 const file_size = try block.fileSize();505 const file_type = header.fileType();
506 const file_size = try header.fileSize();
507 self.padding = filePadding(file_size);
416508
417 switch (file_type) {509 switch (file_type) {
510 // file types to retrun from next
418 .directory, .normal, .symbolic_link => {511 .directory, .normal, .symbolic_link => {
419 if (file.size == 0) file.size = file_size;512 if (file.size == 0) file.size = file_size;
513 self.padding = filePadding(file.size);
514
420 if (file.name.len == 0)515 if (file.name.len == 0)
421 file.name = try block.fullFileName((try self.attrs.alloc(std.fs.MAX_PATH_BYTES))[0..std.fs.MAX_PATH_BYTES]);516 file.name = try header.fullFileName((try self.scratch.alloc(std.fs.MAX_PATH_BYTES))[0..std.fs.MAX_PATH_BYTES]);
422 if (file.link_name.len == 0) file.link_name = block.linkName();517 if (file.link_name.len == 0) file.link_name = header.linkName();
423 file.file_type = file_type;518 file.file_type = file_type;
424 return file;519 return file;
425 },520 },
426 .global_extended_header => {521 // prefix header types
427 self.reader.skipFile(file_size) catch return error.TarHeadersTooBig;522 .gnu_long_name => {
523 file.name = nullStr(try self.reader.copy(try self.scratch.alloc(file_size)));
524 },
525 .gnu_long_link => {
526 file.link_name = nullStr(try self.reader.copy(try self.scratch.alloc(file_size)));
428 },527 },
429 .extended_header => {528 .extended_header => {
430 if (file_size == 0) continue;529 if (file_size == 0) continue;
431 // TODO: ovo resetiranje je nezgodno530 // use just last extended header data
432 self.attrs.free();531 self.scratch.free();
433 file = File{ .reader = &self.reader };532 file = File{ .reader = &self.reader };
434533
435 var rdr = self.reader.sliceReader(file_size, false);534 var rdr = self.reader.paxFileReader(file_size);
436 while (try rdr.next()) |slice| {535 while (try rdr.next()) |attr| {
437 const attr = try parsePaxAttribute(slice, rdr.remainingSize());536 switch (attr.key) {
438 try rdr.advance(attr.value_off);537 .path => {
439 if (attr.is("path")) {538 file.name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));
440 file.name = try noNull(try rdr.copy(try self.attrs.alloc(attr.value_len)));539 },
441 } else if (attr.is("linkpath")) {540 .linkpath => {
442 file.link_name = try noNull(try rdr.copy(try self.attrs.alloc(attr.value_len)));541 file.link_name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));
443 } else if (attr.is("size")) {542 },
444 var buf = [_]u8{'0'} ** 32;543 .size => {
445 file.size = try std.fmt.parseInt(usize, try rdr.copy(buf[0..attr.value_len]), 10);544 file.size = try std.fmt.parseInt(usize, try attr.value(try self.scratch.alloc(attr.value_len)), 10);
446 } else {545 },
447 try rdr.advance(attr.value_len);
448 }546 }
449 if (rdr.byte() != '\n') return error.InvalidPaxAttribute;
450 try rdr.advance(1);
451 }547 }
452 try self.reader.skipPadding(file_size);
453 },
454 .gnu_long_name => {
455 file.name = nullStr(try self.reader.copy(try self.attrs.alloc(file_size)));
456 try self.reader.skipPadding(file_size);
457 },548 },
458 .gnu_long_link => {549 // ignored header types
459 file.link_name = nullStr(try self.reader.copy(try self.attrs.alloc(file_size)));550 .global_extended_header => {
460 try self.reader.skipPadding(file_size);551 self.reader.skip(file_size) catch return error.TarHeadersTooBig;
461 },552 },
462 .hard_link => return error.TarUnsupportedFileType,553 // unsupported header types
463 else => {554 else => {
464 const d = self.diagnostics orelse return error.TarUnsupportedFileType;555 const d = self.diagnostics orelse return error.TarUnsupportedFileType;
465 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{556 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
466 .file_name = try d.allocator.dupe(u8, block.name()),557 .file_name = try d.allocator.dupe(u8, header.name()),
467 .file_type = file_type,558 .file_type = file_type,
468 } });559 } });
469 },560 },