authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2023-11-30 21:28:10+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-01-13 19:37:33-07:00
log6e7a39c935b13dddc9153e534e5af8fe12bc5cac
tree11ec9502b31a164b87ce614e2c2ab91721f40ed3
parentc761dfc1761b38be8d1dc72dd4c0cbf07d2c0eed

tar: refactor reading pax attributes

1 files changed, 197 insertions(+), 106 deletions(-)

lib/std/tar.zig+197-106
......@@ -198,6 +198,16 @@ fn nullStr(str: []const u8) []const u8 {
198198 return str;
199199}
200200
201// File size rounded to te block boundary.
202inline fn roundedFileSize(file_size: usize) usize {
203 return std.mem.alignForward(usize, file_size, BLOCK_SIZE);
204}
205
206// Number of padding bytes in the last file block.
207inline fn filePadding(file_size: usize) usize {
208 return roundedFileSize(file_size) - file_size;
209}
210
201211fn BufferedReader(comptime ReaderType: type) type {
202212 return struct {
203213 unbuffered_reader: ReaderType,
......@@ -207,16 +217,32 @@ fn BufferedReader(comptime ReaderType: type) type {
207217
208218 const Self = @This();
209219
210 pub fn readChunk(self: *Self, count: usize) ![]const u8 {
211 self.ensureCapacity(1024);
212
220 fn readChunk(self: *Self, count: usize) ![]const u8 {
221 self.ensureCapacity(BLOCK_SIZE * 2);
213222 const ask = @min(self.buffer.len - self.end, count -| (self.end - self.start));
214223 self.end += try self.unbuffered_reader.readAtLeast(self.buffer[self.end..], ask);
215
216224 return self.buffer[self.start..self.end];
217225 }
218226
219 pub fn readBlock(self: *Self) !?[]const u8 {
227 // Returns slice of size count or part of it.
228 pub fn readSlice(self: *Self, count: usize) ![]const u8 {
229 if (count <= self.end - self.start) {
230 // fastpath, we have enough bytes in buffer
231 return self.buffer[self.start .. self.start + count];
232 }
233
234 const chunk_size = roundedFileSize(count) + BLOCK_SIZE;
235 const temp = try self.readChunk(chunk_size);
236 if (temp.len == 0) return error.UnexpectedEndOfStream;
237 return temp[0..@min(count, temp.len)];
238 }
239
240 // Returns tar header block, 512 bytes. Before reading advances buffer
241 // for padding of the previous block, to position reader at the start of
242 // new block. After reading advances for block size, to position reader
243 // at the start of the file body.
244 pub fn readBlock(self: *Self, padding: usize) !?[]const u8 {
245 try self.skip(padding);
220246 const block_bytes = try self.readChunk(BLOCK_SIZE * 2);
221247 switch (block_bytes.len) {
222248 0 => return null,
......@@ -227,11 +253,19 @@ fn BufferedReader(comptime ReaderType: type) type {
227253 return block_bytes[0..BLOCK_SIZE];
228254 }
229255
256 // Retruns byte at current position in buffer.
257 pub fn readByte(self: *@This()) u8 {
258 return self.buffer[self.start];
259 }
260
261 // Advances reader for count bytes, assumes that we have that number of
262 // bytes in buffer.
230263 pub fn advance(self: *Self, count: usize) void {
231264 self.start += count;
232265 assert(self.start <= self.end);
233266 }
234267
268 // Advances reader without assuming that count bytes are in the buffer.
235269 pub fn skip(self: *Self, count: usize) !void {
236270 if (self.start + count > self.end) {
237271 try self.unbuffered_reader.skipBytes(self.start + count - self.end, .{});
......@@ -241,14 +275,6 @@ fn BufferedReader(comptime ReaderType: type) type {
241275 }
242276 }
243277
244 pub fn skipPadding(self: *Self, file_size: usize) !void {
245 return self.skip(filePadding(file_size));
246 }
247
248 pub fn skipFile(self: *Self, file_size: usize) !void {
249 return self.skip(roundedFileSize(file_size));
250 }
251
252278 inline fn ensureCapacity(self: *Self, count: usize) void {
253279 if (self.buffer.len - self.start < count) {
254280 const dest_end = self.end - self.start;
......@@ -258,16 +284,26 @@ fn BufferedReader(comptime ReaderType: type) type {
258284 }
259285 }
260286
261 pub fn write(self: *Self, writer: anytype, size: usize) !void {
262 var rdr = self.sliceReader(size, true);
287 // Write count bytes to the writer.
288 pub fn write(self: *Self, writer: anytype, count: usize) !void {
289 if (self.read(count)) |buf| {
290 try writer.writeAll(buf);
291 return;
292 }
293 var rdr = self.sliceReader(count);
263294 while (try rdr.next()) |slice| {
264295 try writer.writeAll(slice);
265296 }
266297 }
267298
268 // copy dst.len bytes into dst
299 // Copy dst.len bytes into dst buffer.
269300 pub fn copy(self: *Self, dst: []u8) ![]const u8 {
270 var rdr = self.sliceReader(dst.len, true);
301 if (self.read(dst.len)) |buf| {
302 // fastpath we already have enough bytes in buffer
303 @memcpy(dst, buf);
304 return dst;
305 }
306 var rdr = self.sliceReader(dst.len);
271307 var pos: usize = 0;
272308 while (try rdr.next()) |slice| : (pos += slice.len) {
273309 @memcpy(dst[pos .. pos + slice.len], slice);
......@@ -275,91 +311,151 @@ fn BufferedReader(comptime ReaderType: type) type {
275311 return dst;
276312 }
277313
314 // Retruns count bytes from buffer and advances for that number of
315 // bytes. If we don't have that much bytes buffered returns null.
316 fn read(self: *Self, count: usize) ?[]const u8 {
317 if (count <= self.end - self.start) {
318 const buf = self.buffer[self.start .. self.start + count];
319 self.advance(count);
320 return buf;
321 }
322 return null;
323 }
324
278325 const SliceReader = struct {
279326 size: usize,
280 chunk_size: usize,
281327 offset: usize,
282328 reader: *Self,
283 auto_advance: bool,
284
285 pub fn next(self: *@This()) !?[]const u8 {
286 if (self.offset >= self.size) return null;
287329
288 const temp = try self.reader.readChunk(self.chunk_size - self.offset);
289 if (temp.len == 0) return error.UnexpectedEndOfStream;
290 const slice = temp[0..@min(self.remainingSize(), temp.len)];
291 if (self.auto_advance) try self.advance(slice.len);
330 pub fn next(self: *SliceReader) !?[]const u8 {
331 const remaining_size = self.size - self.offset;
332 if (remaining_size == 0) return null;
333 const slice = try self.reader.readSlice(remaining_size);
334 self.advance(slice.len);
292335 return slice;
293336 }
294337
295 pub fn advance(self: *@This(), len: usize) !void {
338 fn advance(self: *SliceReader, len: usize) void {
296339 self.offset += len;
297 try self.reader.skip(len);
298 }
299
300 pub fn byte(self: *@This()) u8 {
301 return self.reader.buffer[self.reader.start];
302 }
303
304 pub fn copy(self: *@This(), dst: []u8) ![]const u8 {
305 _ = try self.reader.copy(dst);
306 self.offset += dst.len;
307 return dst;
308 }
309
310 pub fn remainingSize(self: *@This()) usize {
311 return self.size - self.offset;
340 self.reader.advance(len);
312341 }
313342 };
314343
315 pub fn sliceReader(self: *Self, size: usize, auto_advance: bool) Self.SliceReader {
344 pub fn sliceReader(self: *Self, size: usize) SliceReader {
316345 return .{
317346 .size = size,
318 .chunk_size = roundedFileSize(size) + BLOCK_SIZE,
347 .reader = self,
319348 .offset = 0,
349 };
350 }
351
352 pub fn paxFileReader(self: *Self, size: usize) PaxFileReader {
353 return .{
354 .size = size,
320355 .reader = self,
321 .auto_advance = auto_advance,
356 .offset = 0,
322357 };
323358 }
324 };
325}
326359
327// File size rounded to te block boundary.
328inline fn roundedFileSize(file_size: usize) usize {
329 return std.mem.alignForward(usize, file_size, BLOCK_SIZE);
330}
360 const PaxFileReader = struct {
361 size: usize,
362 offset: usize = 0,
363 reader: *Self,
331364
332// Number of padding bytes in the last file block.
333inline fn filePadding(file_size: usize) usize {
334 return roundedFileSize(file_size) - file_size;
365 const PaxKey = enum {
366 path,
367 linkpath,
368 size,
369 };
370
371 const PaxAttribute = struct {
372 key: PaxKey,
373 value_len: usize,
374 parent: *PaxFileReader,
375
376 // Copies pax attribute value into destination buffer.
377 // Must be called with destination buffer of size at least value_len.
378 pub fn value(self: PaxAttribute, dst: []u8) ![]u8 {
379 assert(dst.len >= self.value_len);
380 const buf = dst[0..self.value_len];
381 _ = try self.parent.reader.copy(buf);
382 self.parent.offset += buf.len;
383 try self.parent.checkAttributeEnding();
384 return buf;
385 }
386 };
387
388 // Caller of the next has to call value in PaxAttribute, to advance
389 // reader across value.
390 pub fn next(self: *PaxFileReader) !?PaxAttribute {
391 const rdr = self.reader;
392 _ = rdr;
393
394 while (true) {
395 const remaining_size = self.size - self.offset;
396 if (remaining_size == 0) return null;
397
398 const inf = try parsePaxAttribute(
399 try self.reader.readSlice(remaining_size),
400 remaining_size,
401 );
402 const key: PaxKey = if (inf.is("path"))
403 .path
404 else if (inf.is("linkpath"))
405 .linkpath
406 else if (inf.is("size"))
407 .size
408 else {
409 try self.advance(inf.value_off + inf.value_len);
410 try self.checkAttributeEnding();
411 continue;
412 };
413 try self.advance(inf.value_off); // position reader at the start of the value
414 return PaxAttribute{ .key = key, .value_len = inf.value_len, .parent = self };
415 }
416 }
417
418 fn checkAttributeEnding(self: *PaxFileReader) !void {
419 if (self.reader.readByte() != '\n') return error.InvalidPaxAttribute;
420 try self.advance(1);
421 }
422
423 fn advance(self: *PaxFileReader, len: usize) !void {
424 self.offset += len;
425 try self.reader.skip(len);
426 }
427 };
428 };
335429}
336430
337431fn Iterator(comptime ReaderType: type) type {
338432 const BufferedReaderType = BufferedReader(ReaderType);
339433 return struct {
340 attrs: struct {
341 buffer: [std.fs.MAX_PATH_BYTES * 2]u8 = undefined,
434 // scratch buffer for file attributes
435 scratch: struct {
436 // size: two paths (name and link_name) and size (24 in pax attribute)
437 buffer: [std.fs.MAX_PATH_BYTES * 2 + 24]u8 = undefined,
342438 tail: usize = 0,
343439
440 // Allocate size of the buffer for some attribute.
344441 fn alloc(self: *@This(), size: usize) ![]u8 {
345 if (size > self.len()) return error.NameTooLong;
442 const free_size = self.buffer.len - self.tail;
443 if (size > free_size) return error.TarScratchBufferOverflow;
346444 const head = self.tail;
347445 self.tail += size;
348446 assert(self.tail <= self.buffer.len);
349447 return self.buffer[head..self.tail];
350448 }
351449
450 // Free whole buffer.
352451 fn free(self: *@This()) void {
353452 self.tail = 0;
354453 }
355
356 fn len(self: *@This()) usize {
357 return self.buffer.len - self.tail;
358 }
359454 } = .{},
360455
361456 reader: BufferedReaderType,
362457 diagnostics: ?*Options.Diagnostics,
458 padding: usize = 0, // bytes of file padding
363459
364460 const Self = @This();
365461
......@@ -372,28 +468,22 @@ fn Iterator(comptime ReaderType: type) type {
372468
373469 pub fn write(self: File, writer: anytype) !void {
374470 try self.reader.write(writer, self.size);
375 try self.skipPadding();
376471 }
377472
378473 pub fn skip(self: File) !void {
379 try self.reader.skip(roundedFileSize(self.size));
380 }
381
382 fn skipPadding(self: File) !void {
383 try self.reader.skip(filePadding(self.size));
474 try self.reader.skip(self.size);
384475 }
385476
386477 fn chksum(self: File) ![16]u8 {
387478 var sum = [_]u8{0} ** 16;
388479 if (self.size == 0) return sum;
389480
390 var rdr = self.reader.sliceReader(self.size, true);
481 var rdr = self.reader.sliceReader(self.size);
391482 var h = std.crypto.hash.Md5.init(.{});
392483 while (try rdr.next()) |slice| {
393484 h.update(slice);
394485 }
395486 h.final(&sum);
396 try self.skipPadding();
397487 return sum;
398488 }
399489 };
......@@ -406,64 +496,65 @@ fn Iterator(comptime ReaderType: type) type {
406496 // "normal file".
407497 pub fn next(self: *Self) !?File {
408498 var file: File = .{ .reader = &self.reader };
409 self.attrs.free();
499 self.scratch.free();
410500
411 while (try self.reader.readBlock()) |block_bytes| {
412 const block = Header{ .bytes = block_bytes[0..BLOCK_SIZE] };
413 if (try block.checkChksum() == 0) return null; // zero block found
414 const file_type = block.fileType();
415 const file_size = try block.fileSize();
501 while (try self.reader.readBlock(self.padding)) |block_bytes| {
502 const header = Header{ .bytes = block_bytes[0..BLOCK_SIZE] };
503 if (try header.checkChksum() == 0) return null; // zero block found
504
505 const file_type = header.fileType();
506 const file_size = try header.fileSize();
507 self.padding = filePadding(file_size);
416508
417509 switch (file_type) {
510 // file types to retrun from next
418511 .directory, .normal, .symbolic_link => {
419512 if (file.size == 0) file.size = file_size;
513 self.padding = filePadding(file.size);
514
420515 if (file.name.len == 0)
421 file.name = try block.fullFileName((try self.attrs.alloc(std.fs.MAX_PATH_BYTES))[0..std.fs.MAX_PATH_BYTES]);
422 if (file.link_name.len == 0) file.link_name = block.linkName();
516 file.name = try header.fullFileName((try self.scratch.alloc(std.fs.MAX_PATH_BYTES))[0..std.fs.MAX_PATH_BYTES]);
517 if (file.link_name.len == 0) file.link_name = header.linkName();
423518 file.file_type = file_type;
424519 return file;
425520 },
426 .global_extended_header => {
427 self.reader.skipFile(file_size) catch return error.TarHeadersTooBig;
521 // prefix header types
522 .gnu_long_name => {
523 file.name = nullStr(try self.reader.copy(try self.scratch.alloc(file_size)));
524 },
525 .gnu_long_link => {
526 file.link_name = nullStr(try self.reader.copy(try self.scratch.alloc(file_size)));
428527 },
429528 .extended_header => {
430529 if (file_size == 0) continue;
431 // TODO: ovo resetiranje je nezgodno
432 self.attrs.free();
530 // use just last extended header data
531 self.scratch.free();
433532 file = File{ .reader = &self.reader };
434533
435 var rdr = self.reader.sliceReader(file_size, false);
436 while (try rdr.next()) |slice| {
437 const attr = try parsePaxAttribute(slice, rdr.remainingSize());
438 try rdr.advance(attr.value_off);
439 if (attr.is("path")) {
440 file.name = try noNull(try rdr.copy(try self.attrs.alloc(attr.value_len)));
441 } else if (attr.is("linkpath")) {
442 file.link_name = try noNull(try rdr.copy(try self.attrs.alloc(attr.value_len)));
443 } else if (attr.is("size")) {
444 var buf = [_]u8{'0'} ** 32;
445 file.size = try std.fmt.parseInt(usize, try rdr.copy(buf[0..attr.value_len]), 10);
446 } else {
447 try rdr.advance(attr.value_len);
534 var rdr = self.reader.paxFileReader(file_size);
535 while (try rdr.next()) |attr| {
536 switch (attr.key) {
537 .path => {
538 file.name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));
539 },
540 .linkpath => {
541 file.link_name = try noNull(try attr.value(try self.scratch.alloc(attr.value_len)));
542 },
543 .size => {
544 file.size = try std.fmt.parseInt(usize, try attr.value(try self.scratch.alloc(attr.value_len)), 10);
545 },
448546 }
449 if (rdr.byte() != '\n') return error.InvalidPaxAttribute;
450 try rdr.advance(1);
451547 }
452 try self.reader.skipPadding(file_size);
453 },
454 .gnu_long_name => {
455 file.name = nullStr(try self.reader.copy(try self.attrs.alloc(file_size)));
456 try self.reader.skipPadding(file_size);
457548 },
458 .gnu_long_link => {
459 file.link_name = nullStr(try self.reader.copy(try self.attrs.alloc(file_size)));
460 try self.reader.skipPadding(file_size);
549 // ignored header types
550 .global_extended_header => {
551 self.reader.skip(file_size) catch return error.TarHeadersTooBig;
461552 },
462 .hard_link => return error.TarUnsupportedFileType,
553 // unsupported header types
463554 else => {
464555 const d = self.diagnostics orelse return error.TarUnsupportedFileType;
465556 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
466 .file_name = try d.allocator.dupe(u8, block.name()),
557 .file_name = try d.allocator.dupe(u8, header.name()),
467558 .file_type = file_type,
468559 } });
469560 },