| ... | ... | @@ -62,8 +62,10 @@ pub const Options = struct { |
| 62 | 62 | }; |
| 63 | 63 | }; |
| 64 | 64 | |
| 65 | const block_size = 512; |
| 66 | |
| 65 | 67 | pub const Header = struct { |
| 66 | | bytes: *const [512]u8, |
| 68 | bytes: *const [block_size]u8, |
| 67 | 69 | |
| 68 | 70 | pub const FileType = enum(u8) { |
| 69 | 71 | normal_alias = 0, |
| ... | ... | @@ -135,7 +137,7 @@ pub const Header = struct { |
| 135 | 137 | return header.bytes[start..i]; |
| 136 | 138 | } |
| 137 | 139 | |
| 138 | | pub fn isZeroBlock(header: Header) bool { |
| 140 | pub fn isZero(header: Header) bool { |
| 139 | 141 | for (header.bytes) |b| { |
| 140 | 142 | if (b != 0) return false; |
| 141 | 143 | } |
| ... | ... | @@ -146,7 +148,7 @@ pub const Header = struct { |
| 146 | 148 | fn BufferedReader(comptime ReaderType: type) type { |
| 147 | 149 | return struct { |
| 148 | 150 | unbuffered_reader: ReaderType, |
| 149 | | buffer: [512 * 8]u8 = undefined, |
| 151 | buffer: [block_size * 8]u8 = undefined, |
| 150 | 152 | start: usize = 0, |
| 151 | 153 | end: usize = 0, |
| 152 | 154 | |
| ... | ... | @@ -161,6 +163,17 @@ fn BufferedReader(comptime ReaderType: type) type { |
| 161 | 163 | return self.buffer[self.start..self.end]; |
| 162 | 164 | } |
| 163 | 165 | |
| 166 | pub fn readBlock(self: *Self) !?[]const u8 { |
| 167 | const block_bytes = try self.readChunk(block_size * 2); |
| 168 | switch (block_bytes.len) { |
| 169 | 0 => return null, |
| 170 | 1...(block_size - 1) => return error.UnexpectedEndOfStream, |
| 171 | else => {}, |
| 172 | } |
| 173 | self.advance(block_size); |
| 174 | return block_bytes[0..block_size]; |
| 175 | } |
| 176 | |
| 164 | 177 | pub fn advance(self: *Self, count: usize) void { |
| 165 | 178 | self.start += count; |
| 166 | 179 | assert(self.start <= self.end); |
| ... | ... | @@ -175,6 +188,14 @@ fn BufferedReader(comptime ReaderType: type) type { |
| 175 | 188 | } |
| 176 | 189 | } |
| 177 | 190 | |
| 191 | pub fn skipPadding(self: *Self, file_size: usize) !void { |
| 192 | return self.skip(filePadding(file_size)); |
| 193 | } |
| 194 | |
| 195 | pub fn skipFile(self: *Self, file_size: usize) !void { |
| 196 | return self.skip(roundedFileSize(file_size)); |
| 197 | } |
| 198 | |
| 178 | 199 | inline fn ensureCapacity(self: *Self, count: usize) void { |
| 179 | 200 | if (self.buffer.len - self.start < count) { |
| 180 | 201 | const dest_end = self.end - self.start; |
| ... | ... | @@ -185,179 +206,200 @@ fn BufferedReader(comptime ReaderType: type) type { |
| 185 | 206 | } |
| 186 | 207 | |
| 187 | 208 | pub fn write(self: *Self, writer: anytype, size: usize) !void { |
| 188 | | const rounded_file_size = std.mem.alignForward(usize, size, 512); |
| 189 | | const chunk_size = rounded_file_size + 512; |
| 190 | | const pad_len: usize = rounded_file_size - size; |
| 191 | | |
| 192 | | var file_off: usize = 0; |
| 193 | | while (true) { |
| 194 | | const temp = try self.readChunk(chunk_size - file_off); |
| 195 | | if (temp.len == 0) return error.UnexpectedEndOfStream; |
| 196 | | const slice = temp[0..@min(size - file_off, temp.len)]; |
| 209 | var rdr = self.sliceReader(size, true); |
| 210 | while (try rdr.next()) |slice| { |
| 197 | 211 | try writer.writeAll(slice); |
| 212 | } |
| 213 | } |
| 198 | 214 | |
| 199 | | file_off += slice.len; |
| 200 | | self.advance(slice.len); |
| 201 | | if (file_off >= size) { |
| 202 | | self.advance(pad_len); |
| 203 | | return; |
| 204 | | } |
| 215 | // copy dst.len bytes into dst |
| 216 | pub fn copy(self: *Self, dst: []u8) ![]const u8 { |
| 217 | var rdr = self.sliceReader(dst.len, true); |
| 218 | var pos: usize = 0; |
| 219 | while (try rdr.next()) |slice| : (pos += slice.len) { |
| 220 | @memcpy(dst[pos .. pos + slice.len], slice); |
| 205 | 221 | } |
| 222 | return dst; |
| 206 | 223 | } |
| 207 | 224 | |
| 208 | | pub fn copy(self: *Self, dst_buffer: []u8, size: usize) !void { |
| 209 | | const rounded_file_size = std.mem.alignForward(usize, size, 512); |
| 210 | | const chunk_size = rounded_file_size + 512; |
| 211 | | |
| 212 | | var i: usize = 0; |
| 213 | | while (i < size) { |
| 214 | | const slice = try self.readChunk(chunk_size - i); |
| 215 | | if (slice.len == 0) return error.UnexpectedEndOfStream; |
| 216 | | const copy_size: usize = @min(size - i, slice.len); |
| 217 | | @memcpy(dst_buffer[i .. i + copy_size], slice[0..copy_size]); |
| 218 | | self.advance(copy_size); |
| 219 | | i += copy_size; |
| 225 | const SliceReader = struct { |
| 226 | size: usize, |
| 227 | chunk_size: usize, |
| 228 | offset: usize, |
| 229 | reader: *Self, |
| 230 | auto_advance: bool, |
| 231 | |
| 232 | fn next(self: *@This()) !?[]const u8 { |
| 233 | if (self.offset >= self.size) return null; |
| 234 | |
| 235 | const temp = try self.reader.readChunk(self.chunk_size - self.offset); |
| 236 | if (temp.len == 0) return error.UnexpectedEndOfStream; |
| 237 | const slice = temp[0..@min(self.remainingSize(), temp.len)]; |
| 238 | if (self.auto_advance) try self.advance(slice.len); |
| 239 | return slice; |
| 240 | } |
| 241 | |
| 242 | fn advance(self: *@This(), len: usize) !void { |
| 243 | self.offset += len; |
| 244 | try self.reader.skip(len); |
| 220 | 245 | } |
| 246 | |
| 247 | fn copy(self: *@This(), dst: []u8) ![]const u8 { |
| 248 | _ = try self.reader.copy(dst); |
| 249 | self.offset += dst.len; |
| 250 | return dst; |
| 251 | } |
| 252 | |
| 253 | fn remainingSize(self: *@This()) usize { |
| 254 | return self.size - self.offset; |
| 255 | } |
| 256 | }; |
| 257 | |
| 258 | pub fn sliceReader(self: *Self, size: usize, auto_advance: bool) Self.SliceReader { |
| 259 | return .{ |
| 260 | .size = size, |
| 261 | .chunk_size = roundedFileSize(size) + block_size, |
| 262 | .offset = 0, |
| 263 | .reader = self, |
| 264 | .auto_advance = auto_advance, |
| 265 | }; |
| 221 | 266 | } |
| 222 | 267 | }; |
| 223 | 268 | } |
| 224 | 269 | |
| 270 | // file_size rouneded to te block boundary |
| 271 | inline fn roundedFileSize(file_size: usize) usize { |
| 272 | return std.mem.alignForward(usize, file_size, block_size); |
| 273 | } |
| 274 | |
| 275 | // number of padding bytes at the last file block |
| 276 | inline fn filePadding(file_size: usize) usize { |
| 277 | return roundedFileSize(file_size) - file_size; |
| 278 | } |
| 279 | |
| 225 | 280 | fn Iterator(comptime ReaderType: type) type { |
| 281 | const BufferedReaderType = BufferedReader(ReaderType); |
| 226 | 282 | return struct { |
| 227 | | file_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined, |
| 228 | | file_name_len: usize = 0, |
| 229 | | link_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined, |
| 230 | | link_name_len: usize = 0, |
| 283 | attrs: struct { |
| 284 | buffer: [std.fs.MAX_PATH_BYTES * 2]u8 = undefined, |
| 285 | tail: usize = 0, |
| 286 | |
| 287 | fn alloc(self: *@This(), size: usize) ![]u8 { |
| 288 | if (size > self.len()) return error.NameTooLong; |
| 289 | const head = self.tail; |
| 290 | self.tail += size; |
| 291 | assert(self.tail <= self.buffer.len); |
| 292 | return self.buffer[head..self.tail]; |
| 293 | } |
| 294 | |
| 295 | fn free(self: *@This()) void { |
| 296 | self.tail = 0; |
| 297 | } |
| 298 | |
| 299 | fn len(self: *@This()) usize { |
| 300 | return self.buffer.len - self.tail; |
| 301 | } |
| 302 | } = .{}, |
| 231 | 303 | |
| 232 | | reader: BufferedReader(ReaderType), |
| 304 | reader: BufferedReaderType, |
| 233 | 305 | diagnostics: ?*Options.Diagnostics, |
| 234 | 306 | |
| 235 | 307 | const Self = @This(); |
| 236 | 308 | |
| 237 | 309 | const File = struct { |
| 238 | | name: []const u8, |
| 239 | | link_name: []const u8, |
| 240 | | size: usize, |
| 241 | | file_type: Header.FileType, |
| 242 | | iter: *Self, |
| 310 | name: []const u8 = &[_]u8{}, |
| 311 | link_name: []const u8 = &[_]u8{}, |
| 312 | size: usize = 0, |
| 313 | file_type: Header.FileType = .normal, |
| 314 | reader: *BufferedReaderType, |
| 243 | 315 | |
| 244 | 316 | pub fn write(self: File, writer: anytype) !void { |
| 245 | | try self.iter.reader.write(writer, self.size); |
| 317 | try self.reader.write(writer, self.size); |
| 318 | try self.skipPadding(); |
| 246 | 319 | } |
| 247 | 320 | |
| 248 | 321 | pub fn skip(self: File) !void { |
| 249 | | const rounded_file_size = std.mem.alignForward(usize, self.size, 512); |
| 250 | | try self.iter.reader.skip(rounded_file_size); |
| 322 | try self.reader.skip(roundedFileSize(self.size)); |
| 323 | } |
| 324 | |
| 325 | fn skipPadding(self: File) !void { |
| 326 | try self.reader.skip(filePadding(self.size)); |
| 251 | 327 | } |
| 252 | 328 | |
| 253 | 329 | fn chksum(self: File) ![16]u8 { |
| 254 | | var cs = [_]u8{0} ** 16; |
| 255 | | if (self.size == 0) return cs; |
| 330 | var sum = [_]u8{0} ** 16; |
| 331 | if (self.size == 0) return sum; |
| 256 | 332 | |
| 257 | | var buffer: [512]u8 = undefined; |
| 333 | var rdr = self.reader.sliceReader(self.size, true); |
| 258 | 334 | var h = std.crypto.hash.Md5.init(.{}); |
| 259 | | |
| 260 | | var remaining_bytes: usize = self.size; |
| 261 | | while (remaining_bytes > 0) { |
| 262 | | const copy_size = @min(buffer.len, remaining_bytes); |
| 263 | | try self.iter.reader.copy(&buffer, copy_size); |
| 264 | | h.update(buffer[0..copy_size]); |
| 265 | | remaining_bytes -= copy_size; |
| 335 | while (try rdr.next()) |slice| { |
| 336 | h.update(slice); |
| 266 | 337 | } |
| 267 | | h.final(&cs); |
| 338 | h.final(&sum); |
| 268 | 339 | try self.skipPadding(); |
| 269 | | return cs; |
| 270 | | } |
| 271 | | |
| 272 | | fn skipPadding(self: File) !void { |
| 273 | | const rounded_file_size = std.mem.alignForward(usize, self.size, 512); |
| 274 | | const pad_len: usize = rounded_file_size - self.size; |
| 275 | | self.iter.reader.advance(pad_len); |
| 340 | return sum; |
| 276 | 341 | } |
| 277 | 342 | }; |
| 278 | 343 | |
| 344 | // Externally, Next iterates through the tar archive as if it is a series of |
| 345 | // files. Internally, the tar format often uses fake "files" to add meta |
| 346 | // data that describes the next file. These meta data "files" should not |
| 347 | // normally be visible to the outside. As such, this loop iterates through |
| 348 | // one or more "header files" until it finds a "normal file". |
| 279 | 349 | pub fn next(self: *Self) !?File { |
| 280 | | self.file_name_len = 0; |
| 281 | | self.link_name_len = 0; |
| 282 | | |
| 283 | | while (true) { |
| 284 | | const chunk = try self.reader.readChunk(1024); |
| 285 | | switch (chunk.len) { |
| 286 | | 0 => return null, |
| 287 | | 1...511 => return error.UnexpectedEndOfStream, |
| 288 | | else => {}, |
| 289 | | } |
| 290 | | self.reader.advance(512); |
| 291 | | |
| 292 | | const header: Header = .{ .bytes = chunk[0..512] }; |
| 293 | | if (header.isZeroBlock()) return null; |
| 294 | | const file_size = try header.fileSize(); |
| 295 | | const rounded_file_size: usize = std.mem.alignForward(usize, file_size, 512); |
| 296 | | const file_type = header.fileType(); |
| 297 | | const link_name = if (self.link_name_len == 0) |
| 298 | | header.linkName() |
| 299 | | else |
| 300 | | self.link_name_buffer[0..self.link_name_len]; |
| 301 | | const file_name = if (self.file_name_len == 0) |
| 302 | | try header.fullFileName(&self.file_name_buffer) |
| 303 | | else |
| 304 | | self.file_name_buffer[0..self.file_name_len]; |
| 350 | var file: File = .{ .reader = &self.reader }; |
| 351 | self.attrs.free(); |
| 352 | |
| 353 | while (try self.reader.readBlock()) |block_bytes| { |
| 354 | const block: Header = .{ .bytes = block_bytes[0..block_size] }; |
| 355 | if (block.isZero()) return null; |
| 356 | const file_type = block.fileType(); |
| 357 | const file_size = try block.fileSize(); |
| 305 | 358 | |
| 306 | 359 | switch (file_type) { |
| 307 | 360 | .directory, .normal, .symbolic_link => { |
| 308 | | return File{ |
| 309 | | .name = file_name, |
| 310 | | .size = file_size, |
| 311 | | .file_type = file_type, |
| 312 | | .link_name = link_name, |
| 313 | | .iter = self, |
| 314 | | }; |
| 361 | if (file.size == 0) file.size = file_size; |
| 362 | if (file.name.len == 0) |
| 363 | file.name = try block.fullFileName((try self.attrs.alloc(std.fs.MAX_PATH_BYTES))[0..std.fs.MAX_PATH_BYTES]); |
| 364 | if (file.link_name.len == 0) file.link_name = block.linkName(); |
| 365 | file.file_type = file_type; |
| 366 | return file; |
| 315 | 367 | }, |
| 316 | 368 | .global_extended_header => { |
| 317 | | self.reader.skip(rounded_file_size) catch return error.TarHeadersTooBig; |
| 369 | self.reader.skipFile(file_size) catch return error.TarHeadersTooBig; |
| 318 | 370 | }, |
| 319 | 371 | .extended_header => { |
| 320 | 372 | if (file_size == 0) continue; |
| 321 | 373 | |
| 322 | | const chunk_size: usize = rounded_file_size + 512; |
| 323 | | var data_off: usize = 0; |
| 324 | | while (data_off < file_size) { |
| 325 | | const slice = try self.reader.readChunk(chunk_size - data_off); |
| 326 | | if (slice.len == 0) return error.UnexpectedEndOfStream; |
| 327 | | const remaining_size: usize = file_size - data_off; |
| 328 | | const attr_info = try parsePaxAttribute(slice[0..@min(remaining_size, slice.len)], remaining_size); |
| 329 | | |
| 330 | | if (std.mem.eql(u8, attr_info.key, "path")) { |
| 331 | | if (attr_info.value_len > self.file_name_buffer.len) return error.NameTooLong; |
| 332 | | self.reader.advance(attr_info.value_off); |
| 333 | | try self.reader.copy(&self.file_name_buffer, attr_info.value_len); |
| 334 | | self.file_name_len = attr_info.value_len; |
| 335 | | self.reader.advance(1); |
| 336 | | } else if (std.mem.eql(u8, attr_info.key, "linkpath")) { |
| 337 | | if (attr_info.value_len > self.link_name_buffer.len) return error.NameTooLong; |
| 338 | | self.reader.advance(attr_info.value_off); |
| 339 | | try self.reader.copy(&self.link_name_buffer, attr_info.value_len); |
| 340 | | self.link_name_len = attr_info.value_len; |
| 341 | | self.reader.advance(1); |
| 374 | var rdr = self.reader.sliceReader(file_size, false); |
| 375 | while (try rdr.next()) |slice| { |
| 376 | const attr = try parsePaxAttribute(slice, rdr.remainingSize()); |
| 377 | try rdr.advance(attr.value_off); |
| 378 | if (attr.is("path")) { |
| 379 | file.name = try rdr.copy(try self.attrs.alloc(attr.value_len)); |
| 380 | } else if (attr.is("linkpath")) { |
| 381 | file.link_name = try rdr.copy(try self.attrs.alloc(attr.value_len)); |
| 382 | } else if (attr.is("size")) { |
| 383 | var buf = [_]u8{'0'} ** 32; |
| 384 | file.size = try std.fmt.parseInt(usize, try rdr.copy(buf[0..attr.value_len]), 10); |
| 342 | 385 | } else { |
| 343 | | try self.reader.skip(attr_info.size); |
| 386 | try rdr.advance(attr.value_len); |
| 344 | 387 | } |
| 345 | | data_off += attr_info.size; |
| 388 | try rdr.advance(1); |
| 346 | 389 | } |
| 347 | | try self.reader.skip(rounded_file_size - data_off); |
| 348 | | |
| 349 | | continue; |
| 390 | try self.reader.skipPadding(file_size); |
| 350 | 391 | }, |
| 351 | 392 | .hard_link => return error.TarUnsupportedFileType, |
| 352 | 393 | else => { |
| 353 | 394 | const d = self.diagnostics orelse return error.TarUnsupportedFileType; |
| 354 | 395 | try d.errors.append(d.allocator, .{ .unsupported_file_type = .{ |
| 355 | | .file_name = try d.allocator.dupe(u8, file_name), |
| 396 | .file_name = try d.allocator.dupe(u8, block.name()), |
| 356 | 397 | .file_type = file_type, |
| 357 | 398 | } }); |
| 358 | 399 | }, |
| 359 | 400 | } |
| 360 | 401 | } |
| 402 | return null; |
| 361 | 403 | } |
| 362 | 404 | }; |
| 363 | 405 | } |
| ... | ... | @@ -481,6 +523,10 @@ const PaxAttributeInfo = struct { |
| 481 | 523 | key: []const u8, |
| 482 | 524 | value_off: usize, |
| 483 | 525 | value_len: usize, |
| 526 | |
| 527 | inline fn is(self: @This(), key: []const u8) bool { |
| 528 | return (std.mem.eql(u8, self.key, key)); |
| 529 | } |
| 484 | 530 | }; |
| 485 | 531 | |
| 486 | 532 | fn parsePaxAttribute(data: []const u8, max_size: usize) !PaxAttributeInfo { |
| ... | ... | @@ -515,7 +561,7 @@ test parsePaxAttribute { |
| 515 | 561 | try expectError(error.InvalidPaxAttribute, parsePaxAttribute("", 0)); |
| 516 | 562 | } |
| 517 | 563 | |
| 518 | | const std = @import("std.zig"); |
| 564 | const std = @import("std"); |
| 519 | 565 | const assert = std.debug.assert; |
| 520 | 566 | |
| 521 | 567 | const TestCase = struct { |
| ... | ... | @@ -628,19 +674,19 @@ test "Go test cases" { |
| 628 | 674 | // }, |
| 629 | 675 | // |
| 630 | 676 | // TODO: giving wrong result because we are not reading pax size header |
| 631 | | // .{ |
| 632 | | // .path = "pax-pos-size-file.tar", |
| 633 | | // .files = &[_]TestCase.File{ |
| 634 | | // .{ |
| 635 | | // .name = "foo", |
| 636 | | // .size = 999, |
| 637 | | // .file_type = .normal, |
| 638 | | // }, |
| 639 | | // }, |
| 640 | | // .chksums = &[_][]const u8{ |
| 641 | | // "0afb597b283fe61b5d4879669a350556", |
| 642 | | // }, |
| 643 | | // }, |
| 677 | .{ |
| 678 | .path = "pax-pos-size-file.tar", |
| 679 | .files = &[_]TestCase.File{ |
| 680 | .{ |
| 681 | .name = "foo", |
| 682 | .size = 999, |
| 683 | .file_type = .normal, |
| 684 | }, |
| 685 | }, |
| 686 | .chksums = &[_][]const u8{ |
| 687 | "0afb597b283fe61b5d4879669a350556", |
| 688 | }, |
| 689 | }, |
| 644 | 690 | .{ |
| 645 | 691 | // has pax records which we are not interested in |
| 646 | 692 | .path = "pax-records.tar", |
Igor Anić <igor.anic@gmail.com> 2023-11-28 23:07:37+01:00
Andrew Kelley <andrew@ziglang.org> 2024-01-13 19:37:33-07:00
lib