| ... | @@ -62,8 +62,10 @@ pub const Options = struct { | ... | @@ -62,8 +62,10 @@ pub const Options = struct { |
| 62 | }; | 62 | }; |
| 63 | }; | 63 | }; |
| 64 | | 64 | |
| | 65 | const block_size = 512; |
| | 66 | |
| 65 | pub const Header = struct { | 67 | pub const Header = struct { |
| 66 | bytes: *const [512]u8, | 68 | bytes: *const [block_size]u8, |
| 67 | | 69 | |
| 68 | pub const FileType = enum(u8) { | 70 | pub const FileType = enum(u8) { |
| 69 | normal_alias = 0, | 71 | normal_alias = 0, |
| ... | @@ -135,7 +137,7 @@ pub const Header = struct { | ... | @@ -135,7 +137,7 @@ pub const Header = struct { |
| 135 | return header.bytes[start..i]; | 137 | return header.bytes[start..i]; |
| 136 | } | 138 | } |
| 137 | | 139 | |
| 138 | pub fn isZeroBlock(header: Header) bool { | 140 | pub fn isZero(header: Header) bool { |
| 139 | for (header.bytes) |b| { | 141 | for (header.bytes) |b| { |
| 140 | if (b != 0) return false; | 142 | if (b != 0) return false; |
| 141 | } | 143 | } |
| ... | @@ -146,7 +148,7 @@ pub const Header = struct { | ... | @@ -146,7 +148,7 @@ pub const Header = struct { |
| 146 | fn BufferedReader(comptime ReaderType: type) type { | 148 | fn BufferedReader(comptime ReaderType: type) type { |
| 147 | return struct { | 149 | return struct { |
| 148 | unbuffered_reader: ReaderType, | 150 | unbuffered_reader: ReaderType, |
| 149 | buffer: [512 * 8]u8 = undefined, | 151 | buffer: [block_size * 8]u8 = undefined, |
| 150 | start: usize = 0, | 152 | start: usize = 0, |
| 151 | end: usize = 0, | 153 | end: usize = 0, |
| 152 | | 154 | |
| ... | @@ -161,6 +163,17 @@ fn BufferedReader(comptime ReaderType: type) type { | ... | @@ -161,6 +163,17 @@ fn BufferedReader(comptime ReaderType: type) type { |
| 161 | return self.buffer[self.start..self.end]; | 163 | return self.buffer[self.start..self.end]; |
| 162 | } | 164 | } |
| 163 | | 165 | |
| | 166 | pub fn readBlock(self: *Self) !?[]const u8 { |
| | 167 | const block_bytes = try self.readChunk(block_size * 2); |
| | 168 | switch (block_bytes.len) { |
| | 169 | 0 => return null, |
| | 170 | 1...(block_size - 1) => return error.UnexpectedEndOfStream, |
| | 171 | else => {}, |
| | 172 | } |
| | 173 | self.advance(block_size); |
| | 174 | return block_bytes[0..block_size]; |
| | 175 | } |
| | 176 | |
| 164 | pub fn advance(self: *Self, count: usize) void { | 177 | pub fn advance(self: *Self, count: usize) void { |
| 165 | self.start += count; | 178 | self.start += count; |
| 166 | assert(self.start <= self.end); | 179 | assert(self.start <= self.end); |
| ... | @@ -175,6 +188,14 @@ fn BufferedReader(comptime ReaderType: type) type { | ... | @@ -175,6 +188,14 @@ fn BufferedReader(comptime ReaderType: type) type { |
| 175 | } | 188 | } |
| 176 | } | 189 | } |
| 177 | | 190 | |
| | 191 | pub fn skipPadding(self: *Self, file_size: usize) !void { |
| | 192 | return self.skip(filePadding(file_size)); |
| | 193 | } |
| | 194 | |
| | 195 | pub fn skipFile(self: *Self, file_size: usize) !void { |
| | 196 | return self.skip(roundedFileSize(file_size)); |
| | 197 | } |
| | 198 | |
| 178 | inline fn ensureCapacity(self: *Self, count: usize) void { | 199 | inline fn ensureCapacity(self: *Self, count: usize) void { |
| 179 | if (self.buffer.len - self.start < count) { | 200 | if (self.buffer.len - self.start < count) { |
| 180 | const dest_end = self.end - self.start; | 201 | const dest_end = self.end - self.start; |
| ... | @@ -185,179 +206,200 @@ fn BufferedReader(comptime ReaderType: type) type { | ... | @@ -185,179 +206,200 @@ fn BufferedReader(comptime ReaderType: type) type { |
| 185 | } | 206 | } |
| 186 | | 207 | |
| 187 | pub fn write(self: *Self, writer: anytype, size: usize) !void { | 208 | pub fn write(self: *Self, writer: anytype, size: usize) !void { |
| 188 | const rounded_file_size = std.mem.alignForward(usize, size, 512); | 209 | var rdr = self.sliceReader(size, true); |
| 189 | const chunk_size = rounded_file_size + 512; | 210 | while (try rdr.next()) |slice| { |
| 190 | const pad_len: usize = rounded_file_size - size; | | |
| 191 | | | |
| 192 | var file_off: usize = 0; | | |
| 193 | while (true) { | | |
| 194 | const temp = try self.readChunk(chunk_size - file_off); | | |
| 195 | if (temp.len == 0) return error.UnexpectedEndOfStream; | | |
| 196 | const slice = temp[0..@min(size - file_off, temp.len)]; | | |
| 197 | try writer.writeAll(slice); | 211 | try writer.writeAll(slice); |
| | 212 | } |
| | 213 | } |
| 198 | | 214 | |
| 199 | file_off += slice.len; | 215 | // copy dst.len bytes into dst |
| 200 | self.advance(slice.len); | 216 | pub fn copy(self: *Self, dst: []u8) ![]const u8 { |
| 201 | if (file_off >= size) { | 217 | var rdr = self.sliceReader(dst.len, true); |
| 202 | self.advance(pad_len); | 218 | var pos: usize = 0; |
| 203 | return; | 219 | while (try rdr.next()) |slice| : (pos += slice.len) { |
| 204 | } | 220 | @memcpy(dst[pos .. pos + slice.len], slice); |
| 205 | } | 221 | } |
| | 222 | return dst; |
| 206 | } | 223 | } |
| 207 | | 224 | |
| 208 | pub fn copy(self: *Self, dst_buffer: []u8, size: usize) !void { | 225 | const SliceReader = struct { |
| 209 | const rounded_file_size = std.mem.alignForward(usize, size, 512); | 226 | size: usize, |
| 210 | const chunk_size = rounded_file_size + 512; | 227 | chunk_size: usize, |
| 211 | | 228 | offset: usize, |
| 212 | var i: usize = 0; | 229 | reader: *Self, |
| 213 | while (i < size) { | 230 | auto_advance: bool, |
| 214 | const slice = try self.readChunk(chunk_size - i); | 231 | |
| 215 | if (slice.len == 0) return error.UnexpectedEndOfStream; | 232 | fn next(self: *@This()) !?[]const u8 { |
| 216 | const copy_size: usize = @min(size - i, slice.len); | 233 | if (self.offset >= self.size) return null; |
| 217 | @memcpy(dst_buffer[i .. i + copy_size], slice[0..copy_size]); | 234 | |
| 218 | self.advance(copy_size); | 235 | const temp = try self.reader.readChunk(self.chunk_size - self.offset); |
| 219 | i += copy_size; | 236 | if (temp.len == 0) return error.UnexpectedEndOfStream; |
| | 237 | const slice = temp[0..@min(self.remainingSize(), temp.len)]; |
| | 238 | if (self.auto_advance) try self.advance(slice.len); |
| | 239 | return slice; |
| | 240 | } |
| | 241 | |
| | 242 | fn advance(self: *@This(), len: usize) !void { |
| | 243 | self.offset += len; |
| | 244 | try self.reader.skip(len); |
| 220 | } | 245 | } |
| | 246 | |
| | 247 | fn copy(self: *@This(), dst: []u8) ![]const u8 { |
| | 248 | _ = try self.reader.copy(dst); |
| | 249 | self.offset += dst.len; |
| | 250 | return dst; |
| | 251 | } |
| | 252 | |
| | 253 | fn remainingSize(self: *@This()) usize { |
| | 254 | return self.size - self.offset; |
| | 255 | } |
| | 256 | }; |
| | 257 | |
| | 258 | pub fn sliceReader(self: *Self, size: usize, auto_advance: bool) Self.SliceReader { |
| | 259 | return .{ |
| | 260 | .size = size, |
| | 261 | .chunk_size = roundedFileSize(size) + block_size, |
| | 262 | .offset = 0, |
| | 263 | .reader = self, |
| | 264 | .auto_advance = auto_advance, |
| | 265 | }; |
| 221 | } | 266 | } |
| 222 | }; | 267 | }; |
| 223 | } | 268 | } |
| 224 | | 269 | |
| | 270 | // file_size rouneded to te block boundary |
| | 271 | inline fn roundedFileSize(file_size: usize) usize { |
| | 272 | return std.mem.alignForward(usize, file_size, block_size); |
| | 273 | } |
| | 274 | |
| | 275 | // number of padding bytes at the last file block |
| | 276 | inline fn filePadding(file_size: usize) usize { |
| | 277 | return roundedFileSize(file_size) - file_size; |
| | 278 | } |
| | 279 | |
| 225 | fn Iterator(comptime ReaderType: type) type { | 280 | fn Iterator(comptime ReaderType: type) type { |
| | 281 | const BufferedReaderType = BufferedReader(ReaderType); |
| 226 | return struct { | 282 | return struct { |
| 227 | file_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined, | 283 | attrs: struct { |
| 228 | file_name_len: usize = 0, | 284 | buffer: [std.fs.MAX_PATH_BYTES * 2]u8 = undefined, |
| 229 | link_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined, | 285 | tail: usize = 0, |
| 230 | link_name_len: usize = 0, | 286 | |
| | 287 | fn alloc(self: *@This(), size: usize) ![]u8 { |
| | 288 | if (size > self.len()) return error.NameTooLong; |
| | 289 | const head = self.tail; |
| | 290 | self.tail += size; |
| | 291 | assert(self.tail <= self.buffer.len); |
| | 292 | return self.buffer[head..self.tail]; |
| | 293 | } |
| | 294 | |
| | 295 | fn free(self: *@This()) void { |
| | 296 | self.tail = 0; |
| | 297 | } |
| | 298 | |
| | 299 | fn len(self: *@This()) usize { |
| | 300 | return self.buffer.len - self.tail; |
| | 301 | } |
| | 302 | } = .{}, |
| 231 | | 303 | |
| 232 | reader: BufferedReader(ReaderType), | 304 | reader: BufferedReaderType, |
| 233 | diagnostics: ?*Options.Diagnostics, | 305 | diagnostics: ?*Options.Diagnostics, |
| 234 | | 306 | |
| 235 | const Self = @This(); | 307 | const Self = @This(); |
| 236 | | 308 | |
| 237 | const File = struct { | 309 | const File = struct { |
| 238 | name: []const u8, | 310 | name: []const u8 = &[_]u8{}, |
| 239 | link_name: []const u8, | 311 | link_name: []const u8 = &[_]u8{}, |
| 240 | size: usize, | 312 | size: usize = 0, |
| 241 | file_type: Header.FileType, | 313 | file_type: Header.FileType = .normal, |
| 242 | iter: *Self, | 314 | reader: *BufferedReaderType, |
| 243 | | 315 | |
| 244 | pub fn write(self: File, writer: anytype) !void { | 316 | pub fn write(self: File, writer: anytype) !void { |
| 245 | try self.iter.reader.write(writer, self.size); | 317 | try self.reader.write(writer, self.size); |
| | 318 | try self.skipPadding(); |
| 246 | } | 319 | } |
| 247 | | 320 | |
| 248 | pub fn skip(self: File) !void { | 321 | pub fn skip(self: File) !void { |
| 249 | const rounded_file_size = std.mem.alignForward(usize, self.size, 512); | 322 | try self.reader.skip(roundedFileSize(self.size)); |
| 250 | try self.iter.reader.skip(rounded_file_size); | 323 | } |
| | 324 | |
| | 325 | fn skipPadding(self: File) !void { |
| | 326 | try self.reader.skip(filePadding(self.size)); |
| 251 | } | 327 | } |
| 252 | | 328 | |
| 253 | fn chksum(self: File) ![16]u8 { | 329 | fn chksum(self: File) ![16]u8 { |
| 254 | var cs = [_]u8{0} ** 16; | 330 | var sum = [_]u8{0} ** 16; |
| 255 | if (self.size == 0) return cs; | 331 | if (self.size == 0) return sum; |
| 256 | | 332 | |
| 257 | var buffer: [512]u8 = undefined; | 333 | var rdr = self.reader.sliceReader(self.size, true); |
| 258 | var h = std.crypto.hash.Md5.init(.{}); | 334 | var h = std.crypto.hash.Md5.init(.{}); |
| 259 | | 335 | while (try rdr.next()) |slice| { |
| 260 | var remaining_bytes: usize = self.size; | 336 | h.update(slice); |
| 261 | while (remaining_bytes > 0) { | | |
| 262 | const copy_size = @min(buffer.len, remaining_bytes); | | |
| 263 | try self.iter.reader.copy(&buffer, copy_size); | | |
| 264 | h.update(buffer[0..copy_size]); | | |
| 265 | remaining_bytes -= copy_size; | | |
| 266 | } | 337 | } |
| 267 | h.final(&cs); | 338 | h.final(&sum); |
| 268 | try self.skipPadding(); | 339 | try self.skipPadding(); |
| 269 | return cs; | 340 | return sum; |
| 270 | } | | |
| 271 | | | |
| 272 | fn skipPadding(self: File) !void { | | |
| 273 | const rounded_file_size = std.mem.alignForward(usize, self.size, 512); | | |
| 274 | const pad_len: usize = rounded_file_size - self.size; | | |
| 275 | self.iter.reader.advance(pad_len); | | |
| 276 | } | 341 | } |
| 277 | }; | 342 | }; |
| 278 | | 343 | |
| | 344 | // Externally, Next iterates through the tar archive as if it is a series of |
| | 345 | // files. Internally, the tar format often uses fake "files" to add meta |
| | 346 | // data that describes the next file. These meta data "files" should not |
| | 347 | // normally be visible to the outside. As such, this loop iterates through |
| | 348 | // one or more "header files" until it finds a "normal file". |
| 279 | pub fn next(self: *Self) !?File { | 349 | pub fn next(self: *Self) !?File { |
| 280 | self.file_name_len = 0; | 350 | var file: File = .{ .reader = &self.reader }; |
| 281 | self.link_name_len = 0; | 351 | self.attrs.free(); |
| 282 | | 352 | |
| 283 | while (true) { | 353 | while (try self.reader.readBlock()) |block_bytes| { |
| 284 | const chunk = try self.reader.readChunk(1024); | 354 | const block: Header = .{ .bytes = block_bytes[0..block_size] }; |
| 285 | switch (chunk.len) { | 355 | if (block.isZero()) return null; |
| 286 | 0 => return null, | 356 | const file_type = block.fileType(); |
| 287 | 1...511 => return error.UnexpectedEndOfStream, | 357 | const file_size = try block.fileSize(); |
| 288 | else => {}, | | |
| 289 | } | | |
| 290 | self.reader.advance(512); | | |
| 291 | | | |
| 292 | const header: Header = .{ .bytes = chunk[0..512] }; | | |
| 293 | if (header.isZeroBlock()) return null; | | |
| 294 | const file_size = try header.fileSize(); | | |
| 295 | const rounded_file_size: usize = std.mem.alignForward(usize, file_size, 512); | | |
| 296 | const file_type = header.fileType(); | | |
| 297 | const link_name = if (self.link_name_len == 0) | | |
| 298 | header.linkName() | | |
| 299 | else | | |
| 300 | self.link_name_buffer[0..self.link_name_len]; | | |
| 301 | const file_name = if (self.file_name_len == 0) | | |
| 302 | try header.fullFileName(&self.file_name_buffer) | | |
| 303 | else | | |
| 304 | self.file_name_buffer[0..self.file_name_len]; | | |
| 305 | | 358 | |
| 306 | switch (file_type) { | 359 | switch (file_type) { |
| 307 | .directory, .normal, .symbolic_link => { | 360 | .directory, .normal, .symbolic_link => { |
| 308 | return File{ | 361 | if (file.size == 0) file.size = file_size; |
| 309 | .name = file_name, | 362 | if (file.name.len == 0) |
| 310 | .size = file_size, | 363 | file.name = try block.fullFileName((try self.attrs.alloc(std.fs.MAX_PATH_BYTES))[0..std.fs.MAX_PATH_BYTES]); |
| 311 | .file_type = file_type, | 364 | if (file.link_name.len == 0) file.link_name = block.linkName(); |
| 312 | .link_name = link_name, | 365 | file.file_type = file_type; |
| 313 | .iter = self, | 366 | return file; |
| 314 | }; | | |
| 315 | }, | 367 | }, |
| 316 | .global_extended_header => { | 368 | .global_extended_header => { |
| 317 | self.reader.skip(rounded_file_size) catch return error.TarHeadersTooBig; | 369 | self.reader.skipFile(file_size) catch return error.TarHeadersTooBig; |
| 318 | }, | 370 | }, |
| 319 | .extended_header => { | 371 | .extended_header => { |
| 320 | if (file_size == 0) continue; | 372 | if (file_size == 0) continue; |
| 321 | | 373 | |
| 322 | const chunk_size: usize = rounded_file_size + 512; | 374 | var rdr = self.reader.sliceReader(file_size, false); |
| 323 | var data_off: usize = 0; | 375 | while (try rdr.next()) |slice| { |
| 324 | while (data_off < file_size) { | 376 | const attr = try parsePaxAttribute(slice, rdr.remainingSize()); |
| 325 | const slice = try self.reader.readChunk(chunk_size - data_off); | 377 | try rdr.advance(attr.value_off); |
| 326 | if (slice.len == 0) return error.UnexpectedEndOfStream; | 378 | if (attr.is("path")) { |
| 327 | const remaining_size: usize = file_size - data_off; | 379 | file.name = try rdr.copy(try self.attrs.alloc(attr.value_len)); |
| 328 | const attr_info = try parsePaxAttribute(slice[0..@min(remaining_size, slice.len)], remaining_size); | 380 | } else if (attr.is("linkpath")) { |
| 329 | | 381 | file.link_name = try rdr.copy(try self.attrs.alloc(attr.value_len)); |
| 330 | if (std.mem.eql(u8, attr_info.key, "path")) { | 382 | } else if (attr.is("size")) { |
| 331 | if (attr_info.value_len > self.file_name_buffer.len) return error.NameTooLong; | 383 | var buf = [_]u8{'0'} ** 32; |
| 332 | self.reader.advance(attr_info.value_off); | 384 | file.size = try std.fmt.parseInt(usize, try rdr.copy(buf[0..attr.value_len]), 10); |
| 333 | try self.reader.copy(&self.file_name_buffer, attr_info.value_len); | | |
| 334 | self.file_name_len = attr_info.value_len; | | |
| 335 | self.reader.advance(1); | | |
| 336 | } else if (std.mem.eql(u8, attr_info.key, "linkpath")) { | | |
| 337 | if (attr_info.value_len > self.link_name_buffer.len) return error.NameTooLong; | | |
| 338 | self.reader.advance(attr_info.value_off); | | |
| 339 | try self.reader.copy(&self.link_name_buffer, attr_info.value_len); | | |
| 340 | self.link_name_len = attr_info.value_len; | | |
| 341 | self.reader.advance(1); | | |
| 342 | } else { | 385 | } else { |
| 343 | try self.reader.skip(attr_info.size); | 386 | try rdr.advance(attr.value_len); |
| 344 | } | 387 | } |
| 345 | data_off += attr_info.size; | 388 | try rdr.advance(1); |
| 346 | } | 389 | } |
| 347 | try self.reader.skip(rounded_file_size - data_off); | 390 | try self.reader.skipPadding(file_size); |
| 348 | | | |
| 349 | continue; | | |
| 350 | }, | 391 | }, |
| 351 | .hard_link => return error.TarUnsupportedFileType, | 392 | .hard_link => return error.TarUnsupportedFileType, |
| 352 | else => { | 393 | else => { |
| 353 | const d = self.diagnostics orelse return error.TarUnsupportedFileType; | 394 | const d = self.diagnostics orelse return error.TarUnsupportedFileType; |
| 354 | try d.errors.append(d.allocator, .{ .unsupported_file_type = .{ | 395 | try d.errors.append(d.allocator, .{ .unsupported_file_type = .{ |
| 355 | .file_name = try d.allocator.dupe(u8, file_name), | 396 | .file_name = try d.allocator.dupe(u8, block.name()), |
| 356 | .file_type = file_type, | 397 | .file_type = file_type, |
| 357 | } }); | 398 | } }); |
| 358 | }, | 399 | }, |
| 359 | } | 400 | } |
| 360 | } | 401 | } |
| | 402 | return null; |
| 361 | } | 403 | } |
| 362 | }; | 404 | }; |
| 363 | } | 405 | } |
| ... | @@ -481,6 +523,10 @@ const PaxAttributeInfo = struct { | ... | @@ -481,6 +523,10 @@ const PaxAttributeInfo = struct { |
| 481 | key: []const u8, | 523 | key: []const u8, |
| 482 | value_off: usize, | 524 | value_off: usize, |
| 483 | value_len: usize, | 525 | value_len: usize, |
| | 526 | |
| | 527 | inline fn is(self: @This(), key: []const u8) bool { |
| | 528 | return (std.mem.eql(u8, self.key, key)); |
| | 529 | } |
| 484 | }; | 530 | }; |
| 485 | | 531 | |
| 486 | fn parsePaxAttribute(data: []const u8, max_size: usize) !PaxAttributeInfo { | 532 | fn parsePaxAttribute(data: []const u8, max_size: usize) !PaxAttributeInfo { |
| ... | @@ -515,7 +561,7 @@ test parsePaxAttribute { | ... | @@ -515,7 +561,7 @@ test parsePaxAttribute { |
| 515 | try expectError(error.InvalidPaxAttribute, parsePaxAttribute("", 0)); | 561 | try expectError(error.InvalidPaxAttribute, parsePaxAttribute("", 0)); |
| 516 | } | 562 | } |
| 517 | | 563 | |
| 518 | const std = @import("std.zig"); | 564 | const std = @import("std"); |
| 519 | const assert = std.debug.assert; | 565 | const assert = std.debug.assert; |
| 520 | | 566 | |
| 521 | const TestCase = struct { | 567 | const TestCase = struct { |
| ... | @@ -628,19 +674,19 @@ test "Go test cases" { | ... | @@ -628,19 +674,19 @@ test "Go test cases" { |
| 628 | // }, | 674 | // }, |
| 629 | // | 675 | // |
| 630 | // TODO: giving wrong result because we are not reading pax size header | 676 | // TODO: giving wrong result because we are not reading pax size header |
| 631 | // .{ | 677 | .{ |
| 632 | // .path = "pax-pos-size-file.tar", | 678 | .path = "pax-pos-size-file.tar", |
| 633 | // .files = &[_]TestCase.File{ | 679 | .files = &[_]TestCase.File{ |
| 634 | // .{ | 680 | .{ |
| 635 | // .name = "foo", | 681 | .name = "foo", |
| 636 | // .size = 999, | 682 | .size = 999, |
| 637 | // .file_type = .normal, | 683 | .file_type = .normal, |
| 638 | // }, | 684 | }, |
| 639 | // }, | 685 | }, |
| 640 | // .chksums = &[_][]const u8{ | 686 | .chksums = &[_][]const u8{ |
| 641 | // "0afb597b283fe61b5d4879669a350556", | 687 | "0afb597b283fe61b5d4879669a350556", |
| 642 | // }, | 688 | }, |
| 643 | // }, | 689 | }, |
| 644 | .{ | 690 | .{ |
| 645 | // has pax records which we are not interested in | 691 | // has pax records which we are not interested in |
| 646 | .path = "pax-records.tar", | 692 | .path = "pax-records.tar", |
Igor Anić <igor.anic@gmail.com> 2023-11-28 23:07:37+01:00
Andrew Kelley <andrew@ziglang.org> 2024-01-13 19:37:33-07:00
lib