authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2023-11-27 14:51:51+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-01-13 19:37:33-07:00
logff8544daa59fa27b1e5c90e5ccdb6c513d719d47
tree4a9abd555a4000149c775516504d50882b8a5204
parent7916cf6f83650517f39a9e6aec23ba53d176a2ba

tar: refactor code to be more testable

Split reading/parsing tar file and writing results to the disk in two separate steps. So we can later test parsing part without need to write everyting to the disk.

1 files changed, 153 insertions(+), 89 deletions(-)

lib/std/tar.zig+153-89
......@@ -174,6 +174,144 @@ const Buffer = struct {
174174 }
175175};
176176
177fn Iterator(comptime ReaderType: type) type {
178 return struct {
179 file_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined,
180 file_name_len: usize = 0,
181 buffer: Buffer = .{},
182 reader: ReaderType,
183 pad_len: usize = 0,
184 diagnostics: ?*Options.Diagnostics,
185
186 const Self = @This();
187
188 const File = struct {
189 file_name: []const u8,
190 link_name: []const u8,
191 size: usize,
192 file_type: Header.FileType,
193 iter: *Self,
194
195 pub fn write(self: File, writer: anytype) !void {
196 const rounded_file_size = std.mem.alignForward(u64, self.size, 512);
197 var file_off: usize = 0;
198 while (true) {
199 const temp = try self.iter.buffer.readChunk(self.iter.reader, @intCast(rounded_file_size + 512 - file_off));
200 if (temp.len == 0) return error.UnexpectedEndOfStream;
201 const slice = temp[0..@intCast(@min(self.size - file_off, temp.len))];
202 try writer.writeAll(slice);
203
204 file_off += slice.len;
205 self.iter.buffer.advance(slice.len);
206 if (file_off >= self.size) {
207 return;
208 // self.iter.buffer.advance(pad_len);
209 // continue :header;
210 }
211 }
212 }
213
214 pub fn skip(self: File) void {
215 _ = self;
216 unreachable;
217 }
218 };
219
220 pub fn next(self: *Self) !?File {
221 self.buffer.advance(self.pad_len);
222 self.pad_len = 0;
223 self.file_name_len = 0;
224
225 while (true) {
226 const chunk = try self.buffer.readChunk(self.reader, 1024);
227 switch (chunk.len) {
228 0 => return null,
229 1...511 => return error.UnexpectedEndOfStream,
230 else => {},
231 }
232 self.buffer.advance(512);
233
234 const header: Header = .{ .bytes = chunk[0..512] };
235 const file_size = try header.fileSize();
236 const file_type = header.fileType();
237 const link_name = header.linkName();
238 const rounded_file_size = std.mem.alignForward(u64, file_size, 512);
239 self.pad_len = @intCast(rounded_file_size - file_size);
240 const file_name = if (self.file_name_len == 0)
241 try header.fullFileName(&self.file_name_buffer)
242 else
243 self.file_name_buffer[0..self.file_name_len];
244
245 switch (file_type) {
246 .directory, .normal, .symbolic_link => {
247 return File{
248 .file_name = file_name,
249 .link_name = link_name,
250 .size = file_size,
251 .file_type = file_type,
252 .iter = self,
253 };
254 },
255 .global_extended_header => {
256 self.buffer.skip(self.reader, @intCast(rounded_file_size)) catch return error.TarHeadersTooBig;
257 },
258 .extended_header => {
259 if (file_size == 0) {
260 self.buffer.advance(@intCast(rounded_file_size));
261 continue;
262 }
263
264 const chunk_size: usize = @intCast(rounded_file_size + 512);
265 var data_off: usize = 0;
266 const file_name_override_len = while (data_off < file_size) {
267 const slice = try self.buffer.readChunk(self.reader, chunk_size - data_off);
268 if (slice.len == 0) return error.UnexpectedEndOfStream;
269 const remaining_size: usize = @intCast(file_size - data_off);
270 const attr_info = try parsePaxAttribute(slice[0..@min(remaining_size, slice.len)], remaining_size);
271
272 if (std.mem.eql(u8, attr_info.key, "path")) {
273 if (attr_info.value_len > self.file_name_buffer.len) return error.NameTooLong;
274 self.buffer.advance(attr_info.value_off);
275 data_off += attr_info.value_off;
276 break attr_info.value_len;
277 }
278
279 try self.buffer.skip(self.reader, attr_info.size);
280 data_off += attr_info.size;
281 } else 0;
282
283 var i: usize = 0;
284 while (i < file_name_override_len) {
285 const slice = try self.buffer.readChunk(self.reader, chunk_size - data_off - i);
286 if (slice.len == 0) return error.UnexpectedEndOfStream;
287 const copy_size: usize = @intCast(@min(file_name_override_len - i, slice.len));
288 @memcpy(self.file_name_buffer[i .. i + copy_size], slice[0..copy_size]);
289 self.buffer.advance(copy_size);
290 i += copy_size;
291 }
292
293 try self.buffer.skip(self.reader, @intCast(rounded_file_size - data_off - file_name_override_len));
294 self.file_name_len = file_name_override_len;
295 continue;
296 },
297 .hard_link => return error.TarUnsupportedFileType,
298 else => {
299 const d = self.diagnostics orelse return error.TarUnsupportedFileType;
300 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
301 .file_name = try d.allocator.dupe(u8, file_name),
302 .file_type = file_type,
303 } });
304 },
305 }
306 }
307 }
308 };
309}
310
311pub fn iterator(reader: anytype, diagnostics: ?*Options.Diagnostics) Iterator(@TypeOf(reader)) {
312 return .{ .reader = reader, .diagnostics = diagnostics };
313}
314
177315pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !void {
178316 switch (options.mode_mode) {
179317 .ignore => {},
......@@ -186,37 +324,20 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
186324 @panic("TODO: unimplemented: tar ModeMode.executable_bit_only");
187325 },
188326 }
189 var file_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
190 var file_name_override_len: usize = 0;
191 var buffer: Buffer = .{};
192 header: while (true) {
193 const chunk = try buffer.readChunk(reader, 1024);
194 switch (chunk.len) {
195 0 => return,
196 1...511 => return error.UnexpectedEndOfStream,
197 else => {},
198 }
199 buffer.advance(512);
200
201 const header: Header = .{ .bytes = chunk[0..512] };
202 const file_size = try header.fileSize();
203 const rounded_file_size = std.mem.alignForward(u64, file_size, 512);
204 const pad_len: usize = @intCast(rounded_file_size - file_size);
205 const unstripped_file_name = if (file_name_override_len > 0)
206 file_name_buffer[0..file_name_override_len]
207 else
208 try header.fullFileName(&file_name_buffer);
209 file_name_override_len = 0;
210 switch (header.fileType()) {
327
328 var iter = iterator(reader, options.diagnostics);
329
330 while (try iter.next()) |iter_file| {
331 switch (iter_file.file_type) {
211332 .directory => {
212 const file_name = try stripComponents(unstripped_file_name, options.strip_components);
333 const file_name = try stripComponents(iter_file.file_name, options.strip_components);
213334 if (file_name.len != 0 and !options.exclude_empty_directories) {
214335 try dir.makePath(file_name);
215336 }
216337 },
217338 .normal => {
218 if (file_size == 0 and unstripped_file_name.len == 0) return;
219 const file_name = try stripComponents(unstripped_file_name, options.strip_components);
339 if (iter_file.size == 0 and iter_file.file_name.len == 0) return;
340 const file_name = try stripComponents(iter_file.file_name, options.strip_components);
220341
221342 const file = dir.createFile(file_name, .{}) catch |err| switch (err) {
222343 error.FileNotFound => again: {
......@@ -240,68 +361,17 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
240361 };
241362 defer if (file) |f| f.close();
242363
243 var file_off: usize = 0;
244 while (true) {
245 const temp = try buffer.readChunk(reader, @intCast(rounded_file_size + 512 - file_off));
246 if (temp.len == 0) return error.UnexpectedEndOfStream;
247 const slice = temp[0..@intCast(@min(file_size - file_off, temp.len))];
248 if (file) |f| try f.writeAll(slice);
249
250 file_off += slice.len;
251 buffer.advance(slice.len);
252 if (file_off >= file_size) {
253 buffer.advance(pad_len);
254 continue :header;
255 }
364 if (file) |f| {
365 try iter_file.write(f);
366 } else {
367 iter_file.skip();
256368 }
257369 },
258 .extended_header => {
259 if (file_size == 0) {
260 buffer.advance(@intCast(rounded_file_size));
261 continue;
262 }
263
264 const chunk_size: usize = @intCast(rounded_file_size + 512);
265 var data_off: usize = 0;
266 file_name_override_len = while (data_off < file_size) {
267 const slice = try buffer.readChunk(reader, chunk_size - data_off);
268 if (slice.len == 0) return error.UnexpectedEndOfStream;
269 const remaining_size: usize = @intCast(file_size - data_off);
270 const attr_info = try parsePaxAttribute(slice[0..@min(remaining_size, slice.len)], remaining_size);
271
272 if (std.mem.eql(u8, attr_info.key, "path")) {
273 if (attr_info.value_len > file_name_buffer.len) return error.NameTooLong;
274 buffer.advance(attr_info.value_off);
275 data_off += attr_info.value_off;
276 break attr_info.value_len;
277 }
278
279 try buffer.skip(reader, attr_info.size);
280 data_off += attr_info.size;
281 } else 0;
282
283 var i: usize = 0;
284 while (i < file_name_override_len) {
285 const slice = try buffer.readChunk(reader, chunk_size - data_off - i);
286 if (slice.len == 0) return error.UnexpectedEndOfStream;
287 const copy_size: usize = @intCast(@min(file_name_override_len - i, slice.len));
288 @memcpy(file_name_buffer[i .. i + copy_size], slice[0..copy_size]);
289 buffer.advance(copy_size);
290 i += copy_size;
291 }
292
293 try buffer.skip(reader, @intCast(rounded_file_size - data_off - file_name_override_len));
294 continue :header;
295 },
296 .global_extended_header => {
297 buffer.skip(reader, @intCast(rounded_file_size)) catch return error.TarHeadersTooBig;
298 },
299 .hard_link => return error.TarUnsupportedFileType,
300370 .symbolic_link => {
301371 // The file system path of the symbolic link.
302 const file_name = try stripComponents(unstripped_file_name, options.strip_components);
372 const file_name = try stripComponents(iter_file.file_name, options.strip_components);
303373 // The data inside the symbolic link.
304 const link_name = header.linkName();
374 const link_name = iter_file.link_name;
305375
306376 dir.symLink(link_name, file_name, .{}) catch |err| again: {
307377 const code = code: {
......@@ -323,13 +393,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
323393 } });
324394 };
325395 },
326 else => |file_type| {
327 const d = options.diagnostics orelse return error.TarUnsupportedFileType;
328 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
329 .file_name = try d.allocator.dupe(u8, unstripped_file_name),
330 .file_type = file_type,
331 } });
332 },
396 else => unreachable,
333397 }
334398 }
335399}