| author |  Andrew Kelley <andrew@ziglang.org> 2024-03-06 21:15:36-07:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2024-03-10 17:51:07-07:00 |
| log | ffd53a459e1b665e5070987f68dd583171a12459 |
| tree | 18691adb98010a6ac185428b740013d917a790ac |
| parent | 91260459e3c534b1583b516a2888f232d3d99581 |
-femit-docs: creating sources.tar
It's always a good day when you get to use File.writeFileAll 😎4 files changed, 196 insertions(+), 99 deletions(-)
lib- compiler
- std
lib/compiler/std-docs.zig+1-76
| ... | @@ -167,9 +167,8 @@ fn serveSourcesTar(request: *std.http.Server.Request, context: *Context) !void { | ... | @@ -167,9 +167,8 @@ fn serveSourcesTar(request: *std.http.Server.Request, context: *Context) !void { |
| 167 | const remainder = stat.size % 512; | 167 | const remainder = stat.size % 512; |
| 168 | break :p if (remainder > 0) 512 - remainder else 0; | 168 | break :p if (remainder > 0) 512 - remainder else 0; |
| 169 | }; | 169 | }; |
| 170 | comptime assert(@sizeOf(TarHeader) == 512); | ||
| 171 | 170 | ||
| 172 | var file_header = TarHeader.init(); | 171 | var file_header = std.tar.output.Header.init(); |
| 173 | file_header.typeflag = .regular; | 172 | file_header.typeflag = .regular; |
| 174 | try file_header.setPath("std", entry.path); | 173 | try file_header.setPath("std", entry.path); |
| 175 | try file_header.setSize(stat.size); | 174 | try file_header.setSize(stat.size); |
| ... | @@ -383,77 +382,3 @@ fn openBrowserTabThread(gpa: Allocator, url: []const u8) !void { | ... | @@ -383,77 +382,3 @@ fn openBrowserTabThread(gpa: Allocator, url: []const u8) !void { |
| 383 | try child.spawn(); | 382 | try child.spawn(); |
| 384 | _ = try child.wait(); | 383 | _ = try child.wait(); |
| 385 | } | 384 | } |
| 386 | |||
| 387 | /// Forked from https://github.com/mattnite/tar/blob/main/src/main.zig which is | ||
| 388 | /// MIT licensed. | ||
| 389 | pub const TarHeader = extern struct { | ||
| 390 | name: [100]u8, | ||
| 391 | mode: [7:0]u8, | ||
| 392 | uid: [7:0]u8, | ||
| 393 | gid: [7:0]u8, | ||
| 394 | size: [11:0]u8, | ||
| 395 | mtime: [11:0]u8, | ||
| 396 | checksum: [7:0]u8, | ||
| 397 | typeflag: FileType, | ||
| 398 | linkname: [100]u8, | ||
| 399 | magic: [5:0]u8, | ||
| 400 | version: [2]u8, | ||
| 401 | uname: [31:0]u8, | ||
| 402 | gname: [31:0]u8, | ||
| 403 | devmajor: [7:0]u8, | ||
| 404 | devminor: [7:0]u8, | ||
| 405 | prefix: [155]u8, | ||
| 406 | pad: [12]u8, | ||
| 407 | |||
| 408 | const FileType = enum(u8) { | ||
| 409 | regular = '0', | ||
| 410 | hard_link = '1', | ||
| 411 | symbolic_link = '2', | ||
| 412 | character = '3', | ||
| 413 | block = '4', | ||
| 414 | directory = '5', | ||
| 415 | fifo = '6', | ||
| 416 | reserved = '7', | ||
| 417 | pax_global = 'g', | ||
| 418 | extended = 'x', | ||
| 419 | _, | ||
| 420 | }; | ||
| 421 | |||
| 422 | fn init() TarHeader { | ||
| 423 | var ret = std.mem.zeroes(TarHeader); | ||
| 424 | ret.magic = [_:0]u8{ 'u', 's', 't', 'a', 'r' }; | ||
| 425 | ret.version = [_:0]u8{ '0', '0' }; | ||
| 426 | return ret; | ||
| 427 | } | ||
| 428 | |||
| 429 | fn setPath(self: *TarHeader, prefix: []const u8, path: []const u8) !void { | ||
| 430 | if (prefix.len + 1 + path.len > 100) { | ||
| 431 | var i: usize = 0; | ||
| 432 | while (i < path.len and path.len - i > 100) { | ||
| 433 | while (path[i] != '/') : (i += 1) {} | ||
| 434 | } | ||
| 435 | |||
| 436 | _ = try std.fmt.bufPrint(&self.prefix, "{s}/{s}", .{ prefix, path[0..i] }); | ||
| 437 | _ = try std.fmt.bufPrint(&self.name, "{s}", .{path[i + 1 ..]}); | ||
| 438 | } else { | ||
| 439 | _ = try std.fmt.bufPrint(&self.name, "{s}/{s}", .{ prefix, path }); | ||
| 440 | } | ||
| 441 | } | ||
| 442 | |||
| 443 | fn setSize(self: *TarHeader, size: u64) !void { | ||
| 444 | _ = try std.fmt.bufPrint(&self.size, "{o:0>11}", .{size}); | ||
| 445 | } | ||
| 446 | |||
| 447 | fn updateChecksum(self: *TarHeader) !void { | ||
| 448 | const offset = @offsetOf(TarHeader, "checksum"); | ||
| 449 | var checksum: usize = 0; | ||
| 450 | for (std.mem.asBytes(self), 0..) |val, i| { | ||
| 451 | checksum += if (i >= offset and i < offset + @sizeOf(@TypeOf(self.checksum))) | ||
| 452 | ' ' | ||
| 453 | else | ||
| 454 | val; | ||
| 455 | } | ||
| 456 | |||
| 457 | _ = try std.fmt.bufPrint(&self.checksum, "{o:0>7}", .{checksum}); | ||
| 458 | } | ||
| 459 | }; |
lib/std/tar.zig+19-17
| ... | @@ -1,23 +1,25 @@ | ... | @@ -1,23 +1,25 @@ |
| 1 | /// Tar archive is single ordinary file which can contain many files (or | 1 | //! Tar archive is single ordinary file which can contain many files (or |
| 2 | /// directories, symlinks, ...). It's build by series of blocks each size of 512 | 2 | //! directories, symlinks, ...). It's build by series of blocks each size of 512 |
| 3 | /// bytes. First block of each entry is header which defines type, name, size | 3 | //! bytes. First block of each entry is header which defines type, name, size |
| 4 | /// permissions and other attributes. Header is followed by series of blocks of | 4 | //! permissions and other attributes. Header is followed by series of blocks of |
| 5 | /// file content, if any that entry has content. Content is padded to the block | 5 | //! file content, if any that entry has content. Content is padded to the block |
| 6 | /// size, so next header always starts at block boundary. | 6 | //! size, so next header always starts at block boundary. |
| 7 | /// | 7 | //! |
| 8 | /// This simple format is extended by GNU and POSIX pax extensions to support | 8 | //! This simple format is extended by GNU and POSIX pax extensions to support |
| 9 | /// file names longer than 256 bytes and additional attributes. | 9 | //! file names longer than 256 bytes and additional attributes. |
| 10 | /// | 10 | //! |
| 11 | /// This is not comprehensive tar parser. Here we are only file types needed to | 11 | //! This is not comprehensive tar parser. Here we are only file types needed to |
| 12 | /// support Zig package manager; normal file, directory, symbolic link. And | 12 | //! support Zig package manager; normal file, directory, symbolic link. And |
| 13 | /// subset of attributes: name, size, permissions. | 13 | //! subset of attributes: name, size, permissions. |
| 14 | /// | 14 | //! |
| 15 | /// GNU tar reference: https://www.gnu.org/software/tar/manual/html_node/Standard.html | 15 | //! GNU tar reference: https://www.gnu.org/software/tar/manual/html_node/Standard.html |
| 16 | /// pax reference: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13 | 16 | //! pax reference: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13 |
| 17 | /// | 17 | |
| 18 | const std = @import("std.zig"); | 18 | const std = @import("std.zig"); |
| 19 | const assert = std.debug.assert; | 19 | const assert = std.debug.assert; |
| 20 | 20 | ||
| 21 | pub const output = @import("tar/output.zig"); | ||
| 22 | |||
| 21 | pub const Options = struct { | 23 | pub const Options = struct { |
| 22 | /// Number of directory levels to skip when extracting files. | 24 | /// Number of directory levels to skip when extracting files. |
| 23 | strip_components: u32 = 0, | 25 | strip_components: u32 = 0, |
lib/std/tar/output.zig created+85
| ... | @@ -0,0 +1,85 @@ | ||
| 1 | /// A struct that is exactly 512 bytes and matches tar file format. This is | ||
| 2 | /// intended to be used for outputting tar files; for parsing there is | ||
| 3 | /// `std.tar.Header`. | ||
| 4 | pub const Header = extern struct { | ||
| 5 | // This struct was originally copied from | ||
| 6 | // https://github.com/mattnite/tar/blob/main/src/main.zig which is MIT | ||
| 7 | // licensed. | ||
| 8 | |||
| 9 | name: [100]u8, | ||
| 10 | mode: [7:0]u8, | ||
| 11 | uid: [7:0]u8, | ||
| 12 | gid: [7:0]u8, | ||
| 13 | size: [11:0]u8, | ||
| 14 | mtime: [11:0]u8, | ||
| 15 | checksum: [7:0]u8, | ||
| 16 | typeflag: FileType, | ||
| 17 | linkname: [100]u8, | ||
| 18 | magic: [5:0]u8, | ||
| 19 | version: [2]u8, | ||
| 20 | uname: [31:0]u8, | ||
| 21 | gname: [31:0]u8, | ||
| 22 | devmajor: [7:0]u8, | ||
| 23 | devminor: [7:0]u8, | ||
| 24 | prefix: [155]u8, | ||
| 25 | pad: [12]u8, | ||
| 26 | |||
| 27 | pub const FileType = enum(u8) { | ||
| 28 | regular = '0', | ||
| 29 | hard_link = '1', | ||
| 30 | symbolic_link = '2', | ||
| 31 | character = '3', | ||
| 32 | block = '4', | ||
| 33 | directory = '5', | ||
| 34 | fifo = '6', | ||
| 35 | reserved = '7', | ||
| 36 | pax_global = 'g', | ||
| 37 | extended = 'x', | ||
| 38 | _, | ||
| 39 | }; | ||
| 40 | |||
| 41 | pub fn init() Header { | ||
| 42 | var ret = std.mem.zeroes(Header); | ||
| 43 | ret.magic = [_:0]u8{ 'u', 's', 't', 'a', 'r' }; | ||
| 44 | ret.version = [_:0]u8{ '0', '0' }; | ||
| 45 | return ret; | ||
| 46 | } | ||
| 47 | |||
| 48 | pub fn setPath(self: *Header, prefix: []const u8, path: []const u8) !void { | ||
| 49 | if (prefix.len + 1 + path.len > 100) { | ||
| 50 | var i: usize = 0; | ||
| 51 | while (i < path.len and path.len - i > 100) { | ||
| 52 | while (path[i] != '/') : (i += 1) {} | ||
| 53 | } | ||
| 54 | |||
| 55 | _ = try std.fmt.bufPrint(&self.prefix, "{s}/{s}", .{ prefix, path[0..i] }); | ||
| 56 | _ = try std.fmt.bufPrint(&self.name, "{s}", .{path[i + 1 ..]}); | ||
| 57 | } else { | ||
| 58 | _ = try std.fmt.bufPrint(&self.name, "{s}/{s}", .{ prefix, path }); | ||
| 59 | } | ||
| 60 | } | ||
| 61 | |||
| 62 | pub fn setSize(self: *Header, size: u64) !void { | ||
| 63 | _ = try std.fmt.bufPrint(&self.size, "{o:0>11}", .{size}); | ||
| 64 | } | ||
| 65 | |||
| 66 | pub fn updateChecksum(self: *Header) !void { | ||
| 67 | const offset = @offsetOf(Header, "checksum"); | ||
| 68 | var checksum: usize = 0; | ||
| 69 | for (std.mem.asBytes(self), 0..) |val, i| { | ||
| 70 | checksum += if (i >= offset and i < offset + @sizeOf(@TypeOf(self.checksum))) | ||
| 71 | ' ' | ||
| 72 | else | ||
| 73 | val; | ||
| 74 | } | ||
| 75 | |||
| 76 | _ = try std.fmt.bufPrint(&self.checksum, "{o:0>7}", .{checksum}); | ||
| 77 | } | ||
| 78 | |||
| 79 | comptime { | ||
| 80 | assert(@sizeOf(Header) == 512); | ||
| 81 | } | ||
| 82 | }; | ||
| 83 | |||
| 84 | const std = @import("../std.zig"); | ||
| 85 | const assert = std.debug.assert; | ||
src/Compilation.zig+91-6
| ... | @@ -733,6 +733,7 @@ pub const MiscTask = enum { | ... | @@ -733,6 +733,7 @@ pub const MiscTask = enum { |
| 733 | compiler_rt, | 733 | compiler_rt, |
| 734 | zig_libc, | 734 | zig_libc, |
| 735 | analyze_mod, | 735 | analyze_mod, |
| 736 | docs_copy, | ||
| 736 | 737 | ||
| 737 | @"musl crti.o", | 738 | @"musl crti.o", |
| 738 | @"musl crtn.o", | 739 | @"musl crtn.o", |
| ... | @@ -3765,22 +3766,106 @@ fn taskDocsWasm(comp: *Compilation, wg: *WaitGroup) !void { | ... | @@ -3765,22 +3766,106 @@ fn taskDocsWasm(comp: *Compilation, wg: *WaitGroup) !void { |
| 3765 | 3766 | ||
| 3766 | fn workerDocsCopy(comp: *Compilation, wg: *WaitGroup) void { | 3767 | fn workerDocsCopy(comp: *Compilation, wg: *WaitGroup) void { |
| 3767 | defer wg.finish(); | 3768 | defer wg.finish(); |
| 3769 | docsCopyFallible(comp) catch |err| { | ||
| 3770 | return comp.lockAndSetMiscFailure( | ||
| 3771 | .docs_copy, | ||
| 3772 | "unable to copy autodocs artifacts: {s}", | ||
| 3773 | .{@errorName(err)}, | ||
| 3774 | ); | ||
| 3775 | }; | ||
| 3776 | } | ||
| 3768 | 3777 | ||
| 3778 | fn docsCopyFallible(comp: *Compilation) anyerror!void { | ||
| 3769 | const emit = comp.docs_emit.?; | 3779 | const emit = comp.docs_emit.?; |
| 3770 | var out_dir = emit.directory.handle.makeOpenPath(emit.sub_path, .{}) catch |err| { | 3780 | var out_dir = emit.directory.handle.makeOpenPath(emit.sub_path, .{}) catch |err| { |
| 3771 | // TODO create an error to be reported instead of logging | 3781 | return comp.lockAndSetMiscFailure( |
| 3772 | log.err("unable to create output directory '{}{s}': {s}", .{ | 3782 | .docs_copy, |
| 3773 | emit.directory, emit.sub_path, @errorName(err), | 3783 | "unable to create output directory '{}{s}': {s}", |
| 3774 | }); | 3784 | .{ emit.directory, emit.sub_path, @errorName(err) }, |
| 3775 | return; | 3785 | ); |
| 3776 | }; | 3786 | }; |
| 3777 | defer out_dir.close(); | 3787 | defer out_dir.close(); |
| 3778 | 3788 | ||
| 3779 | for (&[_][]const u8{ "docs/main.js", "docs/index.html" }) |sub_path| { | 3789 | for (&[_][]const u8{ "docs/main.js", "docs/index.html" }) |sub_path| { |
| 3780 | const basename = std.fs.path.basename(sub_path); | 3790 | const basename = std.fs.path.basename(sub_path); |
| 3781 | comp.zig_lib_directory.handle.copyFile(sub_path, out_dir, basename, .{}) catch |err| { | 3791 | comp.zig_lib_directory.handle.copyFile(sub_path, out_dir, basename, .{}) catch |err| { |
| 3782 | log.err("unable to copy {s}: {s}", .{ sub_path, @errorName(err) }); | 3792 | comp.lockAndSetMiscFailure(.docs_copy, "unable to copy {s}: {s}", .{ |
| 3793 | sub_path, | ||
| 3794 | @errorName(err), | ||
| 3795 | }); | ||
| 3796 | return; | ||
| 3797 | }; | ||
| 3798 | } | ||
| 3799 | |||
| 3800 | var tar_file = out_dir.createFile("sources.tar", .{}) catch |err| { | ||
| 3801 | return comp.lockAndSetMiscFailure( | ||
| 3802 | .docs_copy, | ||
| 3803 | "unable to create '{}{s}/sources.tar': {s}", | ||
| 3804 | .{ emit.directory, emit.sub_path, @errorName(err) }, | ||
| 3805 | ); | ||
| 3806 | }; | ||
| 3807 | defer tar_file.close(); | ||
| 3808 | |||
| 3809 | const root = comp.root_mod.root; | ||
| 3810 | const root_mod_name = comp.root_mod.fully_qualified_name; | ||
| 3811 | const sub_path = if (root.sub_path.len == 0) "." else root.sub_path; | ||
| 3812 | var mod_dir = root.root_dir.handle.openDir(sub_path, .{ .iterate = true }) catch |err| { | ||
| 3813 | return comp.lockAndSetMiscFailure(.docs_copy, "unable to open directory '{}': {s}", .{ | ||
| 3814 | root, @errorName(err), | ||
| 3815 | }); | ||
| 3816 | }; | ||
| 3817 | |||
| 3818 | var walker = try mod_dir.walk(comp.gpa); | ||
| 3819 | defer walker.deinit(); | ||
| 3820 | |||
| 3821 | const padding_buffer = [1]u8{0} ** 512; | ||
| 3822 | |||
| 3823 | while (try walker.next()) |entry| { | ||
| 3824 | switch (entry.kind) { | ||
| 3825 | .file => { | ||
| 3826 | if (!std.mem.endsWith(u8, entry.basename, ".zig")) continue; | ||
| 3827 | if (std.mem.eql(u8, entry.basename, "test.zig")) continue; | ||
| 3828 | if (std.mem.endsWith(u8, entry.basename, "_test.zig")) continue; | ||
| 3829 | }, | ||
| 3830 | else => continue, | ||
| 3831 | } | ||
| 3832 | |||
| 3833 | var file = mod_dir.openFile(entry.path, .{}) catch |err| { | ||
| 3834 | return comp.lockAndSetMiscFailure(.docs_copy, "unable to open '{}{s}': {s}", .{ | ||
| 3835 | root, entry.path, @errorName(err), | ||
| 3836 | }); | ||
| 3837 | }; | ||
| 3838 | defer file.close(); | ||
| 3839 | |||
| 3840 | const stat = file.stat() catch |err| { | ||
| 3841 | return comp.lockAndSetMiscFailure(.docs_copy, "unable to stat '{}{s}': {s}", .{ | ||
| 3842 | root, entry.path, @errorName(err), | ||
| 3843 | }); | ||
| 3844 | }; | ||
| 3845 | |||
| 3846 | var file_header = std.tar.output.Header.init(); | ||
| 3847 | file_header.typeflag = .regular; | ||
| 3848 | try file_header.setPath(root_mod_name, entry.path); | ||
| 3849 | try file_header.setSize(stat.size); | ||
| 3850 | try file_header.updateChecksum(); | ||
| 3851 | |||
| 3852 | const header_bytes = std.mem.asBytes(&file_header); | ||
| 3853 | const padding = p: { | ||
| 3854 | const remainder = stat.size % 512; | ||
| 3855 | const n = if (remainder > 0) 512 - remainder else 0; | ||
| 3856 | break :p padding_buffer[0..n]; | ||
| 3783 | }; | 3857 | }; |
| 3858 | |||
| 3859 | var header_and_trailer: [2]std.os.iovec_const = .{ | ||
| 3860 | .{ .iov_base = header_bytes.ptr, .iov_len = header_bytes.len }, | ||
| 3861 | .{ .iov_base = padding.ptr, .iov_len = padding.len }, | ||
| 3862 | }; | ||
| 3863 | |||
| 3864 | try tar_file.writeFileAll(file, .{ | ||
| 3865 | .in_len = stat.size, | ||
| 3866 | .headers_and_trailers = &header_and_trailer, | ||
| 3867 | .header_count = 1, | ||
| 3868 | }); | ||
| 3784 | } | 3869 | } |
| 3785 | } | 3870 | } |
| 3786 | 3871 |