| ... | ... | @@ -461,14 +461,10 @@ fn runResource( |
| 461 | 461 | }; |
| 462 | 462 | defer tmp_directory.handle.close(); |
| 463 | 463 | |
| 464 | | // Unpack resource into tmp_directory. A non-null return value means |
| 465 | | // that the package contents are inside a `pkg_dir` sub-directory. |
| 466 | | const pkg_dir = try unpackResource(f, resource, uri_path, tmp_directory); |
| 464 | // Fetch and unpack a resource into a temporary directory. |
| 465 | var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); |
| 467 | 466 | |
| 468 | | var pkg_path: Cache.Path = .{ |
| 469 | | .root_dir = tmp_directory, |
| 470 | | .sub_path = if (pkg_dir) |pkg_dir_name| pkg_dir_name else "", |
| 471 | | }; |
| 467 | var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; |
| 472 | 468 | |
| 473 | 469 | // Apply btrfs workaround if needed. Reopen tmp_directory. |
| 474 | 470 | if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { |
| ... | ... | @@ -488,10 +484,9 @@ fn runResource( |
| 488 | 484 | .include_paths = if (f.manifest) |m| m.paths else .{}, |
| 489 | 485 | }; |
| 490 | 486 | |
| 491 | | // TODO: |
| 492 | | // If any error occurred for files that were ultimately excluded, those |
| 493 | | // errors should be ignored, such as failure to create symlinks that |
| 494 | | // weren't supposed to be included anyway. |
| 487 | // Ignore errors that were excluded by manifest, such as failure to |
| 488 | // create symlinks that weren't supposed to be included anyway. |
| 489 | try unpack_result.validate(f, filter); |
| 495 | 490 | |
| 496 | 491 | // Apply the manifest's inclusion rules to the temporary directory by |
| 497 | 492 | // deleting excluded files. |
| ... | ... | @@ -500,8 +495,8 @@ fn runResource( |
| 500 | 495 | // directory. |
| 501 | 496 | f.actual_hash = try computeHash(f, pkg_path, filter); |
| 502 | 497 | |
| 503 | | break :blk if (pkg_dir) |pkg_dir_name| |
| 504 | | try fs.path.join(arena, &.{ tmp_dir_sub_path, pkg_dir_name }) |
| 498 | break :blk if (unpack_result.root_dir.len > 0) |
| 499 | try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir }) |
| 505 | 500 | else |
| 506 | 501 | tmp_dir_sub_path; |
| 507 | 502 | }; |
| ... | ... | @@ -1044,16 +1039,12 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re |
| 1044 | 1039 | )); |
| 1045 | 1040 | } |
| 1046 | 1041 | |
| 1047 | | /// A `null` return value indicates the `tmp_directory` is populated directly |
| 1048 | | /// with the package contents. |
| 1049 | | /// A non-null return value means that the package contents are inside a |
| 1050 | | /// sub-directory indicated by the named path. |
| 1051 | 1042 | fn unpackResource( |
| 1052 | 1043 | f: *Fetch, |
| 1053 | 1044 | resource: *Resource, |
| 1054 | 1045 | uri_path: []const u8, |
| 1055 | 1046 | tmp_directory: Cache.Directory, |
| 1056 | | ) RunError!?[]const u8 { |
| 1047 | ) RunError!UnpackResult { |
| 1057 | 1048 | const eb = &f.error_bundle; |
| 1058 | 1049 | const file_type = switch (resource.*) { |
| 1059 | 1050 | .file => FileType.fromPath(uri_path) orelse |
| ... | ... | @@ -1121,7 +1112,7 @@ fn unpackResource( |
| 1121 | 1112 | .{ uri_path, @errorName(err) }, |
| 1122 | 1113 | )); |
| 1123 | 1114 | }; |
| 1124 | | return null; |
| 1115 | return .{}; |
| 1125 | 1116 | }, |
| 1126 | 1117 | }; |
| 1127 | 1118 | |
| ... | ... | @@ -1156,27 +1147,22 @@ fn unpackResource( |
| 1156 | 1147 | }); |
| 1157 | 1148 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); |
| 1158 | 1149 | }, |
| 1159 | | .git_pack => { |
| 1160 | | unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { |
| 1161 | | error.FetchFailed => return error.FetchFailed, |
| 1162 | | error.OutOfMemory => return error.OutOfMemory, |
| 1163 | | else => |e| return f.fail(f.location_tok, try eb.printString( |
| 1164 | | "unable to unpack git files: {s}", |
| 1165 | | .{@errorName(e)}, |
| 1166 | | )), |
| 1167 | | }; |
| 1168 | | return null; |
| 1150 | .git_pack => return unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { |
| 1151 | error.FetchFailed => return error.FetchFailed, |
| 1152 | error.OutOfMemory => return error.OutOfMemory, |
| 1153 | else => |e| return f.fail(f.location_tok, try eb.printString( |
| 1154 | "unable to unpack git files: {s}", |
| 1155 | .{@errorName(e)}, |
| 1156 | )), |
| 1169 | 1157 | }, |
| 1170 | 1158 | } |
| 1171 | 1159 | } |
| 1172 | 1160 | |
| 1173 | | fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!?[]const u8 { |
| 1161 | fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { |
| 1174 | 1162 | const eb = &f.error_bundle; |
| 1175 | 1163 | const arena = f.arena.allocator(); |
| 1176 | | const gpa = f.arena.child_allocator; |
| 1177 | 1164 | |
| 1178 | | var diagnostics: std.tar.Diagnostics = .{ .allocator = gpa }; |
| 1179 | | defer diagnostics.deinit(); |
| 1165 | var diagnostics: std.tar.Diagnostics = .{ .allocator = arena }; |
| 1180 | 1166 | |
| 1181 | 1167 | std.tar.pipeToFileSystem(out_dir, reader, .{ |
| 1182 | 1168 | .diagnostics = &diagnostics, |
| ... | ... | @@ -1188,53 +1174,27 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!?[]const |
| 1188 | 1174 | .{@errorName(err)}, |
| 1189 | 1175 | )); |
| 1190 | 1176 | |
| 1177 | var res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; |
| 1191 | 1178 | if (diagnostics.errors.items.len > 0) { |
| 1192 | | const notes_len: u32 = @intCast(diagnostics.errors.items.len); |
| 1193 | | try eb.addRootErrorMessage(.{ |
| 1194 | | .msg = try eb.addString("unable to unpack tarball"), |
| 1195 | | .src_loc = try f.srcLoc(f.location_tok), |
| 1196 | | .notes_len = notes_len, |
| 1197 | | }); |
| 1198 | | const notes_start = try eb.reserveNotes(notes_len); |
| 1199 | | for (diagnostics.errors.items, notes_start..) |item, note_i| { |
| 1179 | try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball"); |
| 1180 | for (diagnostics.errors.items) |item| { |
| 1200 | 1181 | switch (item) { |
| 1201 | | .unable_to_create_sym_link => |info| { |
| 1202 | | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ |
| 1203 | | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ |
| 1204 | | info.file_name, info.link_name, @errorName(info.code), |
| 1205 | | }), |
| 1206 | | })); |
| 1207 | | }, |
| 1208 | | .unable_to_create_file => |info| { |
| 1209 | | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ |
| 1210 | | .msg = try eb.printString("unable to create file '{s}': {s}", .{ |
| 1211 | | info.file_name, @errorName(info.code), |
| 1212 | | }), |
| 1213 | | })); |
| 1214 | | }, |
| 1215 | | .unsupported_file_type => |info| { |
| 1216 | | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ |
| 1217 | | .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ |
| 1218 | | info.file_name, @intFromEnum(info.file_type), |
| 1219 | | }), |
| 1220 | | })); |
| 1221 | | }, |
| 1182 | .unable_to_create_file => |i| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code), |
| 1183 | .unable_to_create_sym_link => |i| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code), |
| 1184 | .unsupported_file_type => |i| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)), |
| 1222 | 1185 | } |
| 1223 | 1186 | } |
| 1224 | | return error.FetchFailed; |
| 1225 | 1187 | } |
| 1226 | | |
| 1227 | | return if (diagnostics.root_dir) |root_dir| |
| 1228 | | return try arena.dupe(u8, root_dir) |
| 1229 | | else |
| 1230 | | null; |
| 1188 | return res; |
| 1231 | 1189 | } |
| 1232 | 1190 | |
| 1233 | | fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void { |
| 1234 | | const eb = &f.error_bundle; |
| 1191 | fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!UnpackResult { |
| 1192 | const arena = f.arena.allocator(); |
| 1235 | 1193 | const gpa = f.arena.child_allocator; |
| 1236 | 1194 | const want_oid = resource.git.want_oid; |
| 1237 | 1195 | const reader = resource.git.fetch_stream.reader(); |
| 1196 | |
| 1197 | var res: UnpackResult = .{}; |
| 1238 | 1198 | // The .git directory is used to store the packfile and associated index, but |
| 1239 | 1199 | // we do not attempt to replicate the exact structure of a real .git |
| 1240 | 1200 | // directory, since that isn't relevant for fetching a package. |
| ... | ... | @@ -1265,35 +1225,23 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void |
| 1265 | 1225 | checkout_prog_node.activate(); |
| 1266 | 1226 | var repository = try git.Repository.init(gpa, pack_file, index_file); |
| 1267 | 1227 | defer repository.deinit(); |
| 1268 | | var diagnostics: git.Diagnostics = .{ .allocator = gpa }; |
| 1269 | | defer diagnostics.deinit(); |
| 1228 | var diagnostics: git.Diagnostics = .{ .allocator = arena }; |
| 1270 | 1229 | try repository.checkout(out_dir, want_oid, &diagnostics); |
| 1271 | 1230 | |
| 1272 | 1231 | if (diagnostics.errors.items.len > 0) { |
| 1273 | | const notes_len: u32 = @intCast(diagnostics.errors.items.len); |
| 1274 | | try eb.addRootErrorMessage(.{ |
| 1275 | | .msg = try eb.addString("unable to unpack packfile"), |
| 1276 | | .src_loc = try f.srcLoc(f.location_tok), |
| 1277 | | .notes_len = notes_len, |
| 1278 | | }); |
| 1279 | | const notes_start = try eb.reserveNotes(notes_len); |
| 1280 | | for (diagnostics.errors.items, notes_start..) |item, note_i| { |
| 1232 | try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile"); |
| 1233 | for (diagnostics.errors.items) |item| { |
| 1281 | 1234 | switch (item) { |
| 1282 | | .unable_to_create_sym_link => |info| { |
| 1283 | | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ |
| 1284 | | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ |
| 1285 | | info.file_name, info.link_name, @errorName(info.code), |
| 1286 | | }), |
| 1287 | | })); |
| 1288 | | }, |
| 1235 | .unable_to_create_file => |i| res.unableToCreateFile(i.file_name, i.code), |
| 1236 | .unable_to_create_sym_link => |i| res.unableToCreateSymLink(i.file_name, i.link_name, i.code), |
| 1289 | 1237 | } |
| 1290 | 1238 | } |
| 1291 | | return error.InvalidGitPack; |
| 1292 | 1239 | } |
| 1293 | 1240 | } |
| 1294 | 1241 | } |
| 1295 | 1242 | |
| 1296 | 1243 | try out_dir.deleteTree(".git"); |
| 1244 | return res; |
| 1297 | 1245 | } |
| 1298 | 1246 | |
| 1299 | 1247 | fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void { |
| ... | ... | @@ -1743,6 +1691,7 @@ const native_os = builtin.os.tag; |
| 1743 | 1691 | test { |
| 1744 | 1692 | _ = Filter; |
| 1745 | 1693 | _ = FileType; |
| 1694 | _ = UnpackResult; |
| 1746 | 1695 | } |
| 1747 | 1696 | |
| 1748 | 1697 | // Detects executable header: ELF magic header or shebang line. |
| ... | ... | @@ -1778,3 +1727,472 @@ test FileHeader { |
| 1778 | 1727 | h.update(FileHeader.elf_magic[2..4]); |
| 1779 | 1728 | try std.testing.expect(h.isExecutable()); |
| 1780 | 1729 | } |
| 1730 | |
| 1731 | // Result of the `unpackResource` operation. Enables collecting errors from |
| 1732 | // tar/git diagnostic, filtering that errors by manifest inclusion rules and |
| 1733 | // emitting remaining errors to an `ErrorBundle`. |
| 1734 | const UnpackResult = struct { |
| 1735 | errors: []Error = undefined, |
| 1736 | errors_count: usize = 0, |
| 1737 | root_error_message: []const u8 = "", |
| 1738 | |
| 1739 | // A non empty value means that the package contents are inside a |
| 1740 | // sub-directory indicated by the named path. |
| 1741 | root_dir: []const u8 = "", |
| 1742 | |
| 1743 | const Error = union(enum) { |
| 1744 | unable_to_create_sym_link: struct { |
| 1745 | code: anyerror, |
| 1746 | file_name: []const u8, |
| 1747 | link_name: []const u8, |
| 1748 | }, |
| 1749 | unable_to_create_file: struct { |
| 1750 | code: anyerror, |
| 1751 | file_name: []const u8, |
| 1752 | }, |
| 1753 | unsupported_file_type: struct { |
| 1754 | file_name: []const u8, |
| 1755 | file_type: u8, |
| 1756 | }, |
| 1757 | |
| 1758 | fn excluded(self: Error, filter: Filter) bool { |
| 1759 | const file_name = switch (self) { |
| 1760 | .unable_to_create_file => |info| info.file_name, |
| 1761 | .unable_to_create_sym_link => |info| info.file_name, |
| 1762 | .unsupported_file_type => |info| info.file_name, |
| 1763 | }; |
| 1764 | return !filter.includePath(file_name); |
| 1765 | } |
| 1766 | }; |
| 1767 | |
| 1768 | fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void { |
| 1769 | self.root_error_message = try arena.dupe(u8, root_error_message); |
| 1770 | self.errors = try arena.alloc(UnpackResult.Error, n); |
| 1771 | } |
| 1772 | |
| 1773 | fn hasErrors(self: *UnpackResult) bool { |
| 1774 | return self.errors_count > 0; |
| 1775 | } |
| 1776 | |
| 1777 | fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void { |
| 1778 | self.errors[self.errors_count] = .{ .unable_to_create_file = .{ |
| 1779 | .code = err, |
| 1780 | .file_name = file_name, |
| 1781 | } }; |
| 1782 | self.errors_count += 1; |
| 1783 | } |
| 1784 | |
| 1785 | fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void { |
| 1786 | self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{ |
| 1787 | .code = err, |
| 1788 | .file_name = file_name, |
| 1789 | .link_name = link_name, |
| 1790 | } }; |
| 1791 | self.errors_count += 1; |
| 1792 | } |
| 1793 | |
| 1794 | fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void { |
| 1795 | self.errors[self.errors_count] = .{ .unsupported_file_type = .{ |
| 1796 | .file_name = file_name, |
| 1797 | .file_type = file_type, |
| 1798 | } }; |
| 1799 | self.errors_count += 1; |
| 1800 | } |
| 1801 | |
| 1802 | fn validate(self: *UnpackResult, f: *Fetch, filter: Filter) !void { |
| 1803 | self.filterErrors(filter); |
| 1804 | if (self.hasErrors()) { |
| 1805 | const eb = &f.error_bundle; |
| 1806 | try self.bundleErrors(eb, try f.srcLoc(f.location_tok)); |
| 1807 | return error.FetchFailed; |
| 1808 | } |
| 1809 | } |
| 1810 | |
| 1811 | // Filter errors by manifest inclusion rules. |
| 1812 | fn filterErrors(self: *UnpackResult, filter: Filter) void { |
| 1813 | var i = self.errors_count; |
| 1814 | while (i > 0) { |
| 1815 | i -= 1; |
| 1816 | if (self.errors[i].excluded(filter)) { |
| 1817 | self.errors_count -= 1; |
| 1818 | const tmp = self.errors[i]; |
| 1819 | self.errors[i] = self.errors[self.errors_count]; |
| 1820 | self.errors[self.errors_count] = tmp; |
| 1821 | } |
| 1822 | } |
| 1823 | } |
| 1824 | |
| 1825 | // Emmit errors to an `ErrorBundle`. |
| 1826 | fn bundleErrors( |
| 1827 | self: *UnpackResult, |
| 1828 | eb: *ErrorBundle.Wip, |
| 1829 | src_loc: ErrorBundle.SourceLocationIndex, |
| 1830 | ) !void { |
| 1831 | if (self.errors_count == 0 and self.root_error_message.len == 0) |
| 1832 | return; |
| 1833 | |
| 1834 | const notes_len: u32 = @intCast(self.errors_count); |
| 1835 | try eb.addRootErrorMessage(.{ |
| 1836 | .msg = try eb.addString(self.root_error_message), |
| 1837 | .src_loc = src_loc, |
| 1838 | .notes_len = notes_len, |
| 1839 | }); |
| 1840 | const notes_start = try eb.reserveNotes(notes_len); |
| 1841 | for (self.errors, notes_start..) |item, note_i| { |
| 1842 | switch (item) { |
| 1843 | .unable_to_create_sym_link => |info| { |
| 1844 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ |
| 1845 | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ |
| 1846 | info.file_name, info.link_name, @errorName(info.code), |
| 1847 | }), |
| 1848 | })); |
| 1849 | }, |
| 1850 | .unable_to_create_file => |info| { |
| 1851 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ |
| 1852 | .msg = try eb.printString("unable to create file '{s}': {s}", .{ |
| 1853 | info.file_name, @errorName(info.code), |
| 1854 | }), |
| 1855 | })); |
| 1856 | }, |
| 1857 | .unsupported_file_type => |info| { |
| 1858 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ |
| 1859 | .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ |
| 1860 | info.file_name, info.file_type, |
| 1861 | }), |
| 1862 | })); |
| 1863 | }, |
| 1864 | } |
| 1865 | } |
| 1866 | } |
| 1867 | |
| 1868 | test filterErrors { |
| 1869 | var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); |
| 1870 | defer arena_instance.deinit(); |
| 1871 | const arena = arena_instance.allocator(); |
| 1872 | |
| 1873 | // init |
| 1874 | var res: UnpackResult = .{}; |
| 1875 | try res.allocErrors(arena, 4, "error"); |
| 1876 | try std.testing.expectEqual(0, res.errors_count); |
| 1877 | |
| 1878 | // create errors |
| 1879 | res.unableToCreateFile("dir1/file1", error.File1); |
| 1880 | res.unableToCreateSymLink("dir2/file2", "", error.File2); |
| 1881 | res.unableToCreateFile("dir1/file3", error.File3); |
| 1882 | res.unsupportedFileType("dir2/file4", 'x'); |
| 1883 | try std.testing.expectEqual(4, res.errors_count); |
| 1884 | |
| 1885 | // filter errors |
| 1886 | var filter: Filter = .{}; |
| 1887 | try filter.include_paths.put(arena, "dir2", {}); |
| 1888 | res.filterErrors(filter); |
| 1889 | |
| 1890 | try std.testing.expectEqual(2, res.errors_count); |
| 1891 | try std.testing.expect(res.errors[0] == Error.unsupported_file_type); |
| 1892 | try std.testing.expect(res.errors[1] == Error.unable_to_create_sym_link); |
| 1893 | // filtered: moved to the list end |
| 1894 | try std.testing.expect(res.errors[2] == Error.unable_to_create_file); |
| 1895 | try std.testing.expect(res.errors[3] == Error.unable_to_create_file); |
| 1896 | } |
| 1897 | }; |
| 1898 | |
| 1899 | test "tarball with duplicate paths" { |
| 1900 | // This tarball has duplicate path 'dir1/file1' to simulate case sensitve |
| 1901 | // file system on any file sytstem. |
| 1902 | // |
| 1903 | // duplicate_paths/ |
| 1904 | // duplicate_paths/dir1/ |
| 1905 | // duplicate_paths/dir1/file1 |
| 1906 | // duplicate_paths/dir1/file1 |
| 1907 | // duplicate_paths/build.zig.zon |
| 1908 | // duplicate_paths/src/ |
| 1909 | // duplicate_paths/src/main.zig |
| 1910 | // duplicate_paths/src/root.zig |
| 1911 | // duplicate_paths/build.zig |
| 1912 | // |
| 1913 | |
| 1914 | const gpa = std.testing.allocator; |
| 1915 | var tmp = std.testing.tmpDir(.{}); |
| 1916 | defer tmp.cleanup(); |
| 1917 | |
| 1918 | const tarball_name = "duplicate_paths.tar.gz"; |
| 1919 | try saveEmbedFile(tarball_name, tmp.dir); |
| 1920 | const tarball_path = try std.fmt.allocPrint(gpa, "zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| 1921 | defer gpa.free(tarball_path); |
| 1922 | |
| 1923 | // Run tarball fetch, expect to fail |
| 1924 | var fb: TestFetchBuilder = undefined; |
| 1925 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); |
| 1926 | defer fb.deinit(); |
| 1927 | try std.testing.expectError(error.FetchFailed, fetch.run()); |
| 1928 | |
| 1929 | try fb.expectFetchErrors(1, |
| 1930 | \\error: unable to unpack tarball |
| 1931 | \\ note: unable to create file 'dir1/file1': PathAlreadyExists |
| 1932 | \\ |
| 1933 | ); |
| 1934 | } |
| 1935 | |
| 1936 | test "tarball with excluded duplicate paths" { |
| 1937 | // Same as previous tarball but has build.zig.zon wich excludes 'dir1'. |
| 1938 | // |
| 1939 | // .paths = .{ |
| 1940 | // "build.zig", |
| 1941 | // "build.zig.zon", |
| 1942 | // "src", |
| 1943 | // } |
| 1944 | // |
| 1945 | |
| 1946 | const gpa = std.testing.allocator; |
| 1947 | var tmp = std.testing.tmpDir(.{}); |
| 1948 | defer tmp.cleanup(); |
| 1949 | |
| 1950 | const tarball_name = "duplicate_paths_excluded.tar.gz"; |
| 1951 | try saveEmbedFile(tarball_name, tmp.dir); |
| 1952 | const tarball_path = try std.fmt.allocPrint(gpa, "zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| 1953 | defer gpa.free(tarball_path); |
| 1954 | |
| 1955 | // Run tarball fetch, should succeed |
| 1956 | var fb: TestFetchBuilder = undefined; |
| 1957 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); |
| 1958 | defer fb.deinit(); |
| 1959 | try fetch.run(); |
| 1960 | |
| 1961 | const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash); |
| 1962 | try std.testing.expectEqualStrings( |
| 1963 | "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da", |
| 1964 | &hex_digest, |
| 1965 | ); |
| 1966 | |
| 1967 | const expected_files: []const []const u8 = &.{ |
| 1968 | "build.zig", |
| 1969 | "build.zig.zon", |
| 1970 | "src/main.zig", |
| 1971 | "src/root.zig", |
| 1972 | }; |
| 1973 | try fb.expectPackageFiles(expected_files); |
| 1974 | } |
| 1975 | |
| 1976 | test "tarball without root folder" { |
| 1977 | // Tarball with root folder. Manifest excludes dir1 and dir2. |
| 1978 | // |
| 1979 | // build.zig |
| 1980 | // build.zig.zon |
| 1981 | // dir1/ |
| 1982 | // dir1/file2 |
| 1983 | // dir1/file1 |
| 1984 | // dir2/ |
| 1985 | // dir2/file2 |
| 1986 | // src/ |
| 1987 | // src/main.zig |
| 1988 | // |
| 1989 | |
| 1990 | const gpa = std.testing.allocator; |
| 1991 | var tmp = std.testing.tmpDir(.{}); |
| 1992 | defer tmp.cleanup(); |
| 1993 | |
| 1994 | const tarball_name = "no_root.tar.gz"; |
| 1995 | try saveEmbedFile(tarball_name, tmp.dir); |
| 1996 | const tarball_path = try std.fmt.allocPrint(gpa, "zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| 1997 | defer gpa.free(tarball_path); |
| 1998 | |
| 1999 | // Run tarball fetch, should succeed |
| 2000 | var fb: TestFetchBuilder = undefined; |
| 2001 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); |
| 2002 | defer fb.deinit(); |
| 2003 | try fetch.run(); |
| 2004 | |
| 2005 | const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash); |
| 2006 | try std.testing.expectEqualStrings( |
| 2007 | "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793", |
| 2008 | &hex_digest, |
| 2009 | ); |
| 2010 | |
| 2011 | const expected_files: []const []const u8 = &.{ |
| 2012 | "build.zig", |
| 2013 | "build.zig.zon", |
| 2014 | "src/main.zig", |
| 2015 | }; |
| 2016 | try fb.expectPackageFiles(expected_files); |
| 2017 | } |
| 2018 | |
| 2019 | test "set executable bit based on file content" { |
| 2020 | if (!std.fs.has_executable_bit) return error.SkipZigTest; |
| 2021 | const gpa = std.testing.allocator; |
| 2022 | var tmp = std.testing.tmpDir(.{}); |
| 2023 | defer tmp.cleanup(); |
| 2024 | |
| 2025 | const tarball_name = "executables.tar.gz"; |
| 2026 | try saveEmbedFile(tarball_name, tmp.dir); |
| 2027 | const tarball_path = try std.fmt.allocPrint(gpa, "zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| 2028 | defer gpa.free(tarball_path); |
| 2029 | |
| 2030 | // $ tar -tvf executables.tar.gz |
| 2031 | // drwxrwxr-x 0 executables/ |
| 2032 | // -rwxrwxr-x 170 executables/hello |
| 2033 | // lrwxrwxrwx 0 executables/hello_ln -> hello |
| 2034 | // -rw-rw-r-- 0 executables/file1 |
| 2035 | // -rw-rw-r-- 17 executables/script_with_shebang_without_exec_bit |
| 2036 | // -rwxrwxr-x 7 executables/script_without_shebang |
| 2037 | // -rwxrwxr-x 17 executables/script |
| 2038 | |
| 2039 | var fb: TestFetchBuilder = undefined; |
| 2040 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); |
| 2041 | defer fb.deinit(); |
| 2042 | |
| 2043 | try fetch.run(); |
| 2044 | try std.testing.expectEqualStrings( |
| 2045 | "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3", |
| 2046 | &Manifest.hexDigest(fetch.actual_hash), |
| 2047 | ); |
| 2048 | |
| 2049 | var out = try fb.packageDir(); |
| 2050 | defer out.close(); |
| 2051 | const S = std.posix.S; |
| 2052 | // expect executable bit not set |
| 2053 | try std.testing.expect((try out.statFile("file1")).mode & S.IXUSR == 0); |
| 2054 | try std.testing.expect((try out.statFile("script_without_shebang")).mode & S.IXUSR == 0); |
| 2055 | // expect executable bit set |
| 2056 | try std.testing.expect((try out.statFile("hello")).mode & S.IXUSR != 0); |
| 2057 | try std.testing.expect((try out.statFile("script")).mode & S.IXUSR != 0); |
| 2058 | try std.testing.expect((try out.statFile("script_with_shebang_without_exec_bit")).mode & S.IXUSR != 0); |
| 2059 | try std.testing.expect((try out.statFile("hello_ln")).mode & S.IXUSR != 0); |
| 2060 | |
| 2061 | // |
| 2062 | // $ ls -al zig-cache/tmp/OCz9ovUcstDjTC_U/zig-global-cache/p/1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3 |
| 2063 | // -rw-rw-r-- 1 0 Apr file1 |
| 2064 | // -rwxrwxr-x 1 170 Apr hello |
| 2065 | // lrwxrwxrwx 1 5 Apr hello_ln -> hello |
| 2066 | // -rwxrwxr-x 1 17 Apr script |
| 2067 | // -rw-rw-r-- 1 7 Apr script_without_shebang |
| 2068 | // -rwxrwxr-x 1 17 Apr script_with_shebang_without_exec_bit |
| 2069 | } |
| 2070 | |
| 2071 | fn saveEmbedFile(comptime tarball_name: []const u8, dir: fs.Dir) !void { |
| 2072 | //const tarball_name = "duplicate_paths_excluded.tar.gz"; |
| 2073 | const tarball_content = @embedFile("Fetch/testdata/" ++ tarball_name); |
| 2074 | var tmp_file = try dir.createFile(tarball_name, .{}); |
| 2075 | defer tmp_file.close(); |
| 2076 | try tmp_file.writeAll(tarball_content); |
| 2077 | } |
| 2078 | |
| 2079 | // Builds Fetch with required dependencies, clears dependencies on deinit(). |
| 2080 | const TestFetchBuilder = struct { |
| 2081 | thread_pool: ThreadPool, |
| 2082 | http_client: std.http.Client, |
| 2083 | global_cache_directory: Cache.Directory, |
| 2084 | progress: std.Progress, |
| 2085 | job_queue: Fetch.JobQueue, |
| 2086 | fetch: Fetch, |
| 2087 | |
| 2088 | fn build( |
| 2089 | self: *TestFetchBuilder, |
| 2090 | allocator: std.mem.Allocator, |
| 2091 | cache_parent_dir: std.fs.Dir, |
| 2092 | path_or_url: []const u8, |
| 2093 | ) !*Fetch { |
| 2094 | const cache_dir = try cache_parent_dir.makeOpenPath("zig-global-cache", .{}); |
| 2095 | |
| 2096 | try self.thread_pool.init(.{ .allocator = allocator }); |
| 2097 | self.http_client = .{ .allocator = allocator }; |
| 2098 | self.global_cache_directory = .{ .handle = cache_dir, .path = null }; |
| 2099 | |
| 2100 | self.progress = .{ .dont_print_on_dumb = true }; |
| 2101 | |
| 2102 | self.job_queue = .{ |
| 2103 | .http_client = &self.http_client, |
| 2104 | .thread_pool = &self.thread_pool, |
| 2105 | .global_cache = self.global_cache_directory, |
| 2106 | .recursive = false, |
| 2107 | .read_only = false, |
| 2108 | .debug_hash = false, |
| 2109 | .work_around_btrfs_bug = false, |
| 2110 | }; |
| 2111 | |
| 2112 | self.fetch = .{ |
| 2113 | .arena = std.heap.ArenaAllocator.init(allocator), |
| 2114 | .location = .{ .path_or_url = path_or_url }, |
| 2115 | .location_tok = 0, |
| 2116 | .hash_tok = 0, |
| 2117 | .name_tok = 0, |
| 2118 | .lazy_status = .eager, |
| 2119 | .parent_package_root = Cache.Path{ .root_dir = Cache.Directory{ .handle = cache_dir, .path = null } }, |
| 2120 | .parent_manifest_ast = null, |
| 2121 | .prog_node = self.progress.start("Fetch", 0), |
| 2122 | .job_queue = &self.job_queue, |
| 2123 | .omit_missing_hash_error = true, |
| 2124 | .allow_missing_paths_field = false, |
| 2125 | |
| 2126 | .package_root = undefined, |
| 2127 | .error_bundle = undefined, |
| 2128 | .manifest = null, |
| 2129 | .manifest_ast = undefined, |
| 2130 | .actual_hash = undefined, |
| 2131 | .has_build_zig = false, |
| 2132 | .oom_flag = false, |
| 2133 | .module = null, |
| 2134 | }; |
| 2135 | return &self.fetch; |
| 2136 | } |
| 2137 | |
| 2138 | fn deinit(self: *TestFetchBuilder) void { |
| 2139 | self.fetch.deinit(); |
| 2140 | self.job_queue.deinit(); |
| 2141 | self.fetch.prog_node.end(); |
| 2142 | self.global_cache_directory.handle.close(); |
| 2143 | self.http_client.deinit(); |
| 2144 | self.thread_pool.deinit(); |
| 2145 | } |
| 2146 | |
| 2147 | fn packageDir(self: *TestFetchBuilder) !fs.Dir { |
| 2148 | const root = self.fetch.package_root; |
| 2149 | return try root.root_dir.handle.openDir(root.sub_path, .{ .iterate = true }); |
| 2150 | } |
| 2151 | |
| 2152 | // Test helper, asserts thet package dir constains expected_files. |
| 2153 | // expected_files must be sorted. |
| 2154 | fn expectPackageFiles(self: *TestFetchBuilder, expected_files: []const []const u8) !void { |
| 2155 | var package_dir = try self.packageDir(); |
| 2156 | defer package_dir.close(); |
| 2157 | |
| 2158 | var actual_files: std.ArrayListUnmanaged([]u8) = .{}; |
| 2159 | defer actual_files.deinit(std.testing.allocator); |
| 2160 | defer for (actual_files.items) |file| std.testing.allocator.free(file); |
| 2161 | var walker = try package_dir.walk(std.testing.allocator); |
| 2162 | defer walker.deinit(); |
| 2163 | while (try walker.next()) |entry| { |
| 2164 | if (entry.kind != .file) continue; |
| 2165 | const path = try std.testing.allocator.dupe(u8, entry.path); |
| 2166 | errdefer std.testing.allocator.free(path); |
| 2167 | std.mem.replaceScalar(u8, path, std.fs.path.sep, '/'); |
| 2168 | try actual_files.append(std.testing.allocator, path); |
| 2169 | } |
| 2170 | std.mem.sortUnstable([]u8, actual_files.items, {}, struct { |
| 2171 | fn lessThan(_: void, a: []u8, b: []u8) bool { |
| 2172 | return std.mem.lessThan(u8, a, b); |
| 2173 | } |
| 2174 | }.lessThan); |
| 2175 | |
| 2176 | try std.testing.expectEqual(expected_files.len, actual_files.items.len); |
| 2177 | for (expected_files, 0..) |file_name, i| { |
| 2178 | try std.testing.expectEqualStrings(file_name, actual_files.items[i]); |
| 2179 | } |
| 2180 | try std.testing.expectEqualDeep(expected_files, actual_files.items); |
| 2181 | } |
| 2182 | |
| 2183 | // Test helper, asserts that fetch has failed with `msg` error message. |
| 2184 | fn expectFetchErrors(self: *TestFetchBuilder, notes_len: usize, msg: []const u8) !void { |
| 2185 | var errors = try self.fetch.error_bundle.toOwnedBundle(""); |
| 2186 | defer errors.deinit(std.testing.allocator); |
| 2187 | |
| 2188 | const em = errors.getErrorMessage(errors.getMessages()[0]); |
| 2189 | try std.testing.expectEqual(1, em.count); |
| 2190 | if (notes_len > 0) { |
| 2191 | try std.testing.expectEqual(notes_len, em.notes_len); |
| 2192 | } |
| 2193 | var al = std.ArrayList(u8).init(std.testing.allocator); |
| 2194 | defer al.deinit(); |
| 2195 | try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); |
| 2196 | try std.testing.expectEqualStrings(msg, al.items); |
| 2197 | } |
| 2198 | }; |
Andrew Kelley <andrew@ziglang.org> 2024-04-09 13:22:48-07:00
GitHub <noreply@github.com> 2024-04-09 13:22:48-07:00
Signed by PGP key B5690EEEBB952194
lib