| ... | ... | @@ -463,10 +463,6 @@ fn runResource( |
| 463 | 463 | |
| 464 | 464 | var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); |
| 465 | 465 | defer unpack_result.deinit(); |
| 466 | | if (unpack_result.hasErrors()) { |
| 467 | | try unpack_result.bundleErrors(eb, try f.srcLoc(f.location_tok)); |
| 468 | | return error.FetchFailed; |
| 469 | | } |
| 470 | 466 | |
| 471 | 467 | var pkg_path: Cache.Path = .{ |
| 472 | 468 | .root_dir = tmp_directory, |
| ... | ... | @@ -491,10 +487,14 @@ fn runResource( |
| 491 | 487 | .include_paths = if (f.manifest) |m| m.paths else .{}, |
| 492 | 488 | }; |
| 493 | 489 | |
| 494 | | // TODO: |
| 495 | 490 | // If any error occurred for files that were ultimately excluded, those |
| 496 | 491 | // errors should be ignored, such as failure to create symlinks that |
| 497 | 492 | // weren't supposed to be included anyway. |
| 493 | try unpack_result.filterErrors(filter); |
| 494 | if (unpack_result.hasErrors()) { |
| 495 | try unpack_result.bundleErrors(eb, try f.srcLoc(f.location_tok)); |
| 496 | return error.FetchFailed; |
| 497 | } |
| 498 | 498 | |
| 499 | 499 | // Apply the manifest's inclusion rules to the temporary directory by |
| 500 | 500 | // deleting excluded files. |
| ... | ... | @@ -1701,7 +1701,7 @@ const ThreadPool = std.Thread.Pool; |
| 1701 | 1701 | const WaitGroup = std.Thread.WaitGroup; |
| 1702 | 1702 | const Fetch = @This(); |
| 1703 | 1703 | const git = @import("Fetch/git.zig"); |
| 1704 | | //const Package = @import("../Package.zig"); |
| 1704 | const Package = @import("../Package.zig"); |
| 1705 | 1705 | const Manifest = Package.Manifest; |
| 1706 | 1706 | const ErrorBundle = std.zig.ErrorBundle; |
| 1707 | 1707 | const native_os = builtin.os.tag; |
| ... | ... | @@ -1766,12 +1766,14 @@ const UnpackResult = struct { |
| 1766 | 1766 | file_type: u8, |
| 1767 | 1767 | }, |
| 1768 | 1768 | |
| 1769 | | fn excluded(self: Error, filter: Filter) bool { |
| 1770 | | switch (self) { |
| 1771 | | .unable_to_create_file => |info| return !filter.includePath(info.file_name), |
| 1772 | | .unable_to_create_sym_link => |info| return !filter.includePath(info.file_name), |
| 1773 | | .unsupported_file_type => |info| return !filter.includePath(info.file_name), |
| 1774 | | } |
| 1769 | fn excluded(self: Error, filter: Filter, root_dir: []const u8) bool { |
| 1770 | const file_name = switch (self) { |
| 1771 | .unable_to_create_file => |info| info.file_name, |
| 1772 | .unable_to_create_sym_link => |info| info.file_name, |
| 1773 | .unsupported_file_type => |info| info.file_name, |
| 1774 | }; |
| 1775 | |
| 1776 | return !filter.includePath(stripRoot(file_name, root_dir)); |
| 1775 | 1777 | } |
| 1776 | 1778 | |
| 1777 | 1779 | fn free(self: Error, allocator: std.mem.Allocator) void { |
| ... | ... | @@ -1834,10 +1836,11 @@ const UnpackResult = struct { |
| 1834 | 1836 | |
| 1835 | 1837 | fn filterErrors(self: *UnpackResult, filter: Filter) !void { |
| 1836 | 1838 | var i = self.errors.items.len; |
| 1839 | const root_dir: []const u8 = if (self.root_dir) |root_dir| root_dir else ""; |
| 1837 | 1840 | while (i > 0) { |
| 1838 | 1841 | i -= 1; |
| 1839 | 1842 | const item = self.errors.items[i]; |
| 1840 | | if (item.excluded(filter)) { |
| 1843 | if (item.excluded(filter, root_dir)) { |
| 1841 | 1844 | _ = self.errors.swapRemove(i); |
| 1842 | 1845 | item.free(self.allocator); |
| 1843 | 1846 | } |
| ... | ... | @@ -1888,132 +1891,236 @@ const UnpackResult = struct { |
| 1888 | 1891 | } |
| 1889 | 1892 | }; |
| 1890 | 1893 | |
| 1891 | | // Removing dependencies |
| 1892 | | const Package = struct { |
| 1893 | | const build_zig_basename = "build.zig"; |
| 1894 | | const Module = struct {}; |
| 1895 | | const Manifest = @import("Manifest.zig"); |
| 1896 | | }; |
| 1897 | | |
| 1898 | | test "fetch tarball" { |
| 1894 | test "fetch tarball: fail with unable to create file" { |
| 1899 | 1895 | const testing = std.testing; |
| 1900 | | const gpa = testing.allocator; |
| 1896 | var buf: [4096]u8 = undefined; |
| 1897 | var buf_pos: usize = 0; |
| 1901 | 1898 | |
| 1902 | | var cache_tmp = std.testing.tmpDir(.{}); |
| 1903 | | defer cache_tmp.cleanup(); |
| 1904 | | const global_cache_directory_path = try cache_tmp.dir.realpathAlloc(gpa, "."); |
| 1905 | | defer gpa.free(global_cache_directory_path); |
| 1906 | | |
| 1907 | | const paths: []const []const u8 = &.{ |
| 1908 | | "main.zig", |
| 1909 | | "src/root.zig", |
| 1910 | | // duplicate file paths |
| 1911 | | "dir/file", |
| 1912 | | "dir1/file1", |
| 1913 | | "dir/file", |
| 1914 | | "dir1/file1", |
| 1915 | | }; |
| 1899 | // Create tmp dir |
| 1916 | 1900 | var tmp = std.testing.tmpDir(.{}); |
| 1917 | 1901 | defer tmp.cleanup(); |
| 1918 | | const file = try tmp.dir.createFile("package.tar", .{}); |
| 1919 | | try createTarball(file, "package", paths); |
| 1920 | | file.close(); |
| 1921 | | |
| 1922 | | const tmp_path = try tmp.dir.realpathAlloc(gpa, "."); |
| 1923 | | const path_or_url = try std.fmt.allocPrint( |
| 1924 | | gpa, |
| 1925 | | "file://{s}/package.tar", |
| 1926 | | .{tmp_path}, |
| 1927 | | ); |
| 1928 | | gpa.free(tmp_path); |
| 1929 | | defer gpa.free(path_or_url); |
| 1902 | const tmp_path = try tmp.dir.realpath(".", &buf); |
| 1903 | buf_pos += tmp_path.len; |
| 1930 | 1904 | |
| 1931 | | var thread_pool: ThreadPool = undefined; |
| 1932 | | try thread_pool.init(.{ .allocator = gpa }); |
| 1933 | | defer thread_pool.deinit(); |
| 1905 | // Create tarball in tmp dir without build.zig.zon |
| 1906 | const tarball_name = "package.tar"; |
| 1907 | try createTestTarball(tmp.dir, tarball_name, false); |
| 1934 | 1908 | |
| 1935 | | var http_client: std.http.Client = .{ .allocator = gpa }; |
| 1936 | | defer http_client.deinit(); |
| 1909 | // Get path to the tarball |
| 1910 | const path_or_url = try std.fmt.bufPrint(buf[buf_pos..], "file://{s}/{s}", .{ tmp_path, tarball_name }); |
| 1911 | buf_pos += path_or_url.len; |
| 1937 | 1912 | |
| 1938 | | var global_cache_directory: Cache.Directory = .{ |
| 1939 | | .handle = try fs.cwd().makeOpenPath(global_cache_directory_path, .{}), |
| 1940 | | .path = global_cache_directory_path, |
| 1941 | | }; |
| 1942 | | defer global_cache_directory.handle.close(); |
| 1943 | | |
| 1944 | | var progress: std.Progress = .{ .dont_print_on_dumb = true }; |
| 1945 | | const root_prog_node = progress.start("Fetch", 0); |
| 1946 | | defer root_prog_node.end(); |
| 1947 | | |
| 1948 | | var job_queue: Fetch.JobQueue = .{ |
| 1949 | | .http_client = &http_client, |
| 1950 | | .thread_pool = &thread_pool, |
| 1951 | | .global_cache = global_cache_directory, |
| 1952 | | .recursive = false, |
| 1953 | | .read_only = false, |
| 1954 | | .debug_hash = true, |
| 1955 | | .work_around_btrfs_bug = false, |
| 1956 | | }; |
| 1957 | | defer job_queue.deinit(); |
| 1958 | | |
| 1959 | | var fetch: Fetch = .{ |
| 1960 | | .arena = std.heap.ArenaAllocator.init(gpa), |
| 1961 | | .location = .{ .path_or_url = path_or_url }, |
| 1962 | | .location_tok = 0, |
| 1963 | | .hash_tok = 0, |
| 1964 | | .name_tok = 0, |
| 1965 | | .lazy_status = .eager, |
| 1966 | | .parent_package_root = Cache.Path{ .root_dir = undefined }, |
| 1967 | | .parent_manifest_ast = null, |
| 1968 | | .prog_node = root_prog_node, |
| 1969 | | .job_queue = &job_queue, |
| 1970 | | .omit_missing_hash_error = true, |
| 1971 | | .allow_missing_paths_field = false, |
| 1972 | | |
| 1973 | | .package_root = undefined, |
| 1974 | | .error_bundle = undefined, |
| 1975 | | .manifest = null, |
| 1976 | | .manifest_ast = undefined, |
| 1977 | | .actual_hash = undefined, |
| 1978 | | .has_build_zig = false, |
| 1979 | | .oom_flag = false, |
| 1980 | | |
| 1981 | | .module = null, |
| 1913 | // Global cache directory in tmp |
| 1914 | const cache_path = try std.fmt.bufPrint(buf[buf_pos..], "{s}/{s}", .{ tmp_path, "global_cache" }); |
| 1915 | buf_pos += cache_path.len; |
| 1916 | |
| 1917 | // Run tarball fetch, expect to fail |
| 1918 | var tf: TestFetch = undefined; |
| 1919 | try tf.init(testing.allocator, cache_path, path_or_url); |
| 1920 | defer tf.deinit(); |
| 1921 | try testing.expectError(error.FetchFailed, tf.fetch.run()); |
| 1922 | |
| 1923 | // Expect fetch errors |
| 1924 | { |
| 1925 | var errors = try tf.fetch.error_bundle.toOwnedBundle(""); |
| 1926 | defer errors.deinit(testing.allocator); |
| 1927 | |
| 1928 | const em = errors.getErrorMessage(errors.getMessages()[0]); |
| 1929 | try testing.expectEqual(1, em.count); |
| 1930 | try testing.expectEqual(2, em.notes_len); |
| 1931 | |
| 1932 | var al = std.ArrayList(u8).init(testing.allocator); |
| 1933 | defer al.deinit(); |
| 1934 | try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); |
| 1935 | try testing.expectEqualStrings( |
| 1936 | \\error: unable to unpack tarball |
| 1937 | \\ note: unable to create file 'dir/file': PathAlreadyExists |
| 1938 | \\ note: unable to create file 'dir1/file1': PathAlreadyExists |
| 1939 | \\ |
| 1940 | , al.items); |
| 1941 | } |
| 1942 | } |
| 1943 | |
| 1944 | test "fetch tarball: error path are excluded" { |
| 1945 | const testing = std.testing; |
| 1946 | var buf: [4096]u8 = undefined; |
| 1947 | var buf_pos: usize = 0; |
| 1948 | |
| 1949 | // Create tmp dir |
| 1950 | var tmp = std.testing.tmpDir(.{}); |
| 1951 | defer tmp.cleanup(); |
| 1952 | const tmp_path = try tmp.dir.realpath(".", &buf); |
| 1953 | buf_pos += tmp_path.len; |
| 1954 | |
| 1955 | // Create tarball in tmp dir |
| 1956 | const tarball_name = "package.tar"; |
| 1957 | try createTestTarball(tmp.dir, tarball_name, true); |
| 1958 | |
| 1959 | // Get path to the tarball |
| 1960 | const path_or_url = try std.fmt.bufPrint(buf[buf_pos..], "file://{s}/{s}", .{ tmp_path, tarball_name }); |
| 1961 | buf_pos += path_or_url.len; |
| 1962 | |
| 1963 | // Global cache directory in tmp |
| 1964 | const cache_path = try std.fmt.bufPrint(buf[buf_pos..], "{s}/{s}", .{ tmp_path, "global_cache" }); |
| 1965 | buf_pos += cache_path.len; |
| 1966 | |
| 1967 | // Run tarball fetch |
| 1968 | var tf: TestFetch = undefined; |
| 1969 | try tf.init(testing.allocator, cache_path, path_or_url); |
| 1970 | defer tf.deinit(); |
| 1971 | try tf.fetch.run(); |
| 1972 | |
| 1973 | const hex_digest = Package.Manifest.hexDigest(tf.fetch.actual_hash); |
| 1974 | try testing.expectEqualStrings("122022afac878639d5ea6fcca14a123e21fd0395c1f2ef2c89017fa71390f73024af", &hex_digest); |
| 1975 | |
| 1976 | const expected_files: []const []const u8 = &.{ |
| 1977 | "build.zig", |
| 1978 | "build.zig.zon", |
| 1979 | "src/main.zig", |
| 1982 | 1980 | }; |
| 1983 | | defer fetch.deinit(); |
| 1984 | | |
| 1985 | | try testing.expectError(error.FetchFailed, fetch.run()); |
| 1986 | | |
| 1987 | | try testing.expectEqual(1, fetch.error_bundle.root_list.items.len); |
| 1988 | | var errors = try fetch.error_bundle.toOwnedBundle(""); |
| 1989 | | defer errors.deinit(gpa); |
| 1990 | | |
| 1991 | | const em = errors.getErrorMessage(errors.getMessages()[0]); |
| 1992 | | try testing.expectEqual(2, em.notes_len); |
| 1993 | | |
| 1994 | | var al = std.ArrayList(u8).init(gpa); |
| 1995 | | defer al.deinit(); |
| 1996 | | try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); |
| 1997 | | try testing.expectEqualStrings( |
| 1998 | | \\error: unable to unpack tarball |
| 1999 | | \\ note: unable to create file 'dir/file': PathAlreadyExists |
| 2000 | | \\ note: unable to create file 'dir1/file1': PathAlreadyExists |
| 2001 | | \\ |
| 2002 | | , al.items); |
| 1981 | // Unpacked package contains expected files |
| 1982 | { |
| 1983 | const package_path = try std.fmt.bufPrint(buf[buf_pos..], "global_cache/p/{s}", .{hex_digest}); |
| 1984 | buf_pos += package_path.len; |
| 1985 | var package_dir = try tmp.dir.openDir(package_path, .{ .iterate = true }); |
| 1986 | |
| 1987 | var actual_files: std.ArrayListUnmanaged([]u8) = .{}; |
| 1988 | defer actual_files.deinit(testing.allocator); |
| 1989 | defer for (actual_files.items) |file| testing.allocator.free(file); |
| 1990 | var walker = try package_dir.walk(testing.allocator); |
| 1991 | defer walker.deinit(); |
| 1992 | while (try walker.next()) |entry| { |
| 1993 | if (entry.kind != .file) continue; |
| 1994 | //std.debug.print("{s}\n", .{entry.path}); |
| 1995 | const path = try testing.allocator.dupe(u8, entry.path); |
| 1996 | errdefer testing.allocator.free(path); |
| 1997 | std.mem.replaceScalar(u8, path, std.fs.path.sep, '/'); |
| 1998 | try actual_files.append(testing.allocator, path); |
| 1999 | } |
| 2000 | std.mem.sortUnstable([]u8, actual_files.items, {}, struct { |
| 2001 | fn lessThan(_: void, a: []u8, b: []u8) bool { |
| 2002 | return std.mem.lessThan(u8, a, b); |
| 2003 | } |
| 2004 | }.lessThan); |
| 2005 | try testing.expectEqualDeep(expected_files, actual_files.items); |
| 2006 | } |
| 2003 | 2007 | } |
| 2004 | 2008 | |
| 2005 | | const TarHeader = std.tar.output.Header; |
| 2009 | const TestFetch = struct { |
| 2010 | thread_pool: ThreadPool, |
| 2011 | http_client: std.http.Client, |
| 2012 | global_cache_directory: Cache.Directory, |
| 2013 | progress: std.Progress, |
| 2014 | root_prog_node: *std.Progress.Node, |
| 2015 | job_queue: Fetch.JobQueue, |
| 2016 | fetch: Fetch, |
| 2017 | gpa: std.mem.Allocator, |
| 2018 | |
| 2019 | fn init( |
| 2020 | tf: *TestFetch, |
| 2021 | gpa: std.mem.Allocator, |
| 2022 | global_cache_directory_path: []const u8, |
| 2023 | path_or_url: []const u8, |
| 2024 | ) !void { |
| 2025 | try tf.thread_pool.init(.{ .allocator = gpa }); |
| 2026 | tf.http_client = .{ .allocator = gpa }; |
| 2027 | tf.global_cache_directory = .{ |
| 2028 | .handle = try fs.cwd().makeOpenPath(global_cache_directory_path, .{}), |
| 2029 | .path = global_cache_directory_path, |
| 2030 | }; |
| 2031 | |
| 2032 | tf.progress = .{ .dont_print_on_dumb = true }; |
| 2033 | tf.root_prog_node = tf.progress.start("Fetch", 0); |
| 2034 | |
| 2035 | tf.job_queue = .{ |
| 2036 | .http_client = &tf.http_client, |
| 2037 | .thread_pool = &tf.thread_pool, |
| 2038 | .global_cache = tf.global_cache_directory, |
| 2039 | .recursive = false, |
| 2040 | .read_only = false, |
| 2041 | .debug_hash = false, |
| 2042 | .work_around_btrfs_bug = false, |
| 2043 | }; |
| 2044 | |
| 2045 | tf.fetch = .{ |
| 2046 | .arena = std.heap.ArenaAllocator.init(gpa), |
| 2047 | .location = .{ .path_or_url = path_or_url }, |
| 2048 | .location_tok = 0, |
| 2049 | .hash_tok = 0, |
| 2050 | .name_tok = 0, |
| 2051 | .lazy_status = .eager, |
| 2052 | .parent_package_root = Cache.Path{ .root_dir = undefined }, |
| 2053 | .parent_manifest_ast = null, |
| 2054 | .prog_node = tf.root_prog_node, |
| 2055 | .job_queue = &tf.job_queue, |
| 2056 | .omit_missing_hash_error = true, |
| 2057 | .allow_missing_paths_field = false, |
| 2058 | |
| 2059 | .package_root = undefined, |
| 2060 | .error_bundle = undefined, |
| 2061 | .manifest = null, |
| 2062 | .manifest_ast = undefined, |
| 2063 | .actual_hash = undefined, |
| 2064 | .has_build_zig = false, |
| 2065 | .oom_flag = false, |
| 2066 | |
| 2067 | .module = null, |
| 2068 | }; |
| 2069 | } |
| 2006 | 2070 | |
| 2007 | | fn createTarball(file: fs.File, prefix: []const u8, paths: []const []const u8) !void { |
| 2008 | | for (paths) |path| { |
| 2071 | fn deinit(self: *TestFetch) void { |
| 2072 | self.fetch.deinit(); |
| 2073 | self.job_queue.deinit(); |
| 2074 | self.root_prog_node.end(); |
| 2075 | self.global_cache_directory.handle.close(); |
| 2076 | self.http_client.deinit(); |
| 2077 | self.thread_pool.deinit(); |
| 2078 | } |
| 2079 | }; |
| 2080 | |
| 2081 | fn createTestTarball(dir: fs.Dir, tarball_name: []const u8, with_manifest: bool) !void { |
| 2082 | const file = try dir.createFile(tarball_name, .{}); |
| 2083 | defer file.close(); |
| 2084 | |
| 2085 | const TarHeader = std.tar.output.Header; |
| 2086 | const prefix = tarball_name; |
| 2087 | |
| 2088 | const files: []const []const u8 = &.{ |
| 2089 | "build.zig", |
| 2090 | "src/main.zig", |
| 2091 | // duplicate file paths |
| 2092 | "dir/file", |
| 2093 | "dir1/file1", |
| 2094 | "dir/file", |
| 2095 | "dir1/file1", |
| 2096 | }; |
| 2097 | for (files) |path| { |
| 2009 | 2098 | var hdr = TarHeader.init(); |
| 2010 | 2099 | hdr.typeflag = .regular; |
| 2011 | | //if (prefix.len > 0) { |
| 2012 | 2100 | try hdr.setPath(prefix, path); |
| 2013 | | // } else { |
| 2014 | | // hdr.setName(path); |
| 2015 | | // } |
| 2016 | 2101 | try hdr.updateChecksum(); |
| 2017 | 2102 | try file.writeAll(std.mem.asBytes(&hdr)); |
| 2018 | 2103 | } |
| 2104 | |
| 2105 | if (with_manifest) { |
| 2106 | const build_zig_zon = |
| 2107 | \\ .{ |
| 2108 | \\ .name = "fetch", |
| 2109 | \\ .version = "0.0.0", |
| 2110 | \\ .paths = .{ |
| 2111 | \\ "src", |
| 2112 | \\ "build.zig", |
| 2113 | \\ "build.zig.zon" |
| 2114 | \\ }, |
| 2115 | \\ } |
| 2116 | ; |
| 2117 | var hdr = TarHeader.init(); |
| 2118 | hdr.typeflag = .regular; |
| 2119 | try hdr.setPath(prefix, "build.zig.zon"); |
| 2120 | try hdr.setSize(build_zig_zon.len); |
| 2121 | try hdr.updateChecksum(); |
| 2122 | try file.writeAll(std.mem.asBytes(&hdr)); |
| 2123 | try file.writeAll(build_zig_zon); |
| 2124 | try file.writeAll(&[_]u8{0} ** (512 - build_zig_zon.len)); |
| 2125 | } |
| 2019 | 2126 | } |
Igor Anić <igor.anic@gmail.com> 2024-03-29 01:25:57+01:00
src