diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f3337589ce71b5036d7eab8b50df6f929a9367f..b08dca6f04c01f7ed277ac3c91c3d29378e57efd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -528,7 +528,7 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/src/Liveness.zig" "${CMAKE_SOURCE_DIR}/src/Module.zig" "${CMAKE_SOURCE_DIR}/src/Package.zig" - "${CMAKE_SOURCE_DIR}/src/Package/hash.zig" + "${CMAKE_SOURCE_DIR}/src/Package/Fetch.zig" "${CMAKE_SOURCE_DIR}/src/RangeSet.zig" "${CMAKE_SOURCE_DIR}/src/Sema.zig" "${CMAKE_SOURCE_DIR}/src/TypedValue.zig" diff --git a/build.zig b/build.zig index 423f80b12a1b163ba3c28027b16e0761a468a302..6f93585872367377b0c6f8d8b022d093a12e1cc8 100644 --- a/build.zig +++ b/build.zig @@ -88,7 +88,7 @@ pub fn build(b: *std.Build) !void { .name = "check-case", .root_source_file = .{ .path = "test/src/Cases.zig" }, .optimize = optimize, - .main_pkg_path = .{ .path = "." }, + .main_mod_path = .{ .path = "." }, }); check_case_exe.stack_size = stack_size; check_case_exe.single_threaded = single_threaded; diff --git a/doc/build.zig.zon.md b/doc/build.zig.zon.md new file mode 100644 index 0000000000000000000000000000000000000000..7d1af4f678ac0db38de21b55ebc2b7f088b0b5a3 --- /dev/null +++ b/doc/build.zig.zon.md @@ -0,0 +1,65 @@ +# build.zig.zon Documentation + +This is the manifest file for build.zig scripts. It is named build.zig.zon in +order to make it clear that it is metadata specifically pertaining to +build.zig. + +- **build root** - the directory that contains `build.zig` + +## Top-Level Fields + +### `name` + +String. Required. + +### `version` + +String. Required. + +[semver](https://semver.org/) + +### `dependencies` + +Struct. + +Each dependency must either provide a `url` and `hash`, or a `path`. + +#### `url` + +String. + +When updating this field to a new URL, be sure to delete the corresponding +`hash`, otherwise you are communicating that you expect to find the old hash at +the new URL. + +#### `hash` + +String. + +[multihash](https://multiformats.io/multihash/) + +This is computed from the file contents of the directory of files that is +obtained after fetching `url` and applying the inclusion rules given by +`paths`. + +This field is the source of truth; packages do not come from an `url`; they +come from a `hash`. `url` is just one of many possible mirrors for how to +obtain a package matching this `hash`. + +#### `path` + +String. + +When this is provided, the package is found in a directory relative to the +build root. In this case the package's hash is irrelevant and therefore not +computed. + +### `paths` + +List. Required. + +Specifies the set of files and directories that are included in this package. +Paths are relative to the build root. Use the empty string (`""`) to refer to +the build root itself. + +Only files included in the package are used to compute a package's `hash`. diff --git a/lib/build_runner.zig b/lib/build_runner.zig index 45c1f41fd3ade95e15f4a347de10e3777b6061da..48c70646b1664a440a5030a9b6cbc34dcd0138c8 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -997,6 +997,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi \\ -j Limit concurrent jobs (default is to use all CPU cores) \\ --maxrss Limit memory usage (default is to use available memory) \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss + \\ --fetch Exit after fetching dependency tree \\ \\Project-Specific Options: \\ diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 49afd73778251f42d22dbee4c9c35f69d4f14f7f..5c80f4972c25756d9bd126b751a5878019246b39 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -634,6 +634,9 @@ pub const ExecutableOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -652,7 +655,7 @@ pub fn addExecutable(b: *Build, options: ExecutableOptions) *Step.Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -667,6 +670,9 @@ pub const ObjectOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -683,7 +689,7 @@ pub fn addObject(b: *Build, options: ObjectOptions) *Step.Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -699,6 +705,9 @@ pub const SharedLibraryOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -717,7 +726,7 @@ pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *Step.Compile .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -733,6 +742,9 @@ pub const StaticLibraryOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -751,7 +763,7 @@ pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *Step.Compile .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -769,6 +781,9 @@ pub const TestOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -787,7 +802,7 @@ pub fn addTest(b: *Build, options: TestOptions) *Step.Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index f76985a01aef1157dda0db336cc9490f69c108f0..b1ceaa1b09be9c3cc8783ab2b47c39ca1fa5b3b2 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -9,6 +9,20 @@ pub const Directory = struct { path: ?[]const u8, handle: fs.Dir, + pub fn clone(d: Directory, arena: Allocator) Allocator.Error!Directory { + return .{ + .path = if (d.path) |p| try arena.dupe(u8, p) else null, + .handle = d.handle, + }; + } + + pub fn cwd() Directory { + return .{ + .path = null, + .handle = fs.cwd(), + }; + } + pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 { if (self.path) |p| { // TODO clean way to do this with only 1 allocation @@ -47,12 +61,16 @@ pub const Directory = struct { writer: anytype, ) !void { _ = options; - if (fmt_string.len != 0) fmt.invalidFmtError(fmt, self); + if (fmt_string.len != 0) fmt.invalidFmtError(fmt_string, self); if (self.path) |p| { try writer.writeAll(p); try writer.writeAll(fs.path.sep_str); } } + + pub fn eql(self: Directory, other: Directory) bool { + return self.handle.fd == other.handle.fd; + } }; gpa: Allocator, diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 07eb3ded01830282b40692b1b7d84d7375e50967..9e711b6f52ad61222a5ff4e251dd1ac0e0c1104f 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -68,7 +68,7 @@ c_std: std.Build.CStd, /// Set via options; intended to be read-only after that. zig_lib_dir: ?LazyPath, /// Set via options; intended to be read-only after that. -main_pkg_path: ?LazyPath, +main_mod_path: ?LazyPath, exec_cmd_args: ?[]const ?[]const u8, filter: ?[]const u8, test_evented_io: bool = false, @@ -316,6 +316,9 @@ pub const Options = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -480,7 +483,7 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .installed_headers = ArrayList(*Step).init(owner.allocator), .c_std = std.Build.CStd.C99, .zig_lib_dir = null, - .main_pkg_path = null, + .main_mod_path = null, .exec_cmd_args = null, .filter = options.filter, .test_runner = options.test_runner, @@ -515,8 +518,8 @@ pub fn create(owner: *std.Build, options: Options) *Compile { lp.addStepDependencies(&self.step); } - if (options.main_pkg_path) |lp| { - self.main_pkg_path = lp.dupe(self.step.owner); + if (options.main_mod_path orelse options.main_pkg_path) |lp| { + self.main_mod_path = lp.dupe(self.step.owner); lp.addStepDependencies(&self.step); } @@ -1998,8 +2001,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try zig_args.append(dir.getPath(b)); } - if (self.main_pkg_path) |dir| { - try zig_args.append("--main-pkg-path"); + if (self.main_mod_path) |dir| { + try zig_args.append("--main-mod-path"); try zig_args.append(dir.getPath(b)); } diff --git a/lib/std/array_hash_map.zig b/lib/std/array_hash_map.zig index 1bb85a255d7b00d4157b4c0193d959214449cb5e..75a86f63f6c3f8d34401804542d1e4e3e1bf66e9 100644 --- a/lib/std/array_hash_map.zig +++ b/lib/std/array_hash_map.zig @@ -1229,14 +1229,41 @@ pub fn ArrayHashMapUnmanaged( /// Sorts the entries and then rebuilds the index. /// `sort_ctx` must have this method: /// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool` + /// Uses a stable sorting algorithm. pub inline fn sort(self: *Self, sort_ctx: anytype) void { if (@sizeOf(ByIndexContext) != 0) @compileError("Cannot infer context " ++ @typeName(Context) ++ ", call sortContext instead."); - return self.sortContext(sort_ctx, undefined); + return sortContextInternal(self, .stable, sort_ctx, undefined); } - pub fn sortContext(self: *Self, sort_ctx: anytype, ctx: Context) void { - self.entries.sort(sort_ctx); + /// Sorts the entries and then rebuilds the index. + /// `sort_ctx` must have this method: + /// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool` + /// Uses an unstable sorting algorithm. + pub inline fn sortUnstable(self: *Self, sort_ctx: anytype) void { + if (@sizeOf(ByIndexContext) != 0) + @compileError("Cannot infer context " ++ @typeName(Context) ++ ", call sortUnstableContext instead."); + return self.sortContextInternal(.unstable, sort_ctx, undefined); + } + + pub inline fn sortContext(self: *Self, sort_ctx: anytype, ctx: Context) void { + return sortContextInternal(self, .stable, sort_ctx, ctx); + } + + pub inline fn sortUnstableContext(self: *Self, sort_ctx: anytype, ctx: Context) void { + return sortContextInternal(self, .unstable, sort_ctx, ctx); + } + + fn sortContextInternal( + self: *Self, + comptime mode: std.sort.Mode, + sort_ctx: anytype, + ctx: Context, + ) void { + switch (mode) { + .stable => self.entries.sort(sort_ctx), + .unstable => self.entries.sortUnstable(sort_ctx), + } const header = self.index_header orelse return; header.reset(); self.insertAllEntriesIntoNewHeader(if (store_hash) {} else ctx, header); diff --git a/lib/std/fs/path.zig b/lib/std/fs/path.zig index 1f9632b9c1dbfa94bd91d385862e16cb65af2702..8457991f5994dc4a36ad60f8399b70ba78ee73eb 100644 --- a/lib/std/fs/path.zig +++ b/lib/std/fs/path.zig @@ -728,15 +728,17 @@ pub fn resolvePosix(allocator: Allocator, paths: []const []const u8) Allocator.E } } -test "resolve" { +test resolve { try testResolveWindows(&[_][]const u8{ "a\\b\\c\\", "..\\..\\.." }, "."); try testResolveWindows(&[_][]const u8{"."}, "."); + try testResolveWindows(&[_][]const u8{""}, "."); try testResolvePosix(&[_][]const u8{ "a/b/c/", "../../.." }, "."); try testResolvePosix(&[_][]const u8{"."}, "."); + try testResolvePosix(&[_][]const u8{""}, "."); } -test "resolveWindows" { +test resolveWindows { try testResolveWindows( &[_][]const u8{ "Z:\\", "/usr/local", "lib\\zig\\std\\array_list.zig" }, "Z:\\usr\\local\\lib\\zig\\std\\array_list.zig", @@ -764,7 +766,7 @@ test "resolveWindows" { try testResolveWindows(&[_][]const u8{"a/b"}, "a\\b"); } -test "resolvePosix" { +test resolvePosix { try testResolvePosix(&.{ "/a/b", "c" }, "/a/b/c"); try testResolvePosix(&.{ "/a/b", "c", "//d", "e///" }, "/d/e"); try testResolvePosix(&.{ "/a/b/c", "..", "../" }, "/a"); diff --git a/lib/std/multi_array_list.zig b/lib/std/multi_array_list.zig index c18d2c2a5aae64cac6bb8c9a1b1af46fcd4f869a..11dec780362071f097fa7132fddc215d696fe4c7 100644 --- a/lib/std/multi_array_list.zig +++ b/lib/std/multi_array_list.zig @@ -467,7 +467,7 @@ pub fn MultiArrayList(comptime T: type) type { /// `ctx` has the following method: /// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool` - fn sortInternal(self: Self, a: usize, b: usize, ctx: anytype, comptime mode: enum { stable, unstable }) void { + fn sortInternal(self: Self, a: usize, b: usize, ctx: anytype, comptime mode: std.sort.Mode) void { const sort_context: struct { sub_ctx: @TypeOf(ctx), slice: Slice, diff --git a/lib/std/process.zig b/lib/std/process.zig index a7f9d8d9d8a8c31d0600d97fff555329b08df549..518def42159494068d0ac52b9394b2dfcf3179b3 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -46,7 +46,7 @@ pub fn getCwdAlloc(allocator: Allocator) ![]u8 { } } -test "getCwdAlloc" { +test getCwdAlloc { if (builtin.os.tag == .wasi) return error.SkipZigTest; const cwd = try getCwdAlloc(testing.allocator); diff --git a/lib/std/sort.zig b/lib/std/sort.zig index abeae292475d6d51dfe34e87176e2bfea6e2b2b6..e110a8beb8a08b78d7002b64bd5f4d227272c132 100644 --- a/lib/std/sort.zig +++ b/lib/std/sort.zig @@ -4,6 +4,8 @@ const testing = std.testing; const mem = std.mem; const math = std.math; +pub const Mode = enum { stable, unstable }; + pub const block = @import("sort/block.zig").block; pub const pdq = @import("sort/pdq.zig").pdq; pub const pdqContext = @import("sort/pdq.zig").pdqContext; diff --git a/lib/std/tar.zig b/lib/std/tar.zig index b71c118d52b041eb52bab250c9c7e47e59b19e09..bdbec87e39becb00eb47c28bd1b1897aaf169e38 100644 --- a/lib/std/tar.zig +++ b/lib/std/tar.zig @@ -3,6 +3,8 @@ pub const Options = struct { strip_components: u32 = 0, /// How to handle the "mode" property of files from within the tar file. mode_mode: ModeMode = .executable_bit_only, + /// Prevents creation of empty directories. + exclude_empty_directories: bool = false, /// Provide this to receive detailed error messages. /// When this is provided, some errors which would otherwise be returned immediately /// will instead be added to this structure. The API user must check the errors @@ -29,6 +31,10 @@ pub const Options = struct { file_name: []const u8, link_name: []const u8, }, + unable_to_create_file: struct { + code: anyerror, + file_name: []const u8, + }, unsupported_file_type: struct { file_name: []const u8, file_type: Header.FileType, @@ -42,6 +48,9 @@ pub const Options = struct { d.allocator.free(info.file_name); d.allocator.free(info.link_name); }, + .unable_to_create_file => |info| { + d.allocator.free(info.file_name); + }, .unsupported_file_type => |info| { d.allocator.free(info.file_name); }, @@ -201,7 +210,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi switch (header.fileType()) { .directory => { const file_name = try stripComponents(unstripped_file_name, options.strip_components); - if (file_name.len != 0) { + if (file_name.len != 0 and !options.exclude_empty_directories) { try dir.makePath(file_name); } }, @@ -209,18 +218,34 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi if (file_size == 0 and unstripped_file_name.len == 0) return; const file_name = try stripComponents(unstripped_file_name, options.strip_components); - if (std.fs.path.dirname(file_name)) |dir_name| { - try dir.makePath(dir_name); - } - var file = try dir.createFile(file_name, .{}); - defer file.close(); + var file = dir.createFile(file_name, .{}) catch |err| switch (err) { + error.FileNotFound => again: { + const code = code: { + if (std.fs.path.dirname(file_name)) |dir_name| { + dir.makePath(dir_name) catch |code| break :code code; + break :again dir.createFile(file_name, .{}) catch |code| { + break :code code; + }; + } + break :code err; + }; + const d = options.diagnostics orelse return error.UnableToCreateFile; + try d.errors.append(d.allocator, .{ .unable_to_create_file = .{ + .code = code, + .file_name = try d.allocator.dupe(u8, file_name), + } }); + break :again null; + }, + else => |e| return e, + }; + defer if (file) |f| f.close(); var file_off: usize = 0; while (true) { const temp = try buffer.readChunk(reader, @intCast(rounded_file_size + 512 - file_off)); if (temp.len == 0) return error.UnexpectedEndOfStream; const slice = temp[0..@intCast(@min(file_size - file_off, temp.len))]; - try file.writeAll(slice); + if (file) |f| try f.writeAll(slice); file_off += slice.len; buffer.advance(slice.len); @@ -273,13 +298,26 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi }, .hard_link => return error.TarUnsupportedFileType, .symbolic_link => { + // The file system path of the symbolic link. const file_name = try stripComponents(unstripped_file_name, options.strip_components); + // The data inside the symbolic link. const link_name = header.linkName(); - dir.symLink(link_name, file_name, .{}) catch |err| { + dir.symLink(link_name, file_name, .{}) catch |err| again: { + const code = code: { + if (err == error.FileNotFound) { + if (std.fs.path.dirname(file_name)) |dir_name| { + dir.makePath(dir_name) catch |code| break :code code; + break :again dir.symLink(link_name, file_name, .{}) catch |code| { + break :code code; + }; + } + } + break :code err; + }; const d = options.diagnostics orelse return error.UnableToCreateSymLink; try d.errors.append(d.allocator, .{ .unable_to_create_sym_link = .{ - .code = err, + .code = code, .file_name = try d.allocator.dupe(u8, file_name), .link_name = try d.allocator.dupe(u8, link_name), } }); diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 63b620f674932400e93961af2485102229fbf435..85d1ae4868d57ed586bf534bfc3cbf6b6df0925d 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -1,6 +1,6 @@ const std = @import("std.zig"); const tokenizer = @import("zig/tokenizer.zig"); -const fmt = @import("zig/fmt.zig"); +pub const fmt = @import("zig/fmt.zig"); const assert = std.debug.assert; pub const ErrorBundle = @import("zig/ErrorBundle.zig"); diff --git a/lib/std/zig/ErrorBundle.zig b/lib/std/zig/ErrorBundle.zig index cbde70bd707642c5584ac88cb3fe35c5b67a34a2..2b1980c58397a96c6f6ff8b1217ba0017a458c8e 100644 --- a/lib/std/zig/ErrorBundle.zig +++ b/lib/std/zig/ErrorBundle.zig @@ -383,7 +383,7 @@ pub const Wip = struct { }; } - pub fn addString(wip: *Wip, s: []const u8) !u32 { + pub fn addString(wip: *Wip, s: []const u8) Allocator.Error!u32 { const gpa = wip.gpa; const index: u32 = @intCast(wip.string_bytes.items.len); try wip.string_bytes.ensureUnusedCapacity(gpa, s.len + 1); @@ -392,7 +392,7 @@ pub const Wip = struct { return index; } - pub fn printString(wip: *Wip, comptime fmt: []const u8, args: anytype) !u32 { + pub fn printString(wip: *Wip, comptime fmt: []const u8, args: anytype) Allocator.Error!u32 { const gpa = wip.gpa; const index: u32 = @intCast(wip.string_bytes.items.len); try wip.string_bytes.writer(gpa).print(fmt, args); @@ -400,12 +400,12 @@ pub const Wip = struct { return index; } - pub fn addRootErrorMessage(wip: *Wip, em: ErrorMessage) !void { + pub fn addRootErrorMessage(wip: *Wip, em: ErrorMessage) Allocator.Error!void { try wip.root_list.ensureUnusedCapacity(wip.gpa, 1); wip.root_list.appendAssumeCapacity(try addErrorMessage(wip, em)); } - pub fn addErrorMessage(wip: *Wip, em: ErrorMessage) !MessageIndex { + pub fn addErrorMessage(wip: *Wip, em: ErrorMessage) Allocator.Error!MessageIndex { return @enumFromInt(try addExtra(wip, em)); } @@ -413,15 +413,15 @@ pub const Wip = struct { return @enumFromInt(addExtraAssumeCapacity(wip, em)); } - pub fn addSourceLocation(wip: *Wip, sl: SourceLocation) !SourceLocationIndex { + pub fn addSourceLocation(wip: *Wip, sl: SourceLocation) Allocator.Error!SourceLocationIndex { return @enumFromInt(try addExtra(wip, sl)); } - pub fn addReferenceTrace(wip: *Wip, rt: ReferenceTrace) !void { + pub fn addReferenceTrace(wip: *Wip, rt: ReferenceTrace) Allocator.Error!void { _ = try addExtra(wip, rt); } - pub fn addBundleAsNotes(wip: *Wip, other: ErrorBundle) !void { + pub fn addBundleAsNotes(wip: *Wip, other: ErrorBundle) Allocator.Error!void { const gpa = wip.gpa; try wip.string_bytes.ensureUnusedCapacity(gpa, other.string_bytes.len); diff --git a/lib/std/zig/fmt.zig b/lib/std/zig/fmt.zig index 4afcc7ac2e1719025cca935553d3fb0b49a800f2..5375b93025258d905b1d5343bd545e8796e90afc 100644 --- a/lib/std/zig/fmt.zig +++ b/lib/std/zig/fmt.zig @@ -13,7 +13,7 @@ fn formatId( return writer.writeAll(bytes); } try writer.writeAll("@\""); - try formatEscapes(bytes, "", options, writer); + try stringEscape(bytes, "", options, writer); try writer.writeByte('"'); } @@ -47,7 +47,7 @@ test "isValidId" { /// Print the string as escaped contents of a double quoted or single-quoted string. /// Format `{}` treats contents as a double-quoted string. /// Format `{'}` treats contents as a single-quoted string. -fn formatEscapes( +pub fn stringEscape( bytes: []const u8, comptime fmt: []const u8, options: std.fmt.FormatOptions, @@ -90,7 +90,7 @@ fn formatEscapes( /// The format specifier must be one of: /// * `{}` treats contents as a double-quoted string. /// * `{'}` treats contents as a single-quoted string. -pub fn fmtEscapes(bytes: []const u8) std.fmt.Formatter(formatEscapes) { +pub fn fmtEscapes(bytes: []const u8) std.fmt.Formatter(stringEscape) { return .{ .data = bytes }; } diff --git a/src/Autodoc.zig b/src/Autodoc.zig index a70040558c169c1af4a013dc6112d7a097bb3722..5a241e51abcf6eb900045de48418860a2f35d939 100644 --- a/src/Autodoc.zig +++ b/src/Autodoc.zig @@ -6,7 +6,7 @@ const Autodoc = @This(); const Compilation = @import("Compilation.zig"); const CompilationModule = @import("Module.zig"); const File = CompilationModule.File; -const Module = @import("Package.zig"); +const Module = @import("Package.zig").Module; const Tokenizer = std.zig.Tokenizer; const InternPool = @import("InternPool.zig"); const Zir = @import("Zir.zig"); @@ -98,9 +98,8 @@ pub fn generate(cm: *CompilationModule, output_dir: std.fs.Dir) !void { } fn generateZirData(self: *Autodoc, output_dir: std.fs.Dir) !void { - const root_src_dir = self.comp_module.main_pkg.root_src_directory; - const root_src_path = self.comp_module.main_pkg.root_src_path; - const joined_src_path = try root_src_dir.join(self.arena, &.{root_src_path}); + const root_src_path = self.comp_module.main_mod.root_src_path; + const joined_src_path = try self.comp_module.main_mod.root.joinString(self.arena, root_src_path); defer self.arena.free(joined_src_path); const abs_root_src_path = try std.fs.path.resolve(self.arena, &.{ ".", joined_src_path }); @@ -295,20 +294,20 @@ fn generateZirData(self: *Autodoc, output_dir: std.fs.Dir) !void { } const rootName = blk: { - const rootName = std.fs.path.basename(self.comp_module.main_pkg.root_src_path); + const rootName = std.fs.path.basename(self.comp_module.main_mod.root_src_path); break :blk rootName[0 .. rootName.len - 4]; }; const main_type_index = self.types.items.len; { - try self.modules.put(self.arena, self.comp_module.main_pkg, .{ + try self.modules.put(self.arena, self.comp_module.main_mod, .{ .name = rootName, .main = main_type_index, .table = .{}, }); try self.modules.entries.items(.value)[0].table.put( self.arena, - self.comp_module.main_pkg, + self.comp_module.main_mod, .{ .name = rootName, .value = 0, @@ -412,7 +411,7 @@ fn generateZirData(self: *Autodoc, output_dir: std.fs.Dir) !void { while (files_iterator.next()) |entry| { const sub_file_path = entry.key_ptr.*.sub_file_path; - const file_module = entry.key_ptr.*.pkg; + const file_module = entry.key_ptr.*.mod; const module_name = (self.modules.get(file_module) orelse continue).name; const file_path = std.fs.path.dirname(sub_file_path) orelse ""; @@ -986,12 +985,12 @@ fn walkInstruction( // importFile cannot error out since all files // are already loaded at this point - if (file.pkg.table.get(path)) |other_module| { + if (file.mod.deps.get(path)) |other_module| { const result = try self.modules.getOrPut(self.arena, other_module); // Immediately add this module to the import table of our // current module, regardless of wether it's new or not. - if (self.modules.getPtr(file.pkg)) |current_module| { + if (self.modules.getPtr(file.mod)) |current_module| { // TODO: apparently, in the stdlib a file gets analyzed before // its module gets added. I guess we're importing a file // that belongs to another module through its file path? @@ -1025,12 +1024,12 @@ fn walkInstruction( // TODO: Add this module as a dependency to the current module // TODO: this seems something that could be done in bulk // at the beginning or the end, or something. - const root_src_dir = other_module.root_src_directory; - const root_src_path = other_module.root_src_path; - const joined_src_path = try root_src_dir.join(self.arena, &.{root_src_path}); - defer self.arena.free(joined_src_path); - - const abs_root_src_path = try std.fs.path.resolve(self.arena, &.{ ".", joined_src_path }); + const abs_root_src_path = try std.fs.path.resolve(self.arena, &.{ + ".", + other_module.root.root_dir.path orelse ".", + other_module.root.sub_path, + other_module.root_src_path, + }); defer self.arena.free(abs_root_src_path); const new_file = self.comp_module.import_table.get(abs_root_src_path).?; @@ -5683,7 +5682,7 @@ fn writeFileTableToJson( while (it.next()) |entry| { try jsw.beginArray(); try jsw.write(entry.key_ptr.*.sub_file_path); - try jsw.write(mods.getIndex(entry.key_ptr.*.pkg) orelse 0); + try jsw.write(mods.getIndex(entry.key_ptr.*.mod) orelse 0); try jsw.endArray(); } try jsw.endArray(); @@ -5840,7 +5839,7 @@ fn addGuide(self: *Autodoc, file: *File, guide_path: []const u8, section: *Secti file.sub_file_path, "..", guide_path, }); - var guide_file = try file.pkg.root_src_directory.handle.openFile(resolved_path, .{}); + var guide_file = try file.mod.root.openFile(resolved_path, .{}); defer guide_file.close(); const guide = guide_file.reader().readAllAlloc(self.arena, 1 * 1024 * 1024) catch |err| switch (err) { diff --git a/src/Compilation.zig b/src/Compilation.zig index 28b67ff7346cac9dbcf79b263063b9aa7ef8e28b..cd4c6ea11b680a35f7971961d9f2b430be6bedcf 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -41,8 +41,9 @@ const resinator = @import("resinator.zig"); /// General-purpose allocator. Used for both temporary and long-term storage. gpa: Allocator, -/// Arena-allocated memory used during initialization. Should be untouched until deinit. -arena_state: std.heap.ArenaAllocator.State, +/// Arena-allocated memory, mostly used during initialization. However, it can be used +/// for other things requiring the same lifetime as the `Compilation`. +arena: std.heap.ArenaAllocator, bin_file: *link.File, c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{}, win32_resource_table: if (build_options.only_core_functionality) void else std.AutoArrayHashMapUnmanaged(*Win32Resource, void) = @@ -124,7 +125,7 @@ cache_parent: *Cache, /// Path to own executable for invoking `zig clang`. self_exe_path: ?[]const u8, /// null means -fno-emit-bin. -/// This is mutable memory allocated into the Compilation-lifetime arena (`arena_state`) +/// This is mutable memory allocated into the Compilation-lifetime arena (`arena`) /// of exactly the correct size for "o/[digest]/[basename]". /// The basename is of the outputted binary file in case we don't know the directory yet. whole_bin_sub_path: ?[]u8, @@ -273,8 +274,8 @@ const Job = union(enum) { /// The source file containing the Decl has been updated, and so the /// Decl may need its line number information updated in the debug info. update_line_number: Module.Decl.Index, - /// The main source file for the package needs to be analyzed. - analyze_pkg: *Package, + /// The main source file for the module needs to be analyzed. + analyze_mod: *Package.Module, /// one of the glibc static objects glibc_crt_file: glibc.CRTFile, @@ -414,7 +415,7 @@ pub const MiscTask = enum { compiler_rt, libssp, zig_libc, - analyze_pkg, + analyze_mod, @"musl crti.o", @"musl crtn.o", @@ -544,7 +545,7 @@ pub const InitOptions = struct { global_cache_directory: Directory, target: Target, root_name: []const u8, - main_pkg: ?*Package, + main_mod: ?*Package.Module, output_mode: std.builtin.OutputMode, thread_pool: *ThreadPool, dynamic_linker: ?[]const u8 = null, @@ -736,53 +737,55 @@ pub const InitOptions = struct { pdb_out_path: ?[]const u8 = null, }; -fn addPackageTableToCacheHash( +fn addModuleTableToCacheHash( hash: *Cache.HashHelper, arena: *std.heap.ArenaAllocator, - pkg_table: Package.Table, - seen_table: *std.AutoHashMap(*Package, void), + mod_table: Package.Module.Deps, + seen_table: *std.AutoHashMap(*Package.Module, void), hash_type: union(enum) { path_bytes, files: *Cache.Manifest }, ) (error{OutOfMemory} || std.os.GetCwdError)!void { const allocator = arena.allocator(); - const packages = try allocator.alloc(Package.Table.KV, pkg_table.count()); + const modules = try allocator.alloc(Package.Module.Deps.KV, mod_table.count()); { // Copy over the hashmap entries to our slice - var table_it = pkg_table.iterator(); + var table_it = mod_table.iterator(); var idx: usize = 0; while (table_it.next()) |entry| : (idx += 1) { - packages[idx] = .{ + modules[idx] = .{ .key = entry.key_ptr.*, .value = entry.value_ptr.*, }; } } // Sort the slice by package name - mem.sort(Package.Table.KV, packages, {}, struct { - fn lessThan(_: void, lhs: Package.Table.KV, rhs: Package.Table.KV) bool { + mem.sortUnstable(Package.Module.Deps.KV, modules, {}, struct { + fn lessThan(_: void, lhs: Package.Module.Deps.KV, rhs: Package.Module.Deps.KV) bool { return std.mem.lessThan(u8, lhs.key, rhs.key); } }.lessThan); - for (packages) |pkg| { - if ((try seen_table.getOrPut(pkg.value)).found_existing) continue; + for (modules) |mod| { + if ((try seen_table.getOrPut(mod.value)).found_existing) continue; // Finally insert the package name and path to the cache hash. - hash.addBytes(pkg.key); + hash.addBytes(mod.key); switch (hash_type) { .path_bytes => { - hash.addBytes(pkg.value.root_src_path); - hash.addOptionalBytes(pkg.value.root_src_directory.path); + hash.addBytes(mod.value.root_src_path); + hash.addOptionalBytes(mod.value.root.root_dir.path); + hash.addBytes(mod.value.root.sub_path); }, .files => |man| { - const pkg_zig_file = try pkg.value.root_src_directory.join(allocator, &[_][]const u8{ - pkg.value.root_src_path, - }); + const pkg_zig_file = try mod.value.root.joinString( + allocator, + mod.value.root_src_path, + ); _ = try man.addFile(pkg_zig_file, null); }, } - // Recurse to handle the package's dependencies - try addPackageTableToCacheHash(hash, arena, pkg.value.table, seen_table, hash_type); + // Recurse to handle the module's dependencies + try addModuleTableToCacheHash(hash, arena, mod.value.deps, seen_table, hash_type); } } @@ -839,7 +842,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { break :blk true; // If we have no zig code to compile, no need for LLVM. - if (options.main_pkg == null) + if (options.main_mod == null) break :blk false; // If LLVM does not support the target, then we can't use it. @@ -869,7 +872,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { // compiler state, the second clause here can be removed so that incremental // cache mode is used for LLVM backend too. We need some fuzz testing before // that can be enabled. - const cache_mode = if ((use_llvm or options.main_pkg == null) and !options.disable_lld_caching) + const cache_mode = if ((use_llvm or options.main_mod == null) and !options.disable_lld_caching) CacheMode.whole else options.cache_mode; @@ -925,7 +928,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { if (use_llvm) { // If stage1 generates an object file, self-hosted linker is not // yet sophisticated enough to handle that. - break :blk options.main_pkg != null; + break :blk options.main_mod != null; } break :blk false; @@ -1210,7 +1213,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model); // TODO audit this and make sure everything is in it - const module: ?*Module = if (options.main_pkg) |main_pkg| blk: { + const module: ?*Module = if (options.main_mod) |main_mod| blk: { // Options that are specific to zig source files, that cannot be // modified between incremental updates. var hash = cache.hash; @@ -1223,11 +1226,12 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { // do want to namespace different source file names because they are // likely different compilations and therefore this would be likely to // cause cache hits. - hash.addBytes(main_pkg.root_src_path); - hash.addOptionalBytes(main_pkg.root_src_directory.path); + hash.addBytes(main_mod.root_src_path); + hash.addOptionalBytes(main_mod.root.root_dir.path); + hash.addBytes(main_mod.root.sub_path); { - var seen_table = std.AutoHashMap(*Package, void).init(arena); - try addPackageTableToCacheHash(&hash, &arena_allocator, main_pkg.table, &seen_table, .path_bytes); + var seen_table = std.AutoHashMap(*Package.Module, void).init(arena); + try addModuleTableToCacheHash(&hash, &arena_allocator, main_mod.deps, &seen_table, .path_bytes); } }, .whole => { @@ -1283,81 +1287,83 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}), }; - const builtin_pkg = try Package.createWithDir( - gpa, - zig_cache_artifact_directory, - null, - "builtin.zig", - ); - errdefer builtin_pkg.destroy(gpa); + const builtin_mod = try Package.Module.create(arena, .{ + .root = .{ .root_dir = zig_cache_artifact_directory }, + .root_src_path = "builtin.zig", + .fully_qualified_name = "builtin", + }); - // When you're testing std, the main module is std. In that case, we'll just set the std - // module to the main one, since avoiding the errors caused by duplicating it is more - // effort than it's worth. - const main_pkg_is_std = m: { + // When you're testing std, the main module is std. In that case, + // we'll just set the std module to the main one, since avoiding + // the errors caused by duplicating it is more effort than it's + // worth. + const main_mod_is_std = m: { const std_path = try std.fs.path.resolve(arena, &[_][]const u8{ options.zig_lib_directory.path orelse ".", "std", "std.zig", }); - defer arena.free(std_path); const main_path = try std.fs.path.resolve(arena, &[_][]const u8{ - main_pkg.root_src_directory.path orelse ".", - main_pkg.root_src_path, + main_mod.root.root_dir.path orelse ".", + main_mod.root.sub_path, + main_mod.root_src_path, }); - defer arena.free(main_path); break :m mem.eql(u8, main_path, std_path); }; - const std_pkg = if (main_pkg_is_std) - main_pkg + const std_mod = if (main_mod_is_std) + main_mod else - try Package.createWithDir( - gpa, - options.zig_lib_directory, - "std", - "std.zig", - ); + try Package.Module.create(arena, .{ + .root = .{ + .root_dir = options.zig_lib_directory, + .sub_path = "std", + }, + .root_src_path = "std.zig", + .fully_qualified_name = "std", + }); - errdefer if (!main_pkg_is_std) std_pkg.destroy(gpa); + const root_mod = if (options.is_test) root_mod: { + const test_mod = if (options.test_runner_path) |test_runner| test_mod: { + const pkg = try Package.Module.create(arena, .{ + .root = .{ + .root_dir = Directory.cwd(), + .sub_path = std.fs.path.dirname(test_runner) orelse "", + }, + .root_src_path = std.fs.path.basename(test_runner), + .fully_qualified_name = "root", + }); - const root_pkg = if (options.is_test) root_pkg: { - const test_pkg = if (options.test_runner_path) |test_runner| test_pkg: { - const test_dir = std.fs.path.dirname(test_runner); - const basename = std.fs.path.basename(test_runner); - const pkg = try Package.create(gpa, test_dir, basename); + pkg.deps = try main_mod.deps.clone(arena); + break :test_mod pkg; + } else try Package.Module.create(arena, .{ + .root = .{ + .root_dir = options.zig_lib_directory, + }, + .root_src_path = "test_runner.zig", + .fully_qualified_name = "root", + }); - // copy package table from main_pkg to root_pkg - pkg.table = try main_pkg.table.clone(gpa); - break :test_pkg pkg; - } else try Package.createWithDir( - gpa, - options.zig_lib_directory, - null, - "test_runner.zig", - ); - errdefer test_pkg.destroy(gpa); + break :root_mod test_mod; + } else main_mod; - break :root_pkg test_pkg; - } else main_pkg; - errdefer if (options.is_test) root_pkg.destroy(gpa); - - const compiler_rt_pkg = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_pkg: { - break :compiler_rt_pkg try Package.createWithDir( - gpa, - options.zig_lib_directory, - null, - "compiler_rt.zig", - ); + const compiler_rt_mod = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_mod: { + break :compiler_rt_mod try Package.Module.create(arena, .{ + .root = .{ + .root_dir = options.zig_lib_directory, + }, + .root_src_path = "compiler_rt.zig", + .fully_qualified_name = "compiler_rt", + }); } else null; - errdefer if (compiler_rt_pkg) |p| p.destroy(gpa); - try main_pkg.add(gpa, "builtin", builtin_pkg); - try main_pkg.add(gpa, "root", root_pkg); - try main_pkg.add(gpa, "std", std_pkg); - - if (compiler_rt_pkg) |p| { - try main_pkg.add(gpa, "compiler_rt", p); + { + try main_mod.deps.ensureUnusedCapacity(arena, 4); + main_mod.deps.putAssumeCapacity("builtin", builtin_mod); + main_mod.deps.putAssumeCapacity("root", root_mod); + main_mod.deps.putAssumeCapacity("std", std_mod); + if (compiler_rt_mod) |m| + main_mod.deps.putAssumeCapacity("compiler_rt", m); } // Pre-open the directory handles for cached ZIR code so that it does not need @@ -1395,8 +1401,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { module.* = .{ .gpa = gpa, .comp = comp, - .main_pkg = main_pkg, - .root_pkg = root_pkg, + .main_mod = main_mod, + .root_mod = root_mod, .zig_cache_artifact_directory = zig_cache_artifact_directory, .global_zir_cache = global_zir_cache, .local_zir_cache = local_zir_cache, @@ -1664,7 +1670,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { errdefer bin_file.destroy(); comp.* = .{ .gpa = gpa, - .arena_state = arena_allocator.state, + .arena = arena_allocator, .zig_lib_directory = options.zig_lib_directory, .local_cache_directory = options.local_cache_directory, .global_cache_directory = options.global_cache_directory, @@ -1982,7 +1988,8 @@ pub fn destroy(self: *Compilation) void { if (self.owned_link_dir) |*dir| dir.close(); // This destroys `self`. - self.arena_state.promote(gpa).deinit(); + var arena_instance = self.arena; + arena_instance.deinit(); } pub fn clearMiscFailures(comp: *Compilation) void { @@ -2005,8 +2012,8 @@ fn restorePrevZigCacheArtifactDirectory(comp: *Compilation, directory: *Director // This is only for cleanup purposes; Module.deinit calls close // on the handle of zig_cache_artifact_directory. if (comp.bin_file.options.module) |module| { - const builtin_pkg = module.main_pkg.table.get("builtin").?; - module.zig_cache_artifact_directory = builtin_pkg.root_src_directory; + const builtin_mod = module.main_mod.deps.get("builtin").?; + module.zig_cache_artifact_directory = builtin_mod.root.root_dir; } } @@ -2148,8 +2155,8 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void // Make sure std.zig is inside the import_table. We unconditionally need // it for start.zig. - const std_pkg = module.main_pkg.table.get("std").?; - _ = try module.importPkg(std_pkg); + const std_mod = module.main_mod.deps.get("std").?; + _ = try module.importPkg(std_mod); // Normally we rely on importing std to in turn import the root source file // in the start code, but when using the stage1 backend that won't happen, @@ -2158,11 +2165,11 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void // Likewise, in the case of `zig test`, the test runner is the root source file, // and so there is nothing to import the main file. if (comp.bin_file.options.is_test) { - _ = try module.importPkg(module.main_pkg); + _ = try module.importPkg(module.main_mod); } - if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| { - _ = try module.importPkg(compiler_rt_pkg); + if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| { + _ = try module.importPkg(compiler_rt_mod); } // Put a work item in for every known source file to detect if @@ -2185,13 +2192,13 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void } } - try comp.work_queue.writeItem(.{ .analyze_pkg = std_pkg }); + try comp.work_queue.writeItem(.{ .analyze_mod = std_mod }); if (comp.bin_file.options.is_test) { - try comp.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg }); + try comp.work_queue.writeItem(.{ .analyze_mod = module.main_mod }); } - if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| { - try comp.work_queue.writeItem(.{ .analyze_pkg = compiler_rt_pkg }); + if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| { + try comp.work_queue.writeItem(.{ .analyze_mod = compiler_rt_mod }); } } @@ -2420,19 +2427,17 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes comptime assert(link_hash_implementation_version == 10); if (comp.bin_file.options.module) |mod| { - const main_zig_file = try mod.main_pkg.root_src_directory.join(arena, &[_][]const u8{ - mod.main_pkg.root_src_path, - }); + const main_zig_file = try mod.main_mod.root.joinString(arena, mod.main_mod.root_src_path); _ = try man.addFile(main_zig_file, null); { - var seen_table = std.AutoHashMap(*Package, void).init(arena); + var seen_table = std.AutoHashMap(*Package.Module, void).init(arena); // Skip builtin.zig; it is useless as an input, and we don't want to have to // write it before checking for a cache hit. - const builtin_pkg = mod.main_pkg.table.get("builtin").?; - try seen_table.put(builtin_pkg, {}); + const builtin_mod = mod.main_mod.deps.get("builtin").?; + try seen_table.put(builtin_mod, {}); - try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = man }); + try addModuleTableToCacheHash(&man.hash, &arena_allocator, mod.main_mod.deps, &seen_table, .{ .files = man }); } // Synchronize with other matching comments: ZigOnlyHashStuff @@ -2616,23 +2621,19 @@ fn reportMultiModuleErrors(mod: *Module) !void { errdefer for (notes[0..i]) |*n| n.deinit(mod.gpa); note.* = switch (ref) { .import => |loc| blk: { - const name = try loc.file_scope.pkg.getName(mod.gpa, mod.*); - defer mod.gpa.free(name); break :blk try Module.ErrorMsg.init( mod.gpa, loc, "imported from module {s}", - .{name}, + .{loc.file_scope.mod.fully_qualified_name}, ); }, .root => |pkg| blk: { - const name = try pkg.getName(mod.gpa, mod.*); - defer mod.gpa.free(name); break :blk try Module.ErrorMsg.init( mod.gpa, .{ .file_scope = file, .parent_decl_node = 0, .lazy = .entire_file }, "root of module {s}", - .{name}, + .{pkg.fully_qualified_name}, ); }, }; @@ -3564,8 +3565,8 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v decl.analysis = .codegen_failure_retryable; }; }, - .analyze_pkg => |pkg| { - const named_frame = tracy.namedFrame("analyze_pkg"); + .analyze_mod => |pkg| { + const named_frame = tracy.namedFrame("analyze_mod"); defer named_frame.end(); const module = comp.bin_file.options.module.?; @@ -3904,17 +3905,12 @@ pub fn obtainWin32ResourceCacheManifest(comp: *const Compilation) Cache.Manifest return man; } -test "cImport" { - _ = cImport; -} - pub const CImportResult = struct { out_zig_path: []u8, cache_hit: bool, errors: std.zig.ErrorBundle, pub fn deinit(result: *CImportResult, gpa: std.mem.Allocator) void { - gpa.free(result.out_zig_path); result.errors.deinit(gpa); } }; @@ -4059,7 +4055,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult { }; } - const out_zig_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{ + const out_zig_path = try comp.local_cache_directory.join(comp.arena.allocator(), &.{ "o", &digest, cimport_zig_basename, }); if (comp.verbose_cimport) { @@ -4214,17 +4210,9 @@ fn reportRetryableAstGenError( }, }; - const err_msg = if (file.pkg.root_src_directory.path) |dir_path| - try Module.ErrorMsg.create( - gpa, - src_loc, - "unable to load '{s}" ++ std.fs.path.sep_str ++ "{s}': {s}", - .{ dir_path, file.sub_file_path, @errorName(err) }, - ) - else - try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{s}': {s}", .{ - file.sub_file_path, @errorName(err), - }); + const err_msg = try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{ + file.mod.root, file.sub_file_path, @errorName(err), + }); errdefer err_msg.destroy(gpa); { @@ -4244,17 +4232,10 @@ fn reportRetryableEmbedFileError( const src_loc: Module.SrcLoc = mod.declPtr(embed_file.owner_decl).srcLoc(mod); - const err_msg = if (embed_file.pkg.root_src_directory.path) |dir_path| - try Module.ErrorMsg.create( - gpa, - src_loc, - "unable to load '{s}" ++ std.fs.path.sep_str ++ "{s}': {s}", - .{ dir_path, embed_file.sub_file_path, @errorName(err) }, - ) - else - try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{s}': {s}", .{ - embed_file.sub_file_path, @errorName(err), - }); + const err_msg = try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{ + embed_file.mod.root, embed_file.sub_file_path, @errorName(err), + }); + errdefer err_msg.destroy(gpa); { @@ -6377,13 +6358,13 @@ fn buildOutputFromZig( const tracy_trace = trace(@src()); defer tracy_trace.end(); - std.debug.assert(output_mode != .Exe); + assert(output_mode != .Exe); - var main_pkg: Package = .{ - .root_src_directory = comp.zig_lib_directory, + var main_mod: Package.Module = .{ + .root = .{ .root_dir = comp.zig_lib_directory }, .root_src_path = src_basename, + .fully_qualified_name = "root", }; - defer main_pkg.deinitTable(comp.gpa); const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len]; const target = comp.getTarget(); const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{ @@ -6404,7 +6385,7 @@ fn buildOutputFromZig( .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = &main_pkg, + .main_mod = &main_mod, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, @@ -6481,7 +6462,7 @@ pub fn build_crt_file( .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = null, + .main_mod = null, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, diff --git a/src/Manifest.zig b/src/Manifest.zig deleted file mode 100644 index 2ff54e613266783c5716ab67864a5a3203d7bf73..0000000000000000000000000000000000000000 --- a/src/Manifest.zig +++ /dev/null @@ -1,519 +0,0 @@ -pub const basename = "build.zig.zon"; -pub const Hash = std.crypto.hash.sha2.Sha256; - -pub const Dependency = struct { - location: union(enum) { - url: []const u8, - path: []const u8, - }, - location_tok: Ast.TokenIndex, - hash: ?[]const u8, - hash_tok: Ast.TokenIndex, -}; - -pub const ErrorMessage = struct { - msg: []const u8, - tok: Ast.TokenIndex, - off: u32, -}; - -pub const MultihashFunction = enum(u16) { - identity = 0x00, - sha1 = 0x11, - @"sha2-256" = 0x12, - @"sha2-512" = 0x13, - @"sha3-512" = 0x14, - @"sha3-384" = 0x15, - @"sha3-256" = 0x16, - @"sha3-224" = 0x17, - @"sha2-384" = 0x20, - @"sha2-256-trunc254-padded" = 0x1012, - @"sha2-224" = 0x1013, - @"sha2-512-224" = 0x1014, - @"sha2-512-256" = 0x1015, - @"blake2b-256" = 0xb220, - _, -}; - -pub const multihash_function: MultihashFunction = switch (Hash) { - std.crypto.hash.sha2.Sha256 => .@"sha2-256", - else => @compileError("unreachable"), -}; -comptime { - // We avoid unnecessary uleb128 code in hexDigest by asserting here the - // values are small enough to be contained in the one-byte encoding. - assert(@intFromEnum(multihash_function) < 127); - assert(Hash.digest_length < 127); -} -pub const multihash_len = 1 + 1 + Hash.digest_length; - -name: []const u8, -version: std.SemanticVersion, -dependencies: std.StringArrayHashMapUnmanaged(Dependency), - -errors: []ErrorMessage, -arena_state: std.heap.ArenaAllocator.State, - -pub const Error = Allocator.Error; - -pub fn parse(gpa: Allocator, ast: std.zig.Ast) Error!Manifest { - const node_tags = ast.nodes.items(.tag); - const node_datas = ast.nodes.items(.data); - assert(node_tags[0] == .root); - const main_node_index = node_datas[0].lhs; - - var arena_instance = std.heap.ArenaAllocator.init(gpa); - errdefer arena_instance.deinit(); - - var p: Parse = .{ - .gpa = gpa, - .ast = ast, - .arena = arena_instance.allocator(), - .errors = .{}, - - .name = undefined, - .version = undefined, - .dependencies = .{}, - .buf = .{}, - }; - defer p.buf.deinit(gpa); - defer p.errors.deinit(gpa); - defer p.dependencies.deinit(gpa); - - p.parseRoot(main_node_index) catch |err| switch (err) { - error.ParseFailure => assert(p.errors.items.len > 0), - else => |e| return e, - }; - - return .{ - .name = p.name, - .version = p.version, - .dependencies = try p.dependencies.clone(p.arena), - .errors = try p.arena.dupe(ErrorMessage, p.errors.items), - .arena_state = arena_instance.state, - }; -} - -pub fn deinit(man: *Manifest, gpa: Allocator) void { - man.arena_state.promote(gpa).deinit(); - man.* = undefined; -} - -const hex_charset = "0123456789abcdef"; - -pub fn hex64(x: u64) [16]u8 { - var result: [16]u8 = undefined; - var i: usize = 0; - while (i < 8) : (i += 1) { - const byte = @as(u8, @truncate(x >> @as(u6, @intCast(8 * i)))); - result[i * 2 + 0] = hex_charset[byte >> 4]; - result[i * 2 + 1] = hex_charset[byte & 15]; - } - return result; -} - -test hex64 { - const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]"; - try std.testing.expectEqualStrings("[00efcdab78563412]", s); -} - -pub fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 { - var result: [multihash_len * 2]u8 = undefined; - - result[0] = hex_charset[@intFromEnum(multihash_function) >> 4]; - result[1] = hex_charset[@intFromEnum(multihash_function) & 15]; - - result[2] = hex_charset[Hash.digest_length >> 4]; - result[3] = hex_charset[Hash.digest_length & 15]; - - for (digest, 0..) |byte, i| { - result[4 + i * 2] = hex_charset[byte >> 4]; - result[5 + i * 2] = hex_charset[byte & 15]; - } - return result; -} - -const Parse = struct { - gpa: Allocator, - ast: std.zig.Ast, - arena: Allocator, - buf: std.ArrayListUnmanaged(u8), - errors: std.ArrayListUnmanaged(ErrorMessage), - - name: []const u8, - version: std.SemanticVersion, - dependencies: std.StringArrayHashMapUnmanaged(Dependency), - - const InnerError = error{ ParseFailure, OutOfMemory }; - - fn parseRoot(p: *Parse, node: Ast.Node.Index) !void { - const ast = p.ast; - const main_tokens = ast.nodes.items(.main_token); - const main_token = main_tokens[node]; - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - return fail(p, main_token, "expected top level expression to be a struct", .{}); - }; - - var have_name = false; - var have_version = false; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - const field_name = try identifierTokenString(p, name_token); - // We could get fancy with reflection and comptime logic here but doing - // things manually provides an opportunity to do any additional verification - // that is desirable on a per-field basis. - if (mem.eql(u8, field_name, "dependencies")) { - try parseDependencies(p, field_init); - } else if (mem.eql(u8, field_name, "name")) { - p.name = try parseString(p, field_init); - have_name = true; - } else if (mem.eql(u8, field_name, "version")) { - const version_text = try parseString(p, field_init); - p.version = std.SemanticVersion.parse(version_text) catch |err| v: { - try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)}); - break :v undefined; - }; - have_version = true; - } else { - // Ignore unknown fields so that we can add fields in future zig - // versions without breaking older zig versions. - } - } - - if (!have_name) { - try appendError(p, main_token, "missing top-level 'name' field", .{}); - } - - if (!have_version) { - try appendError(p, main_token, "missing top-level 'version' field", .{}); - } - } - - fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { - const ast = p.ast; - const main_tokens = ast.nodes.items(.main_token); - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - const tok = main_tokens[node]; - return fail(p, tok, "expected dependencies expression to be a struct", .{}); - }; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - const dep_name = try identifierTokenString(p, name_token); - const dep = try parseDependency(p, field_init); - try p.dependencies.put(p.gpa, dep_name, dep); - } - } - - fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { - const ast = p.ast; - const main_tokens = ast.nodes.items(.main_token); - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - const tok = main_tokens[node]; - return fail(p, tok, "expected dependency expression to be a struct", .{}); - }; - - var dep: Dependency = .{ - .location = undefined, - .location_tok = undefined, - .hash = null, - .hash_tok = undefined, - }; - var has_location = false; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - const field_name = try identifierTokenString(p, name_token); - // We could get fancy with reflection and comptime logic here but doing - // things manually provides an opportunity to do any additional verification - // that is desirable on a per-field basis. - if (mem.eql(u8, field_name, "url")) { - if (has_location) { - return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{}); - } - dep.location = .{ - .url = parseString(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }, - }; - has_location = true; - dep.location_tok = main_tokens[field_init]; - } else if (mem.eql(u8, field_name, "path")) { - if (has_location) { - return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{}); - } - dep.location = .{ - .path = parseString(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }, - }; - has_location = true; - dep.location_tok = main_tokens[field_init]; - } else if (mem.eql(u8, field_name, "hash")) { - dep.hash = parseHash(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }; - dep.hash_tok = main_tokens[field_init]; - } else { - // Ignore unknown fields so that we can add fields in future zig - // versions without breaking older zig versions. - } - } - - if (!has_location) { - try appendError(p, main_tokens[node], "dependency requires location field, one of 'url' or 'path'.", .{}); - } - - return dep; - } - - fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { - const ast = p.ast; - const node_tags = ast.nodes.items(.tag); - const main_tokens = ast.nodes.items(.main_token); - if (node_tags[node] != .string_literal) { - return fail(p, main_tokens[node], "expected string literal", .{}); - } - const str_lit_token = main_tokens[node]; - const token_bytes = ast.tokenSlice(str_lit_token); - p.buf.clearRetainingCapacity(); - try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); - const duped = try p.arena.dupe(u8, p.buf.items); - return duped; - } - - fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { - const ast = p.ast; - const main_tokens = ast.nodes.items(.main_token); - const tok = main_tokens[node]; - const h = try parseString(p, node); - - if (h.len >= 2) { - const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| { - return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{ - @errorName(err), - }); - }; - if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) { - return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{}); - } - } - - const hex_multihash_len = 2 * Manifest.multihash_len; - if (h.len != hex_multihash_len) { - return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{ - hex_multihash_len, h.len, - }); - } - - return h; - } - - /// TODO: try to DRY this with AstGen.identifierTokenString - fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { - const ast = p.ast; - const token_tags = ast.tokens.items(.tag); - assert(token_tags[token] == .identifier); - const ident_name = ast.tokenSlice(token); - if (!mem.startsWith(u8, ident_name, "@")) { - return ident_name; - } - p.buf.clearRetainingCapacity(); - try parseStrLit(p, token, &p.buf, ident_name, 1); - const duped = try p.arena.dupe(u8, p.buf.items); - return duped; - } - - /// TODO: try to DRY this with AstGen.parseStrLit - fn parseStrLit( - p: *Parse, - token: Ast.TokenIndex, - buf: *std.ArrayListUnmanaged(u8), - bytes: []const u8, - offset: u32, - ) InnerError!void { - const raw_string = bytes[offset..]; - var buf_managed = buf.toManaged(p.gpa); - const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string); - buf.* = buf_managed.moveToUnmanaged(); - switch (try result) { - .success => {}, - .failure => |err| try p.appendStrLitError(err, token, bytes, offset), - } - } - - /// TODO: try to DRY this with AstGen.failWithStrLitError - fn appendStrLitError( - p: *Parse, - err: std.zig.string_literal.Error, - token: Ast.TokenIndex, - bytes: []const u8, - offset: u32, - ) Allocator.Error!void { - const raw_string = bytes[offset..]; - switch (err) { - .invalid_escape_character => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "invalid escape character: '{c}'", - .{raw_string[bad_index]}, - ); - }, - .expected_hex_digit => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected hex digit, found '{c}'", - .{raw_string[bad_index]}, - ); - }, - .empty_unicode_escape_sequence => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "empty unicode escape sequence", - .{}, - ); - }, - .expected_hex_digit_or_rbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected hex digit or '}}', found '{c}'", - .{raw_string[bad_index]}, - ); - }, - .invalid_unicode_codepoint => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "unicode escape does not correspond to a valid codepoint", - .{}, - ); - }, - .expected_lbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected '{{', found '{c}", - .{raw_string[bad_index]}, - ); - }, - .expected_rbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected '}}', found '{c}", - .{raw_string[bad_index]}, - ); - }, - .expected_single_quote => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected single quote ('), found '{c}", - .{raw_string[bad_index]}, - ); - }, - .invalid_character => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "invalid byte in string or character literal: '{c}'", - .{raw_string[bad_index]}, - ); - }, - } - } - - fn fail( - p: *Parse, - tok: Ast.TokenIndex, - comptime fmt: []const u8, - args: anytype, - ) InnerError { - try appendError(p, tok, fmt, args); - return error.ParseFailure; - } - - fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { - return appendErrorOff(p, tok, 0, fmt, args); - } - - fn appendErrorOff( - p: *Parse, - tok: Ast.TokenIndex, - byte_offset: u32, - comptime fmt: []const u8, - args: anytype, - ) Allocator.Error!void { - try p.errors.append(p.gpa, .{ - .msg = try std.fmt.allocPrint(p.arena, fmt, args), - .tok = tok, - .off = byte_offset, - }); - } -}; - -const Manifest = @This(); -const std = @import("std"); -const mem = std.mem; -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; -const Ast = std.zig.Ast; -const testing = std.testing; - -test "basic" { - const gpa = testing.allocator; - - const example = - \\.{ - \\ .name = "foo", - \\ .version = "3.2.1", - \\ .dependencies = .{ - \\ .bar = .{ - \\ .url = "https://example.com/baz.tar.gz", - \\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", - \\ }, - \\ }, - \\} - ; - - var ast = try std.zig.Ast.parse(gpa, example, .zon); - defer ast.deinit(gpa); - - try testing.expect(ast.errors.len == 0); - - var manifest = try Manifest.parse(gpa, ast); - defer manifest.deinit(gpa); - - try testing.expectEqualStrings("foo", manifest.name); - - try testing.expectEqual(@as(std.SemanticVersion, .{ - .major = 3, - .minor = 2, - .patch = 1, - }), manifest.version); - - try testing.expect(manifest.dependencies.count() == 1); - try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); - try testing.expectEqualStrings( - "https://example.com/baz.tar.gz", - manifest.dependencies.values()[0].url, - ); - try testing.expectEqualStrings( - "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", - manifest.dependencies.values()[0].hash orelse return error.TestFailed, - ); -} diff --git a/src/Module.zig b/src/Module.zig index 8c4035cc9af78f99f4fd72079202436b53215a85..41f4ec2b41014001a6110977bd1e523193e6cd2d 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -55,10 +55,10 @@ comp: *Compilation, /// Where build artifacts and incremental compilation metadata serialization go. zig_cache_artifact_directory: Compilation.Directory, /// Pointer to externally managed resource. -root_pkg: *Package, -/// Normally, `main_pkg` and `root_pkg` are the same. The exception is `zig test`, in which -/// `root_pkg` is the test runner, and `main_pkg` is the user's source file which has the tests. -main_pkg: *Package, +root_mod: *Package.Module, +/// Normally, `main_mod` and `root_mod` are the same. The exception is `zig test`, in which +/// `root_mod` is the test runner, and `main_mod` is the user's source file which has the tests. +main_mod: *Package.Module, sema_prog_node: std.Progress.Node = undefined, /// Used by AstGen worker to load and store ZIR cache. @@ -973,8 +973,8 @@ pub const File = struct { tree: Ast, /// Whether this is populated or not depends on `zir_loaded`. zir: Zir, - /// Package that this file is a part of, managed externally. - pkg: *Package, + /// Module that this file is a part of, managed externally. + mod: *Package.Module, /// Whether this file is a part of multiple packages. This is an error condition which will be reported after AstGen. multi_pkg: bool = false, /// List of references to this file, used for multi-package errors. @@ -998,8 +998,8 @@ pub const File = struct { pub const Reference = union(enum) { /// The file is imported directly (i.e. not as a package) with @import. import: SrcLoc, - /// The file is the root of a package. - root: *Package, + /// The file is the root of a module. + root: *Package.Module, }; pub fn unload(file: *File, gpa: Allocator) void { @@ -1058,14 +1058,9 @@ pub const File = struct { .stat = file.stat, }; - const root_dir_path = file.pkg.root_src_directory.path orelse "."; - log.debug("File.getSource, not cached. pkgdir={s} sub_file_path={s}", .{ - root_dir_path, file.sub_file_path, - }); - // Keep track of inode, file size, mtime, hash so we can detect which files // have been modified when an incremental update is requested. - var f = try file.pkg.root_src_directory.handle.openFile(file.sub_file_path, .{}); + var f = try file.mod.root.openFile(file.sub_file_path, .{}); defer f.close(); const stat = try f.stat(); @@ -1134,14 +1129,12 @@ pub const File = struct { return ip.getOrPutTrailingString(mod.gpa, ip.string_bytes.items.len - start); } - /// Returns the full path to this file relative to its package. pub fn fullPath(file: File, ally: Allocator) ![]u8 { - return file.pkg.root_src_directory.join(ally, &[_][]const u8{file.sub_file_path}); + return file.mod.root.joinString(ally, file.sub_file_path); } - /// Returns the full path to this file relative to its package. pub fn fullPathZ(file: File, ally: Allocator) ![:0]u8 { - return file.pkg.root_src_directory.joinZ(ally, &[_][]const u8{file.sub_file_path}); + return file.mod.root.joinStringZ(ally, file.sub_file_path); } pub fn dumpSrc(file: *File, src: LazySrcLoc) void { @@ -1181,10 +1174,10 @@ pub const File = struct { } const pkg = switch (ref) { - .import => |loc| loc.file_scope.pkg, + .import => |loc| loc.file_scope.mod, .root => |pkg| pkg, }; - if (pkg != file.pkg) file.multi_pkg = true; + if (pkg != file.mod) file.multi_pkg = true; } /// Mark this file and every file referenced by it as multi_pkg and report an @@ -1226,7 +1219,7 @@ pub const EmbedFile = struct { bytes: [:0]const u8, stat: Cache.File.Stat, /// Package that this file is a part of, managed externally. - pkg: *Package, + mod: *Package.Module, /// The Decl that was created from the `@embedFile` to own this resource. /// This is how zig knows what other Decl objects to invalidate if the file /// changes on disk. @@ -2542,28 +2535,6 @@ pub fn deinit(mod: *Module) void { } mod.deletion_set.deinit(gpa); - - // The callsite of `Compilation.create` owns the `main_pkg`, however - // Module owns the builtin and std packages that it adds. - if (mod.main_pkg.table.fetchRemove("builtin")) |kv| { - gpa.free(kv.key); - kv.value.destroy(gpa); - } - if (mod.main_pkg.table.fetchRemove("std")) |kv| { - gpa.free(kv.key); - // It's possible for main_pkg to be std when running 'zig test'! In this case, we must not - // destroy it, since it would lead to a double-free. - if (kv.value != mod.main_pkg) { - kv.value.destroy(gpa); - } - } - if (mod.main_pkg.table.fetchRemove("root")) |kv| { - gpa.free(kv.key); - } - if (mod.root_pkg != mod.main_pkg) { - mod.root_pkg.destroy(gpa); - } - mod.compile_log_text.deinit(gpa); mod.zig_cache_artifact_directory.handle.close(); @@ -2710,18 +2681,19 @@ pub fn astGenFile(mod: *Module, file: *File) !void { const gpa = mod.gpa; // In any case we need to examine the stat of the file to determine the course of action. - var source_file = try file.pkg.root_src_directory.handle.openFile(file.sub_file_path, .{}); + var source_file = try file.mod.root.openFile(file.sub_file_path, .{}); defer source_file.close(); const stat = try source_file.stat(); - const want_local_cache = file.pkg == mod.main_pkg; + const want_local_cache = file.mod == mod.main_mod; const digest = hash: { var path_hash: Cache.HashHelper = .{}; path_hash.addBytes(build_options.version); path_hash.add(builtin.zig_backend); if (!want_local_cache) { - path_hash.addOptionalBytes(file.pkg.root_src_directory.path); + path_hash.addOptionalBytes(file.mod.root.root_dir.path); + path_hash.addBytes(file.mod.root.sub_path); } path_hash.addBytes(file.sub_file_path); break :hash path_hash.final(); @@ -2946,10 +2918,8 @@ pub fn astGenFile(mod: *Module, file: *File) !void { }, }; cache_file.writevAll(&iovecs) catch |err| { - const pkg_path = file.pkg.root_src_directory.path orelse "."; - const cache_path = cache_directory.path orelse "."; - log.warn("unable to write cached ZIR code for {s}/{s} to {s}/{s}: {s}", .{ - pkg_path, file.sub_file_path, cache_path, &digest, @errorName(err), + log.warn("unable to write cached ZIR code for {}{s} to {}{s}: {s}", .{ + file.mod.root, file.sub_file_path, cache_directory, &digest, @errorName(err), }); }; @@ -3154,37 +3124,27 @@ pub fn populateBuiltinFile(mod: *Module) !void { defer tracy.end(); const comp = mod.comp; - const pkg_and_file = blk: { + const builtin_mod, const file = blk: { comp.mutex.lock(); defer comp.mutex.unlock(); - const builtin_pkg = mod.main_pkg.table.get("builtin").?; - const result = try mod.importPkg(builtin_pkg); - break :blk .{ - .file = result.file, - .pkg = builtin_pkg, - }; + const builtin_mod = mod.main_mod.deps.get("builtin").?; + const result = try mod.importPkg(builtin_mod); + break :blk .{ builtin_mod, result.file }; }; - const file = pkg_and_file.file; - const builtin_pkg = pkg_and_file.pkg; const gpa = mod.gpa; file.source = try comp.generateBuiltinZigSource(gpa); file.source_loaded = true; - if (builtin_pkg.root_src_directory.handle.statFile(builtin_pkg.root_src_path)) |stat| { + if (builtin_mod.root.statFile(builtin_mod.root_src_path)) |stat| { if (stat.size != file.source.len) { - const full_path = try builtin_pkg.root_src_directory.join(gpa, &.{ - builtin_pkg.root_src_path, - }); - defer gpa.free(full_path); - log.warn( - "the cached file '{s}' had the wrong size. Expected {d}, found {d}. " ++ + "the cached file '{}{s}' had the wrong size. Expected {d}, found {d}. " ++ "Overwriting with correct file contents now", - .{ full_path, file.source.len, stat.size }, + .{ builtin_mod.root, builtin_mod.root_src_path, file.source.len, stat.size }, ); - try writeBuiltinFile(file, builtin_pkg); + try writeBuiltinFile(file, builtin_mod); } else { file.stat = .{ .size = stat.size, @@ -3198,7 +3158,7 @@ pub fn populateBuiltinFile(mod: *Module) !void { error.PipeBusy => unreachable, // it's not a pipe error.WouldBlock => unreachable, // not asking for non-blocking I/O - error.FileNotFound => try writeBuiltinFile(file, builtin_pkg), + error.FileNotFound => try writeBuiltinFile(file, builtin_mod), else => |e| return e, } @@ -3212,8 +3172,8 @@ pub fn populateBuiltinFile(mod: *Module) !void { file.status = .success_zir; } -fn writeBuiltinFile(file: *File, builtin_pkg: *Package) !void { - var af = try builtin_pkg.root_src_directory.handle.atomicFile(builtin_pkg.root_src_path, .{}); +fn writeBuiltinFile(file: *File, builtin_mod: *Package.Module) !void { + var af = try builtin_mod.root.atomicFile(builtin_mod.root_src_path, .{}); defer af.deinit(); try af.file.writeAll(file.source); try af.finish(); @@ -3609,7 +3569,8 @@ pub fn updateEmbedFile(mod: *Module, embed_file: *EmbedFile) SemaError!void { } } -pub fn semaPkg(mod: *Module, pkg: *Package) !void { +/// https://github.com/ziglang/zig/issues/14307 +pub fn semaPkg(mod: *Module, pkg: *Package.Module) !void { const file = (try mod.importPkg(pkg)).file; return mod.semaFile(file); } @@ -3711,13 +3672,11 @@ pub fn semaFile(mod: *Module, file: *File) SemaError!void { return error.AnalysisFail; }; - const resolved_path = std.fs.path.resolve( - gpa, - if (file.pkg.root_src_directory.path) |pkg_path| - &[_][]const u8{ pkg_path, file.sub_file_path } - else - &[_][]const u8{file.sub_file_path}, - ) catch |err| { + const resolved_path = std.fs.path.resolve(gpa, &.{ + file.mod.root.root_dir.path orelse ".", + file.mod.root.sub_path, + file.sub_file_path, + }) catch |err| { try reportRetryableFileError(mod, file, "unable to resolve path: {s}", .{@errorName(err)}); return error.AnalysisFail; }; @@ -3748,8 +3707,8 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !bool { // TODO: figure out how this works under incremental changes to builtin.zig! const builtin_type_target_index: InternPool.Index = blk: { - const std_mod = mod.main_pkg.table.get("std").?; - if (decl.getFileScope(mod).pkg != std_mod) break :blk .none; + const std_mod = mod.main_mod.deps.get("std").?; + if (decl.getFileScope(mod).mod != std_mod) break :blk .none; // We're in the std module. const std_file = (try mod.importPkg(std_mod)).file; const std_decl = mod.declPtr(std_file.root_decl.unwrap().?); @@ -4042,14 +4001,17 @@ pub const ImportFileResult = struct { is_pkg: bool, }; -pub fn importPkg(mod: *Module, pkg: *Package) !ImportFileResult { +/// https://github.com/ziglang/zig/issues/14307 +pub fn importPkg(mod: *Module, pkg: *Package.Module) !ImportFileResult { const gpa = mod.gpa; // The resolved path is used as the key in the import table, to detect if // an import refers to the same as another, despite different relative paths // or differently mapped package names. - const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{ - pkg.root_src_directory.path orelse ".", pkg.root_src_path, + const resolved_path = try std.fs.path.resolve(gpa, &.{ + pkg.root.root_dir.path orelse ".", + pkg.root.sub_path, + pkg.root_src_path, }); var keep_resolved_path = false; defer if (!keep_resolved_path) gpa.free(resolved_path); @@ -4083,7 +4045,7 @@ pub fn importPkg(mod: *Module, pkg: *Package) !ImportFileResult { .tree = undefined, .zir = undefined, .status = .never_loaded, - .pkg = pkg, + .mod = pkg, .root_decl = .none, }; try new_file.addReference(mod.*, .{ .root = pkg }); @@ -4100,29 +4062,33 @@ pub fn importFile( import_string: []const u8, ) !ImportFileResult { if (std.mem.eql(u8, import_string, "std")) { - return mod.importPkg(mod.main_pkg.table.get("std").?); + return mod.importPkg(mod.main_mod.deps.get("std").?); } if (std.mem.eql(u8, import_string, "builtin")) { - return mod.importPkg(mod.main_pkg.table.get("builtin").?); + return mod.importPkg(mod.main_mod.deps.get("builtin").?); } if (std.mem.eql(u8, import_string, "root")) { - return mod.importPkg(mod.root_pkg); + return mod.importPkg(mod.root_mod); } - if (cur_file.pkg.table.get(import_string)) |pkg| { + if (cur_file.mod.deps.get(import_string)) |pkg| { return mod.importPkg(pkg); } if (!mem.endsWith(u8, import_string, ".zig")) { - return error.PackageNotFound; + return error.ModuleNotFound; } const gpa = mod.gpa; // The resolved path is used as the key in the import table, to detect if // an import refers to the same as another, despite different relative paths // or differently mapped package names. - const cur_pkg_dir_path = cur_file.pkg.root_src_directory.path orelse "."; - const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{ - cur_pkg_dir_path, cur_file.sub_file_path, "..", import_string, + const resolved_path = try std.fs.path.resolve(gpa, &.{ + cur_file.mod.root.root_dir.path orelse ".", + cur_file.mod.root.sub_path, + cur_file.sub_file_path, + "..", + import_string, }); + var keep_resolved_path = false; defer if (!keep_resolved_path) gpa.free(resolved_path); @@ -4137,7 +4103,10 @@ pub fn importFile( const new_file = try gpa.create(File); errdefer gpa.destroy(new_file); - const resolved_root_path = try std.fs.path.resolve(gpa, &[_][]const u8{cur_pkg_dir_path}); + const resolved_root_path = try std.fs.path.resolve(gpa, &.{ + cur_file.mod.root.root_dir.path orelse ".", + cur_file.mod.root.sub_path, + }); defer gpa.free(resolved_root_path); const sub_file_path = p: { @@ -4151,7 +4120,7 @@ pub fn importFile( { break :p try gpa.dupe(u8, resolved_path); } - return error.ImportOutsidePkgPath; + return error.ImportOutsideModulePath; }; errdefer gpa.free(sub_file_path); @@ -4171,7 +4140,7 @@ pub fn importFile( .tree = undefined, .zir = undefined, .status = .never_loaded, - .pkg = cur_file.pkg, + .mod = cur_file.mod, .root_decl = .none, }; return ImportFileResult{ @@ -4184,9 +4153,11 @@ pub fn importFile( pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*EmbedFile { const gpa = mod.gpa; - if (cur_file.pkg.table.get(import_string)) |pkg| { - const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{ - pkg.root_src_directory.path orelse ".", pkg.root_src_path, + if (cur_file.mod.deps.get(import_string)) |pkg| { + const resolved_path = try std.fs.path.resolve(gpa, &.{ + pkg.root.root_dir.path orelse ".", + pkg.root.sub_path, + pkg.root_src_path, }); var keep_resolved_path = false; defer if (!keep_resolved_path) gpa.free(resolved_path); @@ -4203,10 +4174,14 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb // The resolved path is used as the key in the table, to detect if a file // refers to the same as another, despite different relative paths. - const cur_pkg_dir_path = cur_file.pkg.root_src_directory.path orelse "."; - const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{ - cur_pkg_dir_path, cur_file.sub_file_path, "..", import_string, + const resolved_path = try std.fs.path.resolve(gpa, &.{ + cur_file.mod.root.root_dir.path orelse ".", + cur_file.mod.root.sub_path, + cur_file.sub_file_path, + "..", + import_string, }); + var keep_resolved_path = false; defer if (!keep_resolved_path) gpa.free(resolved_path); @@ -4214,7 +4189,10 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb errdefer assert(mod.embed_table.remove(resolved_path)); if (gop.found_existing) return gop.value_ptr.*; - const resolved_root_path = try std.fs.path.resolve(gpa, &[_][]const u8{cur_pkg_dir_path}); + const resolved_root_path = try std.fs.path.resolve(gpa, &.{ + cur_file.mod.root.root_dir.path orelse ".", + cur_file.mod.root.sub_path, + }); defer gpa.free(resolved_root_path); const sub_file_path = p: { @@ -4228,16 +4206,17 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb { break :p try gpa.dupe(u8, resolved_path); } - return error.ImportOutsidePkgPath; + return error.ImportOutsideModulePath; }; errdefer gpa.free(sub_file_path); - return newEmbedFile(mod, cur_file.pkg, sub_file_path, resolved_path, &keep_resolved_path, gop); + return newEmbedFile(mod, cur_file.mod, sub_file_path, resolved_path, &keep_resolved_path, gop); } +/// https://github.com/ziglang/zig/issues/14307 fn newEmbedFile( mod: *Module, - pkg: *Package, + pkg: *Package.Module, sub_file_path: []const u8, resolved_path: []const u8, keep_resolved_path: *bool, @@ -4248,7 +4227,7 @@ fn newEmbedFile( const new_file = try gpa.create(EmbedFile); errdefer gpa.destroy(new_file); - var file = try pkg.root_src_directory.handle.openFile(sub_file_path, .{}); + var file = try pkg.root.openFile(sub_file_path, .{}); defer file.close(); const actual_stat = try file.stat(); @@ -4275,14 +4254,14 @@ fn newEmbedFile( .sub_file_path = sub_file_path, .bytes = bytes, .stat = stat, - .pkg = pkg, + .mod = pkg, .owner_decl = undefined, // Set by Sema immediately after this function returns. }; return new_file; } pub fn detectEmbedFileUpdate(mod: *Module, embed_file: *EmbedFile) !void { - var file = try embed_file.pkg.root_src_directory.handle.openFile(embed_file.sub_file_path, .{}); + var file = try embed_file.mod.root.openFile(embed_file.sub_file_path, .{}); defer file.close(); const stat = try file.stat(); @@ -4455,21 +4434,21 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) Allocator.Err gop.key_ptr.* = new_decl_index; // Exported decls, comptime decls, usingnamespace decls, and // test decls if in test mode, get analyzed. - const decl_pkg = namespace.file_scope.pkg; + const decl_mod = namespace.file_scope.mod; const want_analysis = is_exported or switch (decl_name_index) { 0 => true, // comptime or usingnamespace decl 1 => blk: { // test decl with no name. Skip the part where we check against // the test name filter. if (!comp.bin_file.options.is_test) break :blk false; - if (decl_pkg != mod.main_pkg) break :blk false; + if (decl_mod != mod.main_mod) break :blk false; try mod.test_functions.put(gpa, new_decl_index, {}); break :blk true; }, else => blk: { if (!is_named_test) break :blk false; if (!comp.bin_file.options.is_test) break :blk false; - if (decl_pkg != mod.main_pkg) break :blk false; + if (decl_mod != mod.main_mod) break :blk false; if (comp.test_filter) |test_filter| { if (mem.indexOf(u8, ip.stringToSlice(decl_name), test_filter) == null) { break :blk false; @@ -5596,8 +5575,8 @@ pub fn populateTestFunctions( ) !void { const gpa = mod.gpa; const ip = &mod.intern_pool; - const builtin_pkg = mod.main_pkg.table.get("builtin").?; - const builtin_file = (mod.importPkg(builtin_pkg) catch unreachable).file; + const builtin_mod = mod.main_mod.deps.get("builtin").?; + const builtin_file = (mod.importPkg(builtin_mod) catch unreachable).file; const root_decl = mod.declPtr(builtin_file.root_decl.unwrap().?); const builtin_namespace = mod.namespacePtr(root_decl.src_namespace); const test_functions_str = try ip.getOrPutString(gpa, "test_functions"); diff --git a/src/Package.zig b/src/Package.zig index 14052e3de4be81ecc32af6d92c6ac248fcee43a3..c36830d5996e81552926630824dbc1c7fd0a52a6 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -1,1349 +1,153 @@ +pub const Module = @import("Package/Module.zig"); +pub const Fetch = @import("Package/Fetch.zig"); +pub const build_zig_basename = "build.zig"; +pub const Manifest = @import("Package/Manifest.zig"); + +pub const Path = struct { + root_dir: Cache.Directory, + /// The path, relative to the root dir, that this `Path` represents. + /// Empty string means the root_dir is the path. + sub_path: []const u8 = "", + + pub fn clone(p: Path, arena: Allocator) Allocator.Error!Path { + return .{ + .root_dir = try p.root_dir.clone(arena), + .sub_path = try arena.dupe(u8, p.sub_path), + }; + } + + pub fn cwd() Path { + return .{ .root_dir = Cache.Directory.cwd() }; + } + + pub fn join(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path { + if (sub_path.len == 0) return p; + const parts: []const []const u8 = + if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; + return .{ + .root_dir = p.root_dir, + .sub_path = try fs.path.join(arena, parts), + }; + } + + pub fn resolvePosix(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path { + if (sub_path.len == 0) return p; + return .{ + .root_dir = p.root_dir, + .sub_path = try fs.path.resolvePosix(arena, &.{ p.sub_path, sub_path }), + }; + } + + pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 { + const parts: []const []const u8 = + if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; + return p.root_dir.join(allocator, parts); + } + + pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![:0]u8 { + const parts: []const []const u8 = + if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; + return p.root_dir.joinZ(allocator, parts); + } + + pub fn openFile( + p: Path, + sub_path: []const u8, + flags: fs.File.OpenFlags, + ) !fs.File { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.openFile(joined_path, flags); + } + + pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.OpenDirOptions) !fs.Dir { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.makeOpenPath(joined_path, opts); + } + + pub fn statFile(p: Path, sub_path: []const u8) !fs.Dir.Stat { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.statFile(joined_path); + } + + pub fn atomicFile( + p: Path, + sub_path: []const u8, + options: fs.Dir.AtomicFileOptions, + ) !fs.AtomicFile { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.atomicFile(joined_path, options); + } + + pub fn access(p: Path, sub_path: []const u8, flags: fs.File.OpenFlags) !void { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.access(joined_path, flags); + } + + pub fn format( + self: Path, + comptime fmt_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + if (fmt_string.len == 1) { + // Quote-escape the string. + const stringEscape = std.zig.fmt.stringEscape; + const f = switch (fmt_string[0]) { + 'q' => "", + '\'' => '\'', + else => @compileError("unsupported format string: " ++ fmt_string), + }; + if (self.root_dir.path) |p| { + try stringEscape(p, f, options, writer); + if (self.sub_path.len > 0) try writer.writeAll(fs.path.sep_str); + } + if (self.sub_path.len > 0) { + try stringEscape(self.sub_path, f, options, writer); + } + return; + } + if (fmt_string.len > 0) + std.fmt.invalidFmtError(fmt_string, self); + if (self.root_dir.path) |p| { + try writer.writeAll(p); + try writer.writeAll(fs.path.sep_str); + } + if (self.sub_path.len > 0) { + try writer.writeAll(self.sub_path); + try writer.writeAll(fs.path.sep_str); + } + } +}; + const Package = @This(); - const builtin = @import("builtin"); const std = @import("std"); const fs = std.fs; -const mem = std.mem; -const Allocator = mem.Allocator; -const ascii = std.ascii; +const Allocator = std.mem.Allocator; const assert = std.debug.assert; -const log = std.log.scoped(.package); -const main = @import("main.zig"); -const ThreadPool = std.Thread.Pool; - -const Compilation = @import("Compilation.zig"); -const Module = @import("Module.zig"); const Cache = std.Build.Cache; -const build_options = @import("build_options"); -const git = @import("git.zig"); -const computePackageHash = @import("Package/hash.zig").compute; - -pub const Manifest = @import("Manifest.zig"); -pub const Table = std.StringHashMapUnmanaged(*Package); - -root_src_directory: Compilation.Directory, -/// Relative to `root_src_directory`. May contain path separators. -root_src_path: []const u8, -/// The dependency table of this module. Shared dependencies such as 'std', 'builtin', and 'root' -/// are not specified in every dependency table, but instead only in the table of `main_pkg`. -/// `Module.importFile` is responsible for detecting these names and using the correct package. -table: Table = .{}, -/// Whether to free `root_src_directory` on `destroy`. -root_src_directory_owned: bool = false, - -/// Allocate a Package. No references to the slices passed are kept. -pub fn create( - gpa: Allocator, - /// Null indicates the current working directory - root_src_dir_path: ?[]const u8, - /// Relative to root_src_dir_path - root_src_path: []const u8, -) !*Package { - const ptr = try gpa.create(Package); - errdefer gpa.destroy(ptr); - - const owned_dir_path = if (root_src_dir_path) |p| try gpa.dupe(u8, p) else null; - errdefer if (owned_dir_path) |p| gpa.free(p); - - const owned_src_path = try gpa.dupe(u8, root_src_path); - errdefer gpa.free(owned_src_path); - - ptr.* = .{ - .root_src_directory = .{ - .path = owned_dir_path, - .handle = if (owned_dir_path) |p| try fs.cwd().openDir(p, .{}) else fs.cwd(), - }, - .root_src_path = owned_src_path, - .root_src_directory_owned = true, - }; - - return ptr; -} - -pub fn createWithDir( - gpa: Allocator, - directory: Compilation.Directory, - /// Relative to `directory`. If null, means `directory` is the root src dir - /// and is owned externally. - root_src_dir_path: ?[]const u8, - /// Relative to root_src_dir_path - root_src_path: []const u8, -) !*Package { - const ptr = try gpa.create(Package); - errdefer gpa.destroy(ptr); - - const owned_src_path = try gpa.dupe(u8, root_src_path); - errdefer gpa.free(owned_src_path); - - if (root_src_dir_path) |p| { - const owned_dir_path = try directory.join(gpa, &[1][]const u8{p}); - errdefer gpa.free(owned_dir_path); - - ptr.* = .{ - .root_src_directory = .{ - .path = owned_dir_path, - .handle = try directory.handle.openDir(p, .{}), - }, - .root_src_directory_owned = true, - .root_src_path = owned_src_path, - }; - } else { - ptr.* = .{ - .root_src_directory = directory, - .root_src_directory_owned = false, - .root_src_path = owned_src_path, - }; - } - return ptr; -} - -/// Free all memory associated with this package. It does not destroy any packages -/// inside its table; the caller is responsible for calling destroy() on them. -pub fn destroy(pkg: *Package, gpa: Allocator) void { - gpa.free(pkg.root_src_path); - - if (pkg.root_src_directory_owned) { - // If root_src_directory.path is null then the handle is the cwd() - // which shouldn't be closed. - if (pkg.root_src_directory.path) |p| { - gpa.free(p); - pkg.root_src_directory.handle.close(); - } - } - - pkg.deinitTable(gpa); - gpa.destroy(pkg); -} - -/// Only frees memory associated with the table. -pub fn deinitTable(pkg: *Package, gpa: Allocator) void { - pkg.table.deinit(gpa); -} - -pub fn add(pkg: *Package, gpa: Allocator, name: []const u8, package: *Package) !void { - try pkg.table.ensureUnusedCapacity(gpa, 1); - const name_dupe = try gpa.dupe(u8, name); - pkg.table.putAssumeCapacityNoClobber(name_dupe, package); -} - -/// Compute a readable name for the package. The returned name should be freed from gpa. This -/// function is very slow, as it traverses the whole package hierarchy to find a path to this -/// package. It should only be used for error output. -pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8 { - // we'll do a breadth-first search from the root module to try and find a short name for this - // module, using a DoublyLinkedList of module/parent pairs. note that the "parent" there is - // just the first-found shortest path - a module may be children of arbitrarily many other - // modules. This path may vary between executions due to hashmap iteration order, but that - // doesn't matter too much. - var node_arena = std.heap.ArenaAllocator.init(gpa); - defer node_arena.deinit(); - const Parented = struct { - parent: ?*const @This(), - mod: *const Package, - }; - const Queue = std.DoublyLinkedList(Parented); - var to_check: Queue = .{}; - - { - const new = try node_arena.allocator().create(Queue.Node); - new.* = .{ .data = .{ .parent = null, .mod = mod.root_pkg } }; - to_check.prepend(new); - } - - if (mod.main_pkg != mod.root_pkg) { - const new = try node_arena.allocator().create(Queue.Node); - // TODO: once #12201 is resolved, we may want a way of indicating a different name for this - new.* = .{ .data = .{ .parent = null, .mod = mod.main_pkg } }; - to_check.prepend(new); - } - - // set of modules we've already checked to prevent loops - var checked = std.AutoHashMap(*const Package, void).init(gpa); - defer checked.deinit(); - - const linked = while (to_check.pop()) |node| { - const check = &node.data; - - if (checked.contains(check.mod)) continue; - try checked.put(check.mod, {}); - - if (check.mod == target) break check; - - var it = check.mod.table.iterator(); - while (it.next()) |kv| { - var new = try node_arena.allocator().create(Queue.Node); - new.* = .{ .data = .{ - .parent = check, - .mod = kv.value_ptr.*, - } }; - to_check.prepend(new); - } - } else { - // this can happen for e.g. @cImport packages - return gpa.dupe(u8, ""); - }; - - // we found a path to the module! unfortunately, we can only traverse *up* it, so we have to put - // all the names into a buffer so we can then print them in order. - var names = std.ArrayList([]const u8).init(gpa); - defer names.deinit(); - - var cur: *const Parented = linked; - while (cur.parent) |parent| : (cur = parent) { - // find cur's name in parent - var it = parent.mod.table.iterator(); - const name = while (it.next()) |kv| { - if (kv.value_ptr.* == cur.mod) { - break kv.key_ptr.*; - } - } else unreachable; - try names.append(name); - } - - // finally, print the names into a buffer! - var buf = std.ArrayList(u8).init(gpa); - defer buf.deinit(); - try buf.writer().writeAll("root"); - var i: usize = names.items.len; - while (i > 0) { - i -= 1; - try buf.writer().print(".{s}", .{names.items[i]}); - } - - return buf.toOwnedSlice(); -} - -pub const build_zig_basename = "build.zig"; - -/// Fetches a package and all of its dependencies recursively. Writes the -/// corresponding datastructures for the build runner into `dependencies_source`. -pub fn fetchAndAddDependencies( - pkg: *Package, - deps_pkg: *Package, - arena: Allocator, - thread_pool: *ThreadPool, - http_client: *std.http.Client, - directory: Compilation.Directory, - global_cache_directory: Compilation.Directory, - local_cache_directory: Compilation.Directory, - dependencies_source: *std.ArrayList(u8), - error_bundle: *std.zig.ErrorBundle.Wip, - all_modules: *AllModules, - root_prog_node: *std.Progress.Node, - /// null for the root package - this_hash: ?[]const u8, -) !void { - const max_bytes = 10 * 1024 * 1024; - const gpa = thread_pool.allocator; - const build_zig_zon_bytes = directory.handle.readFileAllocOptions( - arena, - Manifest.basename, - max_bytes, - null, - 1, - 0, - ) catch |err| switch (err) { - error.FileNotFound => { - // Handle the same as no dependencies. - if (this_hash) |hash| { - try dependencies_source.writer().print( - \\ pub const {} = struct {{ - \\ pub const build_root = "{}"; - \\ pub const build_zig = @import("{}"); - \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}}; - \\ }}; - \\ - , .{ - std.zig.fmtId(hash), - std.zig.fmtEscapes(pkg.root_src_directory.path.?), - std.zig.fmtEscapes(hash), - }); - } else { - try dependencies_source.writer().writeAll( - \\pub const packages = struct {}; - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; - \\ - ); - } - return; - }, - else => |e| return e, - }; - - var ast = try std.zig.Ast.parse(gpa, build_zig_zon_bytes, .zon); - defer ast.deinit(gpa); - - if (ast.errors.len > 0) { - const file_path = try directory.join(arena, &.{Manifest.basename}); - try main.putAstErrorsIntoBundle(gpa, ast, file_path, error_bundle); - return error.PackageFetchFailed; - } - - var manifest = try Manifest.parse(gpa, ast); - defer manifest.deinit(gpa); - - if (manifest.errors.len > 0) { - const file_path = try directory.join(arena, &.{Manifest.basename}); - for (manifest.errors) |msg| { - const str = try error_bundle.addString(msg.msg); - try Report.addErrorMessage(&ast, file_path, error_bundle, 0, str, msg.tok, msg.off); - } - return error.PackageFetchFailed; - } - - const report: Report = .{ - .ast = &ast, - .directory = directory, - .error_bundle = error_bundle, - }; - - for (manifest.dependencies.values()) |dep| { - // If the hash is invalid, let errors happen later - // We only want to add these for progress reporting - const hash = dep.hash orelse continue; - if (hash.len != hex_multihash_len) continue; - const gop = try all_modules.getOrPut(gpa, hash[0..hex_multihash_len].*); - if (!gop.found_existing) gop.value_ptr.* = null; - } - - root_prog_node.setEstimatedTotalItems(all_modules.count()); - - if (this_hash == null) { - try dependencies_source.writer().writeAll("pub const packages = struct {\n"); - } - - for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, *dep| { - var fetch_location = try FetchLocation.init(gpa, dep.*, directory, report); - defer fetch_location.deinit(gpa); - - // Directories do not provide a hash in build.zig.zon. - // Hash the path to the module rather than its contents. - const sub_mod, const found_existing = if (fetch_location == .directory) - try getDirectoryModule(gpa, fetch_location, directory, all_modules, dep, report) - else - try getCachedPackage( - gpa, - global_cache_directory, - dep.*, - all_modules, - root_prog_node, - ) orelse .{ - try fetchAndUnpack( - fetch_location, - thread_pool, - http_client, - directory, - global_cache_directory, - dep.*, - report, - all_modules, - root_prog_node, - name, - ), - false, - }; - - assert(dep.hash != null); - - switch (sub_mod) { - .zig_pkg => |sub_pkg| { - if (!found_existing) { - try sub_pkg.fetchAndAddDependencies( - deps_pkg, - arena, - thread_pool, - http_client, - sub_pkg.root_src_directory, - global_cache_directory, - local_cache_directory, - dependencies_source, - error_bundle, - all_modules, - root_prog_node, - dep.hash.?, - ); - } - - try pkg.add(gpa, name, sub_pkg); - if (deps_pkg.table.get(dep.hash.?)) |other_sub| { - // This should be the same package (and hence module) since it's the same hash - // TODO: dedup multiple versions of the same package - assert(other_sub == sub_pkg); - } else { - try deps_pkg.add(gpa, dep.hash.?, sub_pkg); - } - }, - .non_zig_pkg => |sub_pkg| { - if (!found_existing) { - try dependencies_source.writer().print( - \\ pub const {} = struct {{ - \\ pub const build_root = "{}"; - \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}}; - \\ }}; - \\ - , .{ - std.zig.fmtId(dep.hash.?), - std.zig.fmtEscapes(sub_pkg.root_src_directory.path.?), - }); - } - }, - } - } - - if (this_hash) |hash| { - try dependencies_source.writer().print( - \\ pub const {} = struct {{ - \\ pub const build_root = "{}"; - \\ pub const build_zig = @import("{}"); - \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{ - \\ - , .{ - std.zig.fmtId(hash), - std.zig.fmtEscapes(pkg.root_src_directory.path.?), - std.zig.fmtEscapes(hash), - }); - for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { - try dependencies_source.writer().print( - " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, - ); - } - try dependencies_source.writer().writeAll( - \\ }; - \\ }; - \\ - ); - } else { - try dependencies_source.writer().writeAll( - \\}; - \\ - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ - \\ - ); - for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { - try dependencies_source.writer().print( - " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, - ); - } - try dependencies_source.writer().writeAll("};\n"); - } -} - -pub fn createFilePkg( - gpa: Allocator, - cache_directory: Compilation.Directory, - basename: []const u8, - contents: []const u8, -) !*Package { - const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Manifest.hex64(rand_int); - { - var tmp_dir = try cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); - defer tmp_dir.close(); - try tmp_dir.writeFile(basename, contents); - } - - var hh: Cache.HashHelper = .{}; - hh.addBytes(build_options.version); - hh.addBytes(contents); - const hex_digest = hh.final(); - - const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest; - try renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path); - - return createWithDir(gpa, cache_directory, o_dir_sub_path, basename); -} - -pub const Report = struct { - ast: ?*const std.zig.Ast, - directory: Compilation.Directory, - error_bundle: *std.zig.ErrorBundle.Wip, - - fn fail( - report: Report, - tok: std.zig.Ast.TokenIndex, - comptime fmt_string: []const u8, - fmt_args: anytype, - ) error{ PackageFetchFailed, OutOfMemory } { - const msg = try report.error_bundle.printString(fmt_string, fmt_args); - return failMsg(report, tok, msg); - } - - fn failMsg( - report: Report, - tok: std.zig.Ast.TokenIndex, - msg: u32, - ) error{ PackageFetchFailed, OutOfMemory } { - const gpa = report.error_bundle.gpa; - - const file_path = try report.directory.join(gpa, &.{Manifest.basename}); - defer gpa.free(file_path); - - const eb = report.error_bundle; - - if (report.ast) |ast| { - try addErrorMessage(ast, file_path, eb, 0, msg, tok, 0); - } else { - try eb.addRootErrorMessage(.{ - .msg = msg, - .src_loc = .none, - .notes_len = 0, - }); - } - - return error.PackageFetchFailed; - } - - fn addErrorWithNotes( - report: Report, - notes_len: u32, - msg: Manifest.ErrorMessage, - ) error{OutOfMemory}!void { - const eb = report.error_bundle; - const msg_str = try eb.addString(msg.msg); - if (report.ast) |ast| { - const gpa = eb.gpa; - const file_path = try report.directory.join(gpa, &.{Manifest.basename}); - defer gpa.free(file_path); - return addErrorMessage(ast, file_path, eb, notes_len, msg_str, msg.tok, msg.off); - } else { - return eb.addRootErrorMessage(.{ - .msg = msg_str, - .src_loc = .none, - .notes_len = notes_len, - }); - } - } - - fn addErrorMessage( - ast: *const std.zig.Ast, - file_path: []const u8, - eb: *std.zig.ErrorBundle.Wip, - notes_len: u32, - msg_str: u32, - msg_tok: std.zig.Ast.TokenIndex, - msg_off: u32, - ) error{OutOfMemory}!void { - const token_starts = ast.tokens.items(.start); - const start_loc = ast.tokenLocation(0, msg_tok); - - try eb.addRootErrorMessage(.{ - .msg = msg_str, - .src_loc = try eb.addSourceLocation(.{ - .src_path = try eb.addString(file_path), - .span_start = token_starts[msg_tok], - .span_end = @as(u32, @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len)), - .span_main = token_starts[msg_tok] + msg_off, - .line = @intCast(start_loc.line), - .column = @as(u32, @intCast(start_loc.column)), - .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), - }), - .notes_len = notes_len, - }); - } -}; - -pub const FetchLocation = union(enum) { - /// The relative path to a file or directory. - /// This may be a file that requires unpacking (such as a .tar.gz), - /// or the path to the root directory of a package. - file: []const u8, - directory: []const u8, - http_request: std.Uri, - git_request: std.Uri, - - pub fn init( - gpa: Allocator, - dep: Manifest.Dependency, - root_dir: Compilation.Directory, - report: Report, - ) !FetchLocation { - switch (dep.location) { - .url => |url| { - const uri = std.Uri.parse(url) catch |err| switch (err) { - error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI", .{}), - else => return err, - }; - return initUri(uri, dep.location_tok, report); - }, - .path => |path| { - if (fs.path.isAbsolute(path)) { - return report.fail(dep.location_tok, "absolute paths are not allowed. Use a relative path instead", .{}); - } - - const is_dir = isDirectory(root_dir, path) catch |err| switch (err) { - error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{path}), - else => return err, - }; - - return if (is_dir) - .{ .directory = try gpa.dupe(u8, path) } - else - .{ .file = try gpa.dupe(u8, path) }; - }, - } - } - - pub fn initUri(uri: std.Uri, location_tok: std.zig.Ast.TokenIndex, report: Report) !FetchLocation { - if (ascii.eqlIgnoreCase(uri.scheme, "file")) { - return report.fail(location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{}); - } else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) { - return .{ .http_request = uri }; - } else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) { - return .{ .git_request = uri }; - } else { - return report.fail(location_tok, "unsupported URL scheme: {s}", .{uri.scheme}); - } - } - - pub fn deinit(f: *FetchLocation, gpa: Allocator) void { - switch (f.*) { - .file, .directory => |path| gpa.free(path), - .http_request, .git_request => {}, - } - f.* = undefined; - } - - pub fn fetch( - f: FetchLocation, - gpa: Allocator, - root_dir: Compilation.Directory, - http_client: *std.http.Client, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, - ) !ReadableResource { - switch (f) { - .file => |file| { - const owned_path = try gpa.dupe(u8, file); - errdefer gpa.free(owned_path); - return .{ - .path = owned_path, - .resource = .{ .file = try root_dir.handle.openFile(file, .{}) }, - }; - }, - .http_request => |uri| { - var h = std.http.Headers{ .allocator = gpa }; - defer h.deinit(); - - var req = try http_client.request(.GET, uri, h, .{}); - errdefer req.deinit(); - - try req.start(.{}); - try req.wait(); - - if (req.response.status != .ok) { - return report.fail(dep_location_tok, "expected response status '200 OK' got '{} {s}'", .{ - @intFromEnum(req.response.status), - req.response.status.phrase() orelse "", - }); - } - - return .{ - .path = try gpa.dupe(u8, uri.path), - .resource = .{ .http_request = req }, - }; - }, - .git_request => |uri| { - var transport_uri = uri; - transport_uri.scheme = uri.scheme["git+".len..]; - var redirect_uri: []u8 = undefined; - var session: git.Session = .{ .transport = http_client, .uri = transport_uri }; - session.discoverCapabilities(gpa, &redirect_uri) catch |e| switch (e) { - error.Redirected => { - defer gpa.free(redirect_uri); - return report.fail(dep_location_tok, "repository moved to {s}", .{redirect_uri}); - }, - else => |other| return other, - }; - - const want_oid = want_oid: { - const want_ref = uri.fragment orelse "HEAD"; - if (git.parseOid(want_ref)) |oid| break :want_oid oid else |_| {} - - const want_ref_head = try std.fmt.allocPrint(gpa, "refs/heads/{s}", .{want_ref}); - defer gpa.free(want_ref_head); - const want_ref_tag = try std.fmt.allocPrint(gpa, "refs/tags/{s}", .{want_ref}); - defer gpa.free(want_ref_tag); - - var ref_iterator = try session.listRefs(gpa, .{ - .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, - .include_peeled = true, - }); - defer ref_iterator.deinit(); - while (try ref_iterator.next()) |ref| { - if (mem.eql(u8, ref.name, want_ref) or - mem.eql(u8, ref.name, want_ref_head) or - mem.eql(u8, ref.name, want_ref_tag)) - { - break :want_oid ref.peeled orelse ref.oid; - } - } - return report.fail(dep_location_tok, "ref not found: {s}", .{want_ref}); - }; - if (uri.fragment == null) { - const notes_len = 1; - try report.addErrorWithNotes(notes_len, .{ - .tok = dep_location_tok, - .off = 0, - .msg = "url field is missing an explicit ref", - }); - const eb = report.error_bundle; - const notes_start = try eb.reserveNotes(notes_len); - eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }), - })); - return error.PackageFetchFailed; - } - - var want_oid_buf: [git.fmt_oid_length]u8 = undefined; - _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{std.fmt.fmtSliceHexLower(&want_oid)}) catch unreachable; - var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf}); - errdefer fetch_stream.deinit(); - - return .{ - .path = try gpa.dupe(u8, &want_oid_buf), - .resource = .{ .git_fetch_stream = fetch_stream }, - }; - }, - .directory => unreachable, // Directories do not require fetching - } - } -}; - -pub const ReadableResource = struct { - path: []const u8, - resource: union(enum) { - file: fs.File, - http_request: std.http.Client.Request, - git_fetch_stream: git.Session.FetchStream, - dir: fs.IterableDir, - }, - - /// Unpack the package into the global cache directory. - /// If `ps` does not require unpacking (for example, if it is a directory), then no caching is performed. - /// In either case, the hash is computed and returned along with the path to the package. - pub fn unpack( - rr: *ReadableResource, - allocator: Allocator, - thread_pool: *ThreadPool, - global_cache_directory: Compilation.Directory, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, - pkg_prog_node: *std.Progress.Node, - ) !PackageLocation { - switch (rr.resource) { - inline .file, .http_request, .git_fetch_stream, .dir => |*r, tag| { - const s = fs.path.sep_str; - const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); - - const actual_hash = h: { - var tmp_directory: Compilation.Directory = d: { - const path = try global_cache_directory.join(allocator, &.{tmp_dir_sub_path}); - errdefer allocator.free(path); - - const iterable_dir = try global_cache_directory.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}); - errdefer iterable_dir.close(); - - break :d .{ - .path = path, - .handle = iterable_dir.dir, - }; - }; - defer tmp_directory.closeAndFree(allocator); - - if (tag != .dir) { - const opt_content_length = try rr.getSize(); - - var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{ - .child_reader = r.reader(), - .prog_node = pkg_prog_node, - .unit = if (opt_content_length) |content_length| unit: { - const kib = content_length / 1024; - const mib = kib / 1024; - if (mib > 0) { - pkg_prog_node.setEstimatedTotalItems(@intCast(mib)); - pkg_prog_node.setUnit("MiB"); - break :unit .mib; - } else { - pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib))); - pkg_prog_node.setUnit("KiB"); - break :unit .kib; - } - } else .any, - }; - - switch (try rr.getFileType(dep_location_tok, report)) { - .tar => try unpackTarball(allocator, prog_reader.reader(), tmp_directory.handle, dep_location_tok, report), - .@"tar.gz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.gzip), - .@"tar.xz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.xz), - .git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle, dep_location_tok, report), - } - } else { - // Recursive directory copy. - var it = try r.walk(allocator); - defer it.deinit(); - while (try it.next()) |entry| { - switch (entry.kind) { - .directory => try tmp_directory.handle.makePath(entry.path), - .file => try r.dir.copyFile( - entry.path, - tmp_directory.handle, - entry.path, - .{}, - ), - .sym_link => { - var buf: [fs.MAX_PATH_BYTES]u8 = undefined; - const link_name = try r.dir.readLink(entry.path, &buf); - // TODO: if this would create a symlink to outside - // the destination directory, fail with an error instead. - try tmp_directory.handle.symLink(link_name, entry.path, .{}); - }, - else => return error.IllegalFileTypeInPackage, - } - } - } - - break :h try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle }); - }; - - const pkg_dir_sub_path = "p" ++ s ++ Manifest.hexDigest(actual_hash); - const unpacked_path = try global_cache_directory.join(allocator, &.{pkg_dir_sub_path}); - defer allocator.free(unpacked_path); - - const relative_unpacked_path = try fs.path.relative(allocator, global_cache_directory.path.?, unpacked_path); - errdefer allocator.free(relative_unpacked_path); - try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, relative_unpacked_path); - - return .{ - .hash = actual_hash, - .relative_unpacked_path = relative_unpacked_path, - }; - }, - } - } - - const FileType = enum { - tar, - @"tar.gz", - @"tar.xz", - git_pack, - }; - - pub fn getSize(rr: ReadableResource) !?u64 { - switch (rr.resource) { - .file => |f| return (try f.metadata()).size(), - // TODO: Handle case of chunked content-length - .http_request => |req| return req.response.content_length, - .git_fetch_stream => |stream| return stream.request.response.content_length, - .dir => unreachable, - } - } - - pub fn getFileType( - rr: ReadableResource, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, - ) !FileType { - switch (rr.resource) { - .file => { - return fileTypeFromPath(rr.path) orelse - return report.fail(dep_location_tok, "unknown file type", .{}); - }, - .http_request => |req| { - const content_type = req.response.headers.getFirstValue("Content-Type") orelse - return report.fail(dep_location_tok, "missing 'Content-Type' header", .{}); - - // If the response has a different content type than the URI indicates, override - // the previously assumed file type. - if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) return .tar; - - return if (ascii.eqlIgnoreCase(content_type, "application/gzip") or - ascii.eqlIgnoreCase(content_type, "application/x-gzip") or - ascii.eqlIgnoreCase(content_type, "application/tar+gzip")) - .@"tar.gz" - else if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) - .@"tar.xz" - else if (ascii.eqlIgnoreCase(content_type, "application/octet-stream")) ty: { - // support gitlab tarball urls such as https://gitlab.com///-/archive//-.tar.gz - // whose content-disposition header is: 'attachment; filename="-.tar.gz"' - const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse - return report.fail(dep_location_tok, "missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{}); - break :ty getAttachmentType(content_disposition) orelse - return report.fail(dep_location_tok, "unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition}); - } else return report.fail(dep_location_tok, "unrecognized value for 'Content-Type' header: {s}", .{content_type}); - }, - .git_fetch_stream => return .git_pack, - .dir => unreachable, - } - } - - fn fileTypeFromPath(file_path: []const u8) ?FileType { - if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; - if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; - if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; - return null; - } - - fn getAttachmentType(content_disposition: []const u8) ?FileType { - const disposition_type_end = ascii.indexOfIgnoreCase(content_disposition, "attachment;") orelse return null; - - var value_start = ascii.indexOfIgnoreCasePos(content_disposition, disposition_type_end + 1, "filename") orelse return null; - value_start += "filename".len; - if (content_disposition[value_start] == '*') { - value_start += 1; - } - if (content_disposition[value_start] != '=') return null; - value_start += 1; - - var value_end = mem.indexOfPos(u8, content_disposition, value_start, ";") orelse content_disposition.len; - if (content_disposition[value_end - 1] == '\"') { - value_end -= 1; - } - return fileTypeFromPath(content_disposition[value_start..value_end]); - } - - pub fn deinit(rr: *ReadableResource, gpa: Allocator) void { - gpa.free(rr.path); - switch (rr.resource) { - .file => |file| file.close(), - .http_request => |*req| req.deinit(), - .git_fetch_stream => |*stream| stream.deinit(), - .dir => |*dir| dir.close(), - } - rr.* = undefined; - } -}; - -pub const PackageLocation = struct { - /// For packages that require unpacking, this is the hash of the package contents. - /// For directories, this is the hash of the absolute file path. - hash: [Manifest.Hash.digest_length]u8, - relative_unpacked_path: []const u8, - - pub fn deinit(pl: *PackageLocation, allocator: Allocator) void { - allocator.free(pl.relative_unpacked_path); - pl.* = undefined; - } -}; - -const hex_multihash_len = 2 * Manifest.multihash_len; -const MultiHashHexDigest = [hex_multihash_len]u8; - -const DependencyModule = union(enum) { - zig_pkg: *Package, - non_zig_pkg: *Package, -}; -/// This is to avoid creating multiple modules for the same build.zig file. -/// If the value is `null`, the package is a known dependency, but has not yet -/// been fetched. -pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?DependencyModule); - -fn ProgressReader(comptime ReaderType: type) type { - return struct { - child_reader: ReaderType, - bytes_read: u64 = 0, - prog_node: *std.Progress.Node, - unit: enum { - kib, - mib, - any, - }, - - pub const Error = ReaderType.Error; - pub const Reader = std.io.Reader(*@This(), Error, read); - - pub fn read(self: *@This(), buf: []u8) Error!usize { - const amt = try self.child_reader.read(buf); - self.bytes_read += amt; - const kib = self.bytes_read / 1024; - const mib = kib / 1024; - switch (self.unit) { - .kib => self.prog_node.setCompletedItems(@intCast(kib)), - .mib => self.prog_node.setCompletedItems(@intCast(mib)), - .any => { - if (mib > 0) { - self.prog_node.setUnit("MiB"); - self.prog_node.setCompletedItems(@intCast(mib)); - } else { - self.prog_node.setUnit("KiB"); - self.prog_node.setCompletedItems(@intCast(kib)); - } - }, - } - self.prog_node.activate(); - return amt; - } - - pub fn reader(self: *@This()) Reader { - return .{ .context = self }; - } - }; -} - -/// Get a cached package if it exists. -/// Returns `null` if the package has not been cached -/// If the package exists in the cache, returns a pointer to the package and a -/// boolean indicating whether this package has already been seen in the build -/// (i.e. whether or not its transitive dependencies have been fetched). -fn getCachedPackage( - gpa: Allocator, - global_cache_directory: Compilation.Directory, - dep: Manifest.Dependency, - all_modules: *AllModules, - root_prog_node: *std.Progress.Node, -) !?struct { DependencyModule, bool } { - const s = fs.path.sep_str; - // Check if the expected_hash is already present in the global package - // cache, and thereby avoid both fetching and unpacking. - if (dep.hash) |h| { - const hex_digest = h[0..hex_multihash_len]; - const pkg_dir_sub_path = "p" ++ s ++ hex_digest; - - var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) { - error.FileNotFound => return null, - else => |e| return e, - }; - errdefer pkg_dir.close(); - - // The compiler has a rule that a file must not be included in multiple modules, - // so we must detect if a module has been created for this package and reuse it. - const gop = try all_modules.getOrPut(gpa, hex_digest.*); - if (gop.found_existing) { - if (gop.value_ptr.*) |mod| { - return .{ mod, true }; - } - } - - root_prog_node.completeOne(); - - const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false; - const basename = if (is_zig_mod) build_zig_basename else ""; - const pkg = try createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, basename); - - const module: DependencyModule = if (is_zig_mod) - .{ .zig_pkg = pkg } - else - .{ .non_zig_pkg = pkg }; - - try all_modules.put(gpa, hex_digest.*, module); - return .{ module, false }; - } - - return null; -} - -fn getDirectoryModule( - gpa: Allocator, - fetch_location: FetchLocation, - directory: Compilation.Directory, - all_modules: *AllModules, - dep: *Manifest.Dependency, - report: Report, -) !struct { DependencyModule, bool } { - assert(fetch_location == .directory); - - if (dep.hash != null) { - return report.fail(dep.hash_tok, "hash not allowed for directory package", .{}); - } - - const hash = try computePathHash(gpa, directory, fetch_location.directory); - const hex_digest = Manifest.hexDigest(hash); - dep.hash = try gpa.dupe(u8, &hex_digest); - - // There is no fixed location to check for directory modules. - // Instead, check whether it is already listed in all_modules. - if (all_modules.get(hex_digest)) |mod| return .{ mod.?, true }; - - var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch |err| switch (err) { - error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{fetch_location.directory}), - else => |e| return e, - }; - defer pkg_dir.close(); - - const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false; - const basename = if (is_zig_mod) build_zig_basename else ""; - - const pkg = try createWithDir(gpa, directory, fetch_location.directory, basename); - const module: DependencyModule = if (is_zig_mod) - .{ .zig_pkg = pkg } - else - .{ .non_zig_pkg = pkg }; - - try all_modules.put(gpa, hex_digest, module); - return .{ module, false }; -} - -fn fetchAndUnpack( - fetch_location: FetchLocation, - thread_pool: *ThreadPool, - http_client: *std.http.Client, - directory: Compilation.Directory, - global_cache_directory: Compilation.Directory, - dep: Manifest.Dependency, - report: Report, - all_modules: *AllModules, - root_prog_node: *std.Progress.Node, - /// This does not have to be any form of canonical or fully-qualified name: it - /// is only intended to be human-readable for progress reporting. - name_for_prog: []const u8, -) !DependencyModule { - assert(fetch_location != .directory); - - const gpa = http_client.allocator; - - var pkg_prog_node = root_prog_node.start(name_for_prog, 0); - defer pkg_prog_node.end(); - pkg_prog_node.activate(); - - var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep.location_tok, report); - defer readable_resource.deinit(gpa); - - var package_location = try readable_resource.unpack( - gpa, - thread_pool, - global_cache_directory, - dep.location_tok, - report, - &pkg_prog_node, - ); - defer package_location.deinit(gpa); - - const actual_hex = Manifest.hexDigest(package_location.hash); - if (dep.hash) |h| { - if (!mem.eql(u8, h, &actual_hex)) { - return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{ - h, actual_hex, - }); - } - } else { - const notes_len = 1; - try report.addErrorWithNotes(notes_len, .{ - .tok = dep.location_tok, - .off = 0, - .msg = "dependency is missing hash field", - }); - const eb = report.error_bundle; - const notes_start = try eb.reserveNotes(notes_len); - eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), - })); - return error.PackageFetchFailed; - } - - const build_zig_path = try fs.path.join(gpa, &.{ package_location.relative_unpacked_path, build_zig_basename }); - defer gpa.free(build_zig_path); - - const is_zig_mod = if (global_cache_directory.handle.access(build_zig_path, .{})) |_| true else |_| false; - const basename = if (is_zig_mod) build_zig_basename else ""; - const pkg = try createWithDir(gpa, global_cache_directory, package_location.relative_unpacked_path, basename); - const module: DependencyModule = if (is_zig_mod) - .{ .zig_pkg = pkg } - else - .{ .non_zig_pkg = pkg }; - - try all_modules.put(gpa, actual_hex, module); - return module; -} - -fn unpackTarballCompressed( - gpa: Allocator, - reader: anytype, - out_dir: fs.Dir, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, - comptime Compression: type, -) !void { - var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); - - var decompress = try Compression.decompress(gpa, br.reader()); - defer decompress.deinit(); - - return unpackTarball(gpa, decompress.reader(), out_dir, dep_location_tok, report); -} - -fn unpackTarball( - gpa: Allocator, - reader: anytype, - out_dir: fs.Dir, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, -) !void { - var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa }; - defer diagnostics.deinit(); - - try std.tar.pipeToFileSystem(out_dir, reader, .{ - .diagnostics = &diagnostics, - .strip_components = 1, - // TODO: we would like to set this to executable_bit_only, but two - // things need to happen before that: - // 1. the tar implementation needs to support it - // 2. the hashing algorithm here needs to support detecting the is_executable - // bit on Windows from the ACLs (see the isExecutable function). - .mode_mode = .ignore, - }); - - if (diagnostics.errors.items.len > 0) { - const notes_len: u32 = @intCast(diagnostics.errors.items.len); - try report.addErrorWithNotes(notes_len, .{ - .tok = dep_location_tok, - .off = 0, - .msg = "unable to unpack tarball", - }); - const eb = report.error_bundle; - const notes_start = try eb.reserveNotes(notes_len); - for (diagnostics.errors.items, notes_start..) |item, note_i| { - switch (item) { - .unable_to_create_sym_link => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ - info.file_name, info.link_name, @errorName(info.code), - }), - })); - }, - .unsupported_file_type => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ - info.file_name, @intFromEnum(info.file_type), - }), - })); - }, - } - } - return error.InvalidTarball; - } -} - -fn unpackGitPack( - gpa: Allocator, - reader: anytype, - want_oid: git.Oid, - out_dir: fs.Dir, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, -) !void { - // The .git directory is used to store the packfile and associated index, but - // we do not attempt to replicate the exact structure of a real .git - // directory, since that isn't relevant for fetching a package. - { - var pack_dir = try out_dir.makeOpenPath(".git", .{}); - defer pack_dir.close(); - var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); - defer pack_file.close(); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(reader.reader(), pack_file.writer()); - try pack_file.sync(); - - var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); - defer index_file.close(); - { - var index_prog_node = reader.prog_node.start("Index pack", 0); - defer index_prog_node.end(); - index_prog_node.activate(); - var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); - try git.indexPack(gpa, pack_file, index_buffered_writer.writer()); - try index_buffered_writer.flush(); - try index_file.sync(); - } - - { - var checkout_prog_node = reader.prog_node.start("Checkout", 0); - defer checkout_prog_node.end(); - checkout_prog_node.activate(); - var repository = try git.Repository.init(gpa, pack_file, index_file); - defer repository.deinit(); - var diagnostics: git.Diagnostics = .{ .allocator = gpa }; - defer diagnostics.deinit(); - try repository.checkout(out_dir, want_oid, &diagnostics); - - if (diagnostics.errors.items.len > 0) { - const notes_len: u32 = @intCast(diagnostics.errors.items.len); - try report.addErrorWithNotes(notes_len, .{ - .tok = dep_location_tok, - .off = 0, - .msg = "unable to unpack packfile", - }); - const eb = report.error_bundle; - const notes_start = try eb.reserveNotes(notes_len); - for (diagnostics.errors.items, notes_start..) |item, note_i| { - switch (item) { - .unable_to_create_sym_link => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ - info.file_name, info.link_name, @errorName(info.code), - }), - })); - }, - } - } - return error.InvalidGitPack; - } - } - } - - try out_dir.deleteTree(".git"); -} - -/// Compute the hash of a file path. -fn computePathHash(gpa: Allocator, dir: Compilation.Directory, path: []const u8) ![Manifest.Hash.digest_length]u8 { - const resolved_path = try std.fs.path.resolve(gpa, &.{ dir.path.?, path }); - defer gpa.free(resolved_path); - var hasher = Manifest.Hash.init(.{}); - hasher.update(resolved_path); - return hasher.finalResult(); -} - -fn isDirectory(root_dir: Compilation.Directory, path: []const u8) !bool { - var dir = root_dir.handle.openDir(path, .{}) catch |err| switch (err) { - error.NotDir => return false, - else => return err, - }; - defer dir.close(); - return true; -} - -fn renameTmpIntoCache( - cache_dir: fs.Dir, - tmp_dir_sub_path: []const u8, - dest_dir_sub_path: []const u8, -) !void { - assert(dest_dir_sub_path[1] == fs.path.sep); - var handled_missing_dir = false; - while (true) { - cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { - error.FileNotFound => { - if (handled_missing_dir) return err; - cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { - error.PathAlreadyExists => handled_missing_dir = true, - else => |e| return e, - }; - continue; - }, - error.PathAlreadyExists, error.AccessDenied => { - // Package has been already downloaded and may already be in use on the system. - cache_dir.deleteTree(tmp_dir_sub_path) catch |del_err| { - std.log.warn("unable to delete temp directory: {s}", .{@errorName(del_err)}); - }; - }, - else => |e| return e, - }; - break; - } -} - -test "getAttachmentType" { - try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); - try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attachment; filename*=\"stuff.tar.gz\"")); - try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.xz"), ReadableResource.getAttachmentType("ATTACHMENT; filename=\"stuff.tar.xz\"")); - try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.xz"), ReadableResource.getAttachmentType("attachment; FileName=\"stuff.tar.xz\"")); - try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); - - try std.testing.expect(ReadableResource.getAttachmentType("attachment FileName=\"stuff.tar.gz\"") == null); - try std.testing.expect(ReadableResource.getAttachmentType("attachment; FileName=\"stuff.tar\"") == null); - try std.testing.expect(ReadableResource.getAttachmentType("attachment; FileName\"stuff.gz\"") == null); - try std.testing.expect(ReadableResource.getAttachmentType("attachment; size=42") == null); - try std.testing.expect(ReadableResource.getAttachmentType("inline; size=42") == null); - try std.testing.expect(ReadableResource.getAttachmentType("FileName=\"stuff.tar.gz\"; attachment;") == null); - try std.testing.expect(ReadableResource.getAttachmentType("FileName=\"stuff.tar.gz\";") == null); -} diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig new file mode 100644 index 0000000000000000000000000000000000000000..3aae5a533fc3947edaa348f47b4b24f3a72b3d9e --- /dev/null +++ b/src/Package/Fetch.zig @@ -0,0 +1,1557 @@ +//! Represents one independent job whose responsibility is to: +//! +//! 1. Check the global zig package cache to see if the hash already exists. +//! If so, load, parse, and validate the build.zig.zon file therein, and +//! goto step 8. Likewise if the location is a relative path, treat this +//! the same as a cache hit. Otherwise, proceed. +//! 2. Fetch and unpack a URL into a temporary directory. +//! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed +//! for the file to be missing, in which case this fetched package is considered +//! to be a "naked" package. +//! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by +//! deleting excluded files. If any files had errors for files that were +//! ultimately excluded, those errors should be ignored, such as failure to +//! create symlinks that weren't supposed to be included anyway. +//! 5. Compute the package hash based on the remaining files in the temporary +//! directory. +//! 6. Rename the temporary directory into the global zig package cache +//! directory. If the hash already exists, delete the temporary directory and +//! leave the zig package cache directory untouched as it may be in use by the +//! system. This is done even if the hash is invalid, in case the package with +//! the different hash is used in the future. +//! 7. Validate the computed hash against the expected hash. If invalid, +//! this job is done. +//! 8. Spawn a new fetch job for each dependency in the manifest file. Use +//! a mutex and a hash map so that redundant jobs do not get queued up. +//! +//! All of this must be done with only referring to the state inside this struct +//! because this work will be done in a dedicated thread. + +arena: std.heap.ArenaAllocator, +location: Location, +location_tok: std.zig.Ast.TokenIndex, +hash_tok: std.zig.Ast.TokenIndex, +parent_package_root: Package.Path, +parent_manifest_ast: ?*const std.zig.Ast, +prog_node: *std.Progress.Node, +job_queue: *JobQueue, +/// If true, don't add an error for a missing hash. This flag is not passed +/// down to recursive dependencies. It's intended to be used only be the CLI. +omit_missing_hash_error: bool, +/// If true, don't fail when a manifest file is missing the `paths` field, +/// which specifies inclusion rules. This is intended to be true for the first +/// fetch task and false for the recursive dependencies. +allow_missing_paths_field: bool, + +// Above this are fields provided as inputs to `run`. +// Below this are fields populated by `run`. + +/// This will either be relative to `global_cache`, or to the build root of +/// the root package. +package_root: Package.Path, +error_bundle: ErrorBundle.Wip, +manifest: ?Manifest, +manifest_ast: std.zig.Ast, +actual_hash: Manifest.Digest, +/// Fetch logic notices whether a package has a build.zig file and sets this flag. +has_build_zig: bool, +/// Indicates whether the task aborted due to an out-of-memory condition. +oom_flag: bool, + +// This field is used by the CLI only, untouched by this file. + +/// The module for this `Fetch` tasks's package, which exposes `build.zig` as +/// the root source file. +module: ?*Package.Module, + +/// Contains shared state among all `Fetch` tasks. +pub const JobQueue = struct { + mutex: std.Thread.Mutex = .{}, + /// It's an array hash map so that it can be sorted before rendering the + /// dependencies.zig source file. + /// Protected by `mutex`. + table: Table = .{}, + /// `table` may be missing some tasks such as ones that failed, so this + /// field contains references to all of them. + /// Protected by `mutex`. + all_fetches: std.ArrayListUnmanaged(*Fetch) = .{}, + + http_client: *std.http.Client, + thread_pool: *ThreadPool, + wait_group: WaitGroup = .{}, + global_cache: Cache.Directory, + recursive: bool, + work_around_btrfs_bug: bool, + + pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch); + + pub fn deinit(jq: *JobQueue) void { + if (jq.all_fetches.items.len == 0) return; + const gpa = jq.all_fetches.items[0].arena.child_allocator; + jq.table.deinit(gpa); + // These must be deinitialized in reverse order because subsequent + // `Fetch` instances are allocated in prior ones' arenas. + // Sorry, I know it's a bit weird, but it slightly simplifies the + // critical section. + while (jq.all_fetches.popOrNull()) |f| f.deinit(); + jq.all_fetches.deinit(gpa); + jq.* = undefined; + } + + /// Dumps all subsequent error bundles into the first one. + pub fn consolidateErrors(jq: *JobQueue) !void { + const root = &jq.all_fetches.items[0].error_bundle; + const gpa = root.gpa; + for (jq.all_fetches.items[1..]) |fetch| { + if (fetch.error_bundle.root_list.items.len > 0) { + var bundle = try fetch.error_bundle.toOwnedBundle(""); + defer bundle.deinit(gpa); + try root.addBundleAsRoots(bundle); + } + } + } + + /// Creates the dependencies.zig source code for the build runner to obtain + /// via `@import("@dependencies")`. + pub fn createDependenciesSource(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { + const keys = jq.table.keys(); + + assert(keys.len != 0); // caller should have added the first one + if (keys.len == 1) { + // This is the first one. It must have no dependencies. + return createEmptyDependenciesSource(buf); + } + + try buf.appendSlice("pub const packages = struct {\n"); + + // Ensure the generated .zig file is deterministic. + jq.table.sortUnstable(@as(struct { + keys: []const Manifest.MultiHashHexDigest, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return std.mem.lessThan(u8, &ctx.keys[a_index], &ctx.keys[b_index]); + } + }, .{ .keys = keys })); + + for (keys, jq.table.values()) |hash, fetch| { + if (fetch == jq.all_fetches.items[0]) { + // The first one is a dummy package for the current project. + continue; + } + try buf.writer().print( + \\ pub const {} = struct {{ + \\ pub const build_root = "{q}"; + \\ + , .{ std.zig.fmtId(&hash), fetch.package_root }); + + if (fetch.has_build_zig) { + try buf.writer().print( + \\ pub const build_zig = @import("{}"); + \\ + , .{std.zig.fmtEscapes(&hash)}); + } + + if (fetch.manifest) |*manifest| { + try buf.appendSlice( + \\ pub const deps: []const struct { []const u8, []const u8 } = &.{ + \\ + ); + for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { + const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; + try buf.writer().print( + " .{{ \"{}\", \"{}\" }},\n", + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) }, + ); + } + + try buf.appendSlice( + \\ }; + \\ }; + \\ + ); + } else { + try buf.appendSlice( + \\ pub const deps: []const struct { []const u8, []const u8 } = &.{}; + \\ }; + \\ + ); + } + } + + try buf.appendSlice( + \\}; + \\ + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ + \\ + ); + + const root_fetch = jq.all_fetches.items[0]; + const root_manifest = &root_fetch.manifest.?; + + for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { + const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; + try buf.writer().print( + " .{{ \"{}\", \"{}\" }},\n", + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) }, + ); + } + try buf.appendSlice("};\n"); + } + + pub fn createEmptyDependenciesSource(buf: *std.ArrayList(u8)) Allocator.Error!void { + try buf.appendSlice( + \\pub const packages = struct {}; + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; + \\ + ); + } +}; + +pub const Location = union(enum) { + remote: Remote, + /// A directory found inside the parent package. + relative_path: Package.Path, + /// Recursive Fetch tasks will never use this Location, but it may be + /// passed in by the CLI. Indicates the file contents here should be copied + /// into the global package cache. It may be a file relative to the cwd or + /// absolute, in which case it should be treated exactly like a `file://` + /// URL, or a directory, in which case it should be treated as an + /// already-unpacked directory (but still needs to be copied into the + /// global package cache and have inclusion rules applied). + path_or_url: []const u8, + + pub const Remote = struct { + url: []const u8, + /// If this is null it means the user omitted the hash field from a dependency. + /// It will be an error but the logic should still fetch and print the discovered hash. + hash: ?Manifest.MultiHashHexDigest, + }; +}; + +pub const RunError = error{ + OutOfMemory, + /// This error code is intended to be handled by inspecting the + /// `error_bundle` field. + FetchFailed, +}; + +pub fn run(f: *Fetch) RunError!void { + const eb = &f.error_bundle; + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const cache_root = f.job_queue.global_cache; + + try eb.init(gpa); + + // Check the global zig package cache to see if the hash already exists. If + // so, load, parse, and validate the build.zig.zon file therein, and skip + // ahead to queuing up jobs for dependencies. Likewise if the location is a + // relative path, treat this the same as a cache hit. Otherwise, proceed. + + const remote = switch (f.location) { + .relative_path => |pkg_root| { + if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail( + f.location_tok, + try eb.addString("expected path relative to build root; found absolute path"), + ); + if (f.hash_tok != 0) return f.fail( + f.hash_tok, + try eb.addString("path-based dependencies are not hashed"), + ); + if (std.mem.startsWith(u8, pkg_root.sub_path, "../") or + std.mem.eql(u8, pkg_root.sub_path, "..")) + { + return f.fail( + f.location_tok, + try eb.printString("dependency path outside project: '{}{s}'", .{ + pkg_root.root_dir, pkg_root.sub_path, + }), + ); + } + f.package_root = pkg_root; + try loadManifest(f, pkg_root); + if (!f.has_build_zig) try checkBuildFileExistence(f); + if (!f.job_queue.recursive) return; + return queueJobsForDeps(f); + }, + .remote => |remote| remote, + .path_or_url => |path_or_url| { + if (fs.cwd().openIterableDir(path_or_url, .{})) |dir| { + var resource: Resource = .{ .dir = dir }; + return runResource(f, path_or_url, &resource, null); + } else |dir_err| { + const file_err = if (dir_err == error.NotDir) e: { + if (fs.cwd().openFile(path_or_url, .{})) |file| { + var resource: Resource = .{ .file = file }; + return runResource(f, path_or_url, &resource, null); + } else |err| break :e err; + } else dir_err; + + const uri = std.Uri.parse(path_or_url) catch |uri_err| { + return f.fail(0, try eb.printString( + "'{s}' could not be recognized as a file path ({s}) or an URL ({s})", + .{ path_or_url, @errorName(file_err), @errorName(uri_err) }, + )); + }; + var resource = try f.initResource(uri); + return runResource(f, uri.path, &resource, null); + } + }, + }; + + const s = fs.path.sep_str; + if (remote.hash) |expected_hash| { + const pkg_sub_path = "p" ++ s ++ expected_hash; + if (cache_root.handle.access(pkg_sub_path, .{})) |_| { + f.package_root = .{ + .root_dir = cache_root, + .sub_path = try arena.dupe(u8, pkg_sub_path), + }; + try loadManifest(f, f.package_root); + try checkBuildFileExistence(f); + if (!f.job_queue.recursive) return; + return queueJobsForDeps(f); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{ + cache_root, pkg_sub_path, @errorName(e), + }), + }); + return error.FetchFailed; + }, + } + } + + // Fetch and unpack the remote into a temporary directory. + + const uri = std.Uri.parse(remote.url) catch |err| return f.fail( + f.location_tok, + try eb.printString("invalid URI: {s}", .{@errorName(err)}), + ); + var resource = try f.initResource(uri); + return runResource(f, uri.path, &resource, remote.hash); +} + +pub fn deinit(f: *Fetch) void { + f.error_bundle.deinit(); + f.arena.deinit(); +} + +/// Consumes `resource`, even if an error is returned. +fn runResource( + f: *Fetch, + uri_path: []const u8, + resource: *Resource, + remote_hash: ?Manifest.MultiHashHexDigest, +) RunError!void { + defer resource.deinit(); + const arena = f.arena.allocator(); + const eb = &f.error_bundle; + const s = fs.path.sep_str; + const cache_root = f.job_queue.global_cache; + const rand_int = std.crypto.random.int(u64); + const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); + + const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); + var tmp_directory: Cache.Directory = .{ + .path = tmp_directory_path, + .handle = handle: { + const dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch |err| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{ + tmp_directory_path, @errorName(err), + }), + }); + return error.FetchFailed; + }; + break :handle dir.dir; + }, + }; + defer tmp_directory.handle.close(); + + try unpackResource(f, resource, uri_path, tmp_directory); + + // Load, parse, and validate the unpacked build.zig.zon file. It is allowed + // for the file to be missing, in which case this fetched package is + // considered to be a "naked" package. + try loadManifest(f, .{ .root_dir = tmp_directory }); + + // Apply the manifest's inclusion rules to the temporary directory by + // deleting excluded files. If any error occurred for files that were + // ultimately excluded, those errors should be ignored, such as failure to + // create symlinks that weren't supposed to be included anyway. + + // Empty directories have already been omitted by `unpackResource`. + + const filter: Filter = .{ + .include_paths = if (f.manifest) |m| m.paths else .{}, + }; + + // Compute the package hash based on the remaining files in the temporary + // directory. + + if (builtin.os.tag == .linux and f.job_queue.work_around_btrfs_bug) { + // https://github.com/ziglang/zig/issues/17095 + tmp_directory.handle.close(); + const iterable_dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch + @panic("btrfs workaround failed"); + tmp_directory.handle = iterable_dir.dir; + } + + f.actual_hash = try computeHash(f, tmp_directory, filter); + + // Rename the temporary directory into the global zig package cache + // directory. If the hash already exists, delete the temporary directory + // and leave the zig package cache directory untouched as it may be in use + // by the system. This is done even if the hash is invalid, in case the + // package with the different hash is used in the future. + + f.package_root = .{ + .root_dir = cache_root, + .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)), + }; + renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, f.package_root.sub_path) catch |err| { + const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); + const dest = try cache_root.join(arena, &.{f.package_root.sub_path}); + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}", + .{ src, dest, @errorName(err) }, + ) }); + return error.FetchFailed; + }; + + // Validate the computed hash against the expected hash. If invalid, this + // job is done. + + const actual_hex = Manifest.hexDigest(f.actual_hash); + if (remote_hash) |declared_hash| { + if (!std.mem.eql(u8, &declared_hash, &actual_hex)) { + return f.fail(f.hash_tok, try eb.printString( + "hash mismatch: manifest declares {s} but the fetched package has {s}", + .{ declared_hash, actual_hex }, + )); + } + } else if (!f.omit_missing_hash_error) { + const notes_len = 1; + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("dependency is missing hash field"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); + const notes_start = try eb.reserveNotes(notes_len); + eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), + })); + return error.FetchFailed; + } + + // Spawn a new fetch job for each dependency in the manifest file. Use + // a mutex and a hash map so that redundant jobs do not get queued up. + if (!f.job_queue.recursive) return; + return queueJobsForDeps(f); +} + +/// `computeHash` gets a free check for the existence of `build.zig`, but when +/// not computing a hash, we need to do a syscall to check for it. +fn checkBuildFileExistence(f: *Fetch) RunError!void { + const eb = &f.error_bundle; + if (f.package_root.access(Package.build_zig_basename, .{})) |_| { + f.has_build_zig = true; + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to access '{}{s}': {s}", .{ + f.package_root, Package.build_zig_basename, @errorName(e), + }), + }); + return error.FetchFailed; + }, + } +} + +/// This function populates `f.manifest` or leaves it `null`. +fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void { + const eb = &f.error_bundle; + const arena = f.arena.allocator(); + const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions( + arena, + try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }), + Manifest.max_bytes, + null, + 1, + 0, + ) catch |err| switch (err) { + error.FileNotFound => return, + else => |e| { + const file_path = try pkg_root.join(arena, Manifest.basename); + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to load package manifest '{}': {s}", .{ + file_path, @errorName(e), + }), + }); + return error.FetchFailed; + }, + }; + + const ast = &f.manifest_ast; + ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); + + if (ast.errors.len > 0) { + const file_path = try std.fmt.allocPrint(arena, "{}" ++ Manifest.basename, .{pkg_root}); + try main.putAstErrorsIntoBundle(arena, ast.*, file_path, eb); + return error.FetchFailed; + } + + f.manifest = try Manifest.parse(arena, ast.*, .{ + .allow_missing_paths_field = f.allow_missing_paths_field, + }); + const manifest = &f.manifest.?; + + if (manifest.errors.len > 0) { + const src_path = try eb.printString("{}{s}", .{ pkg_root, Manifest.basename }); + const token_starts = ast.tokens.items(.start); + + for (manifest.errors) |msg| { + const start_loc = ast.tokenLocation(0, msg.tok); + + try eb.addRootErrorMessage(.{ + .msg = try eb.addString(msg.msg), + .src_loc = try eb.addSourceLocation(.{ + .src_path = src_path, + .span_start = token_starts[msg.tok], + .span_end = @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len), + .span_main = token_starts[msg.tok] + msg.off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }), + }); + } + return error.FetchFailed; + } +} + +fn queueJobsForDeps(f: *Fetch) RunError!void { + assert(f.job_queue.recursive); + + // If the package does not have a build.zig.zon file then there are no dependencies. + const manifest = f.manifest orelse return; + + const new_fetches, const prog_names = nf: { + const parent_arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const cache_root = f.job_queue.global_cache; + const dep_names = manifest.dependencies.keys(); + const deps = manifest.dependencies.values(); + // Grab the new tasks into a temporary buffer so we can unlock that mutex + // as fast as possible. + // This overallocates any fetches that get skipped by the `continue` in the + // loop below. + const new_fetches = try parent_arena.alloc(Fetch, deps.len); + const prog_names = try parent_arena.alloc([]const u8, deps.len); + var new_fetch_index: usize = 0; + + f.job_queue.mutex.lock(); + defer f.job_queue.mutex.unlock(); + + try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); + try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len)); + + // There are four cases here: + // * Correct hash is provided by manifest. + // - Hash map already has the entry, no need to add it again. + // * Incorrect hash is provided by manifest. + // - Hash mismatch error emitted; `queueJobsForDeps` is not called. + // * Hash is not provided by manifest. + // - Hash missing error emitted; `queueJobsForDeps` is not called. + // * path-based location is used without a hash. + // - Hash is added to the table based on the path alone before + // calling run(); no need to add it again. + + for (dep_names, deps) |dep_name, dep| { + const new_fetch = &new_fetches[new_fetch_index]; + const location: Location = switch (dep.location) { + .url => |url| .{ .remote = .{ + .url = url, + .hash = h: { + const h = dep.hash orelse break :h null; + const digest_len = @typeInfo(Manifest.MultiHashHexDigest).Array.len; + const multihash_digest = h[0..digest_len].*; + const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest); + if (gop.found_existing) continue; + gop.value_ptr.* = new_fetch; + break :h multihash_digest; + }, + } }, + .path => |rel_path| l: { + // This might produce an invalid path, which is checked for + // at the beginning of run(). + const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); + const multihash_digest = relativePathDigest(new_root, cache_root); + const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest); + if (gop.found_existing) continue; + gop.value_ptr.* = new_fetch; + break :l .{ .relative_path = new_root }; + }, + }; + prog_names[new_fetch_index] = dep_name; + new_fetch_index += 1; + f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); + new_fetch.* = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = location, + .location_tok = dep.location_tok, + .hash_tok = dep.hash_tok, + .parent_package_root = f.package_root, + .parent_manifest_ast = &f.manifest_ast, + .prog_node = f.prog_node, + .job_queue = f.job_queue, + .omit_missing_hash_error = false, + .allow_missing_paths_field = true, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = null, + .manifest_ast = undefined, + .actual_hash = undefined, + .has_build_zig = false, + .oom_flag = false, + + .module = null, + }; + } + + // job_queue mutex is locked so this is OK. + f.prog_node.unprotected_estimated_total_items += new_fetch_index; + + break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] }; + }; + + // Now it's time to give tasks to the thread pool. + const thread_pool = f.job_queue.thread_pool; + + for (new_fetches, prog_names) |*new_fetch, prog_name| { + f.job_queue.wait_group.start(); + thread_pool.spawn(workerRun, .{ new_fetch, prog_name }) catch |err| switch (err) { + error.OutOfMemory => { + new_fetch.oom_flag = true; + f.job_queue.wait_group.finish(); + continue; + }, + }; + } +} + +pub fn relativePathDigest( + pkg_root: Package.Path, + cache_root: Cache.Directory, +) Manifest.MultiHashHexDigest { + var hasher = Manifest.Hash.init(.{}); + // This hash is a tuple of: + // * whether it relative to the global cache directory or to the root package + // * the relative file path from there to the build root of the package + hasher.update(if (pkg_root.root_dir.eql(cache_root)) + &package_hash_prefix_cached + else + &package_hash_prefix_project); + hasher.update(pkg_root.sub_path); + return Manifest.hexDigest(hasher.finalResult()); +} + +pub fn workerRun(f: *Fetch, prog_name: []const u8) void { + defer f.job_queue.wait_group.finish(); + + var prog_node = f.prog_node.start(prog_name, 0); + defer prog_node.end(); + prog_node.activate(); + + run(f) catch |err| switch (err) { + error.OutOfMemory => f.oom_flag = true, + error.FetchFailed => { + // Nothing to do because the errors are already reported in `error_bundle`, + // and a reference is kept to the `Fetch` task inside `all_fetches`. + }, + }; +} + +fn srcLoc( + f: *Fetch, + tok: std.zig.Ast.TokenIndex, +) Allocator.Error!ErrorBundle.SourceLocationIndex { + const ast = f.parent_manifest_ast orelse return .none; + const eb = &f.error_bundle; + const token_starts = ast.tokens.items(.start); + const start_loc = ast.tokenLocation(0, tok); + const src_path = try eb.printString("{}" ++ Manifest.basename, .{f.parent_package_root}); + const msg_off = 0; + return eb.addSourceLocation(.{ + .src_path = src_path, + .span_start = token_starts[tok], + .span_end = @intCast(token_starts[tok] + ast.tokenSlice(tok).len), + .span_main = token_starts[tok] + msg_off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }); +} + +fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError { + const eb = &f.error_bundle; + try eb.addRootErrorMessage(.{ + .msg = msg_str, + .src_loc = try f.srcLoc(msg_tok), + }); + return error.FetchFailed; +} + +const Resource = union(enum) { + file: fs.File, + http_request: std.http.Client.Request, + git: Git, + dir: fs.IterableDir, + + const Git = struct { + fetch_stream: git.Session.FetchStream, + want_oid: [git.oid_length]u8, + }; + + fn deinit(resource: *Resource) void { + switch (resource.*) { + .file => |*file| file.close(), + .http_request => |*req| req.deinit(), + .git => |*git_resource| git_resource.fetch_stream.deinit(), + .dir => |*dir| dir.close(), + } + resource.* = undefined; + } + + fn reader(resource: *Resource) std.io.AnyReader { + return .{ + .context = resource, + .readFn = read, + }; + } + + fn read(context: *const anyopaque, buffer: []u8) anyerror!usize { + const resource: *Resource = @constCast(@ptrCast(@alignCast(context))); + switch (resource.*) { + .file => |*f| return f.read(buffer), + .http_request => |*r| return r.read(buffer), + .git => |*g| return g.fetch_stream.read(buffer), + .dir => unreachable, + } + } +}; + +const FileType = enum { + tar, + @"tar.gz", + @"tar.xz", + git_pack, + + fn fromPath(file_path: []const u8) ?FileType { + if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; + if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; + return null; + } + + /// Parameter is a content-disposition header value. + fn fromContentDisposition(cd_header: []const u8) ?FileType { + const attach_end = ascii.indexOfIgnoreCase(cd_header, "attachment;") orelse + return null; + + var value_start = ascii.indexOfIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse + return null; + value_start += "filename".len; + if (cd_header[value_start] == '*') { + value_start += 1; + } + if (cd_header[value_start] != '=') return null; + value_start += 1; + + var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len; + if (cd_header[value_end - 1] == '\"') { + value_end -= 1; + } + return fromPath(cd_header[value_start..value_end]); + } + + test fromContentDisposition { + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); + + try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); + try std.testing.expect(fromContentDisposition("attachment; FileName=\"stuff.tar\"") == null); + try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); + try std.testing.expect(fromContentDisposition("attachment; size=42") == null); + try std.testing.expect(fromContentDisposition("inline; size=42") == null); + try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null); + try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null); + } +}; + +fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource { + const gpa = f.arena.child_allocator; + const arena = f.arena.allocator(); + const eb = &f.error_bundle; + + if (ascii.eqlIgnoreCase(uri.scheme, "file")) return .{ + .file = f.parent_package_root.openFile(uri.path, .{}) catch |err| { + return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{ + f.parent_package_root, uri.path, @errorName(err), + })); + }, + }; + + const http_client = f.job_queue.http_client; + + if (ascii.eqlIgnoreCase(uri.scheme, "http") or + ascii.eqlIgnoreCase(uri.scheme, "https")) + { + var h = std.http.Headers{ .allocator = gpa }; + defer h.deinit(); + + var req = http_client.request(.GET, uri, h, .{}) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to connect to server: {s}", + .{@errorName(err)}, + )); + }; + errdefer req.deinit(); // releases more than memory + + req.start(.{}) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "HTTP request failed: {s}", + .{@errorName(err)}, + )); + }; + req.wait() catch |err| { + return f.fail(f.location_tok, try eb.printString( + "invalid HTTP response: {s}", + .{@errorName(err)}, + )); + }; + + if (req.response.status != .ok) { + return f.fail(f.location_tok, try eb.printString( + "bad HTTP response code: '{d} {s}'", + .{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" }, + )); + } + + return .{ .http_request = req }; + } + + if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or + ascii.eqlIgnoreCase(uri.scheme, "git+https")) + { + var transport_uri = uri; + transport_uri.scheme = uri.scheme["git+".len..]; + var redirect_uri: []u8 = undefined; + var session: git.Session = .{ .transport = http_client, .uri = transport_uri }; + session.discoverCapabilities(gpa, &redirect_uri) catch |err| switch (err) { + error.Redirected => { + defer gpa.free(redirect_uri); + return f.fail(f.location_tok, try eb.printString( + "repository moved to {s}", + .{redirect_uri}, + )); + }, + else => |e| { + return f.fail(f.location_tok, try eb.printString( + "unable to discover remote git server capabilities: {s}", + .{@errorName(e)}, + )); + }, + }; + + const want_oid = want_oid: { + const want_ref = uri.fragment orelse "HEAD"; + if (git.parseOid(want_ref)) |oid| break :want_oid oid else |_| {} + + const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); + const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); + + var ref_iterator = session.listRefs(gpa, .{ + .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, + .include_peeled = true, + }) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to list refs: {s}", + .{@errorName(err)}, + )); + }; + defer ref_iterator.deinit(); + while (ref_iterator.next() catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to iterate refs: {s}", + .{@errorName(err)}, + )); + }) |ref| { + if (std.mem.eql(u8, ref.name, want_ref) or + std.mem.eql(u8, ref.name, want_ref_head) or + std.mem.eql(u8, ref.name, want_ref_tag)) + { + break :want_oid ref.peeled orelse ref.oid; + } + } + return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref})); + }; + if (uri.fragment == null) { + const notes_len = 1; + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("url field is missing an explicit ref"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); + const notes_start = try eb.reserveNotes(notes_len); + eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("try .url = \"{+/}#{}\",", .{ + uri, std.fmt.fmtSliceHexLower(&want_oid), + }), + })); + return error.FetchFailed; + } + + var want_oid_buf: [git.fmt_oid_length]u8 = undefined; + _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{ + std.fmt.fmtSliceHexLower(&want_oid), + }) catch unreachable; + var fetch_stream = session.fetch(gpa, &.{&want_oid_buf}) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to create fetch stream: {s}", + .{@errorName(err)}, + )); + }; + errdefer fetch_stream.deinit(); + + return .{ .git = .{ + .fetch_stream = fetch_stream, + .want_oid = want_oid, + } }; + } + + return f.fail(f.location_tok, try eb.printString( + "unsupported URL scheme: {s}", + .{uri.scheme}, + )); +} + +fn unpackResource( + f: *Fetch, + resource: *Resource, + uri_path: []const u8, + tmp_directory: Cache.Directory, +) RunError!void { + const eb = &f.error_bundle; + const file_type = switch (resource.*) { + .file => FileType.fromPath(uri_path) orelse + return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), + + .http_request => |req| ft: { + // Content-Type takes first precedence. + const content_type = req.response.headers.getFirstValue("Content-Type") orelse + return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header")); + + if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) + break :ft .tar; + + if (ascii.eqlIgnoreCase(content_type, "application/gzip") or + ascii.eqlIgnoreCase(content_type, "application/x-gzip") or + ascii.eqlIgnoreCase(content_type, "application/tar+gzip")) + { + break :ft .@"tar.gz"; + } + + if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) + break :ft .@"tar.xz"; + + if (!ascii.eqlIgnoreCase(content_type, "application/octet-stream")) { + return f.fail(f.location_tok, try eb.printString( + "unrecognized 'Content-Type' header: '{s}'", + .{content_type}, + )); + } + + // Next, the filename from 'content-disposition: attachment' takes precedence. + if (req.response.headers.getFirstValue("Content-Disposition")) |cd_header| { + break :ft FileType.fromContentDisposition(cd_header) orelse { + return f.fail(f.location_tok, try eb.printString( + "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", + .{cd_header}, + )); + }; + } + + // Finally, the path from the URI is used. + break :ft FileType.fromPath(uri_path) orelse { + return f.fail(f.location_tok, try eb.printString( + "unknown file type: '{s}'", + .{uri_path}, + )); + }; + }, + + .git => .git_pack, + + .dir => |dir| return f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to copy directory '{s}': {s}", + .{ uri_path, @errorName(err) }, + )); + }, + }; + + switch (file_type) { + .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()), + .@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip), + .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz), + .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { + error.FetchFailed => return error.FetchFailed, + error.OutOfMemory => return error.OutOfMemory, + else => |e| return f.fail(f.location_tok, try eb.printString( + "unable to unpack git files: {s}", + .{@errorName(e)}, + )), + }, + } +} + +fn unpackTarballCompressed( + f: *Fetch, + out_dir: fs.Dir, + resource: *Resource, + comptime Compression: type, +) RunError!void { + const gpa = f.arena.child_allocator; + const eb = &f.error_bundle; + const reader = resource.reader(); + var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); + + var decompress = Compression.decompress(gpa, br.reader()) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to decompress tarball: {s}", + .{@errorName(err)}, + )); + }; + defer decompress.deinit(); + + return unpackTarball(f, out_dir, decompress.reader()); +} + +fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { + const eb = &f.error_bundle; + const gpa = f.arena.child_allocator; + + var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa }; + defer diagnostics.deinit(); + + std.tar.pipeToFileSystem(out_dir, reader, .{ + .diagnostics = &diagnostics, + .strip_components = 1, + // TODO: we would like to set this to executable_bit_only, but two + // things need to happen before that: + // 1. the tar implementation needs to support it + // 2. the hashing algorithm here needs to support detecting the is_executable + // bit on Windows from the ACLs (see the isExecutable function). + .mode_mode = .ignore, + .exclude_empty_directories = true, + }) catch |err| return f.fail(f.location_tok, try eb.printString( + "unable to unpack tarball to temporary directory: {s}", + .{@errorName(err)}, + )); + + if (diagnostics.errors.items.len > 0) { + const notes_len: u32 = @intCast(diagnostics.errors.items.len); + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("unable to unpack tarball"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); + const notes_start = try eb.reserveNotes(notes_len); + for (diagnostics.errors.items, notes_start..) |item, note_i| { + switch (item) { + .unable_to_create_sym_link => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ + info.file_name, info.link_name, @errorName(info.code), + }), + })); + }, + .unable_to_create_file => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create file '{s}': {s}", .{ + info.file_name, @errorName(info.code), + }), + })); + }, + .unsupported_file_type => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ + info.file_name, @intFromEnum(info.file_type), + }), + })); + }, + } + } + return error.FetchFailed; + } +} + +fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void { + const eb = &f.error_bundle; + const gpa = f.arena.child_allocator; + const want_oid = resource.git.want_oid; + const reader = resource.git.fetch_stream.reader(); + // The .git directory is used to store the packfile and associated index, but + // we do not attempt to replicate the exact structure of a real .git + // directory, since that isn't relevant for fetching a package. + { + var pack_dir = try out_dir.makeOpenPath(".git", .{}); + defer pack_dir.close(); + var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); + defer pack_file.close(); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + try fifo.pump(reader, pack_file.writer()); + try pack_file.sync(); + + var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); + defer index_file.close(); + { + var index_prog_node = f.prog_node.start("Index pack", 0); + defer index_prog_node.end(); + index_prog_node.activate(); + var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); + try git.indexPack(gpa, pack_file, index_buffered_writer.writer()); + try index_buffered_writer.flush(); + try index_file.sync(); + } + + { + var checkout_prog_node = f.prog_node.start("Checkout", 0); + defer checkout_prog_node.end(); + checkout_prog_node.activate(); + var repository = try git.Repository.init(gpa, pack_file, index_file); + defer repository.deinit(); + var diagnostics: git.Diagnostics = .{ .allocator = gpa }; + defer diagnostics.deinit(); + try repository.checkout(out_dir, want_oid, &diagnostics); + + if (diagnostics.errors.items.len > 0) { + const notes_len: u32 = @intCast(diagnostics.errors.items.len); + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("unable to unpack packfile"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); + const notes_start = try eb.reserveNotes(notes_len); + for (diagnostics.errors.items, notes_start..) |item, note_i| { + switch (item) { + .unable_to_create_sym_link => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ + info.file_name, info.link_name, @errorName(info.code), + }), + })); + }, + } + } + return error.InvalidGitPack; + } + } + } + + try out_dir.deleteTree(".git"); +} + +fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) anyerror!void { + const gpa = f.arena.child_allocator; + // Recursive directory copy. + var it = try dir.walk(gpa); + defer it.deinit(); + while (try it.next()) |entry| { + switch (entry.kind) { + .directory => {}, // omit empty directories + .file => { + dir.dir.copyFile( + entry.path, + tmp_dir, + entry.path, + .{}, + ) catch |err| switch (err) { + error.FileNotFound => { + if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); + try dir.dir.copyFile(entry.path, tmp_dir, entry.path, .{}); + }, + else => |e| return e, + }; + }, + .sym_link => { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const link_name = try dir.dir.readLink(entry.path, &buf); + // TODO: if this would create a symlink to outside + // the destination directory, fail with an error instead. + tmp_dir.symLink(link_name, entry.path, .{}) catch |err| switch (err) { + error.FileNotFound => { + if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); + try tmp_dir.symLink(link_name, entry.path, .{}); + }, + else => |e| return e, + }; + }, + else => return error.IllegalFileTypeInPackage, + } + } +} + +pub fn renameTmpIntoCache( + cache_dir: fs.Dir, + tmp_dir_sub_path: []const u8, + dest_dir_sub_path: []const u8, +) !void { + assert(dest_dir_sub_path[1] == fs.path.sep); + var handled_missing_dir = false; + while (true) { + cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { + error.FileNotFound => { + if (handled_missing_dir) return err; + cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { + error.PathAlreadyExists => handled_missing_dir = true, + else => |e| return e, + }; + continue; + }, + error.PathAlreadyExists, error.AccessDenied => { + // Package has been already downloaded and may already be in use on the system. + cache_dir.deleteTree(tmp_dir_sub_path) catch { + // Garbage files leftover in zig-cache/tmp/ is, as they say + // on Star Trek, "operating within normal parameters". + }; + }, + else => |e| return e, + }; + break; + } +} + +/// Assumes that files not included in the package have already been filtered +/// prior to calling this function. This ensures that files not protected by +/// the hash are not present on the file system. Empty directories are *not +/// hashed* and must not be present on the file system when calling this +/// function. +fn computeHash( + f: *Fetch, + tmp_directory: Cache.Directory, + filter: Filter, +) RunError!Manifest.Digest { + // All the path name strings need to be in memory for sorting. + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const eb = &f.error_bundle; + const thread_pool = f.job_queue.thread_pool; + + // Collect all files, recursively, then sort. + var all_files = std.ArrayList(*HashedFile).init(gpa); + defer all_files.deinit(); + + var deleted_files = std.ArrayList(*DeletedFile).init(gpa); + defer deleted_files.deinit(); + + // Track directories which had any files deleted from them so that empty directories + // can be deleted. + var sus_dirs: std.StringArrayHashMapUnmanaged(void) = .{}; + defer sus_dirs.deinit(gpa); + + var walker = try @as(fs.IterableDir, .{ .dir = tmp_directory.handle }).walk(gpa); + defer walker.deinit(); + + { + // The final hash will be a hash of each file hashed independently. This + // allows hashing in parallel. + var wait_group: WaitGroup = .{}; + // `computeHash` is called from a worker thread so there must not be + // any waiting without working or a deadlock could occur. + defer thread_pool.waitAndWork(&wait_group); + + while (walker.next() catch |err| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to walk temporary directory '{}': {s}", + .{ tmp_directory, @errorName(err) }, + ) }); + return error.FetchFailed; + }) |entry| { + if (entry.kind == .directory) continue; + + if (!filter.includePath(entry.path)) { + // Delete instead of including in hash calculation. + const fs_path = try arena.dupe(u8, entry.path); + + // Also track the parent directory in case it becomes empty. + if (fs.path.dirname(fs_path)) |parent| + try sus_dirs.put(gpa, parent, {}); + + const deleted_file = try arena.create(DeletedFile); + deleted_file.* = .{ + .fs_path = fs_path, + .failure = undefined, // to be populated by the worker + }; + wait_group.start(); + try thread_pool.spawn(workerDeleteFile, .{ + tmp_directory.handle, deleted_file, &wait_group, + }); + try deleted_files.append(deleted_file); + continue; + } + + const kind: HashedFile.Kind = switch (entry.kind) { + .directory => unreachable, + .file => .file, + .sym_link => .sym_link, + else => return f.fail(f.location_tok, try eb.printString( + "package contains '{s}' which has illegal file type '{s}'", + .{ entry.path, @tagName(entry.kind) }, + )), + }; + + if (std.mem.eql(u8, entry.path, Package.build_zig_basename)) + f.has_build_zig = true; + + const fs_path = try arena.dupe(u8, entry.path); + const hashed_file = try arena.create(HashedFile); + hashed_file.* = .{ + .fs_path = fs_path, + .normalized_path = try normalizePath(arena, fs_path), + .kind = kind, + .hash = undefined, // to be populated by the worker + .failure = undefined, // to be populated by the worker + }; + wait_group.start(); + try thread_pool.spawn(workerHashFile, .{ + tmp_directory.handle, hashed_file, &wait_group, + }); + try all_files.append(hashed_file); + } + } + + { + // Sort by length, descending, so that child directories get removed first. + sus_dirs.sortUnstable(@as(struct { + keys: []const []const u8, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return ctx.keys[b_index].len < ctx.keys[a_index].len; + } + }, .{ .keys = sus_dirs.keys() })); + + // During this loop, more entries will be added, so we must loop by index. + var i: usize = 0; + while (i < sus_dirs.count()) : (i += 1) { + const sus_dir = sus_dirs.keys()[i]; + tmp_directory.handle.deleteDir(sus_dir) catch |err| switch (err) { + error.DirNotEmpty => continue, + error.FileNotFound => continue, + else => |e| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to delete empty directory '{s}': {s}", + .{ sus_dir, @errorName(e) }, + ) }); + return error.FetchFailed; + }, + }; + if (fs.path.dirname(sus_dir)) |parent| { + try sus_dirs.put(gpa, parent, {}); + } + } + } + + std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); + + var hasher = Manifest.Hash.init(.{}); + var any_failures = false; + for (all_files.items) |hashed_file| { + hashed_file.failure catch |err| { + any_failures = true; + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to hash '{s}': {s}", .{ + hashed_file.fs_path, @errorName(err), + }), + }); + }; + hasher.update(&hashed_file.hash); + } + for (deleted_files.items) |deleted_file| { + deleted_file.failure catch |err| { + any_failures = true; + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{ + deleted_file.fs_path, @errorName(err), + }), + }); + }; + } + + if (any_failures) return error.FetchFailed; + return hasher.finalResult(); +} + +fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { + defer wg.finish(); + hashed_file.failure = hashFileFallible(dir, hashed_file); +} + +fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile, wg: *WaitGroup) void { + defer wg.finish(); + deleted_file.failure = deleteFileFallible(dir, deleted_file); +} + +fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { + var buf: [8000]u8 = undefined; + var hasher = Manifest.Hash.init(.{}); + hasher.update(hashed_file.normalized_path); + switch (hashed_file.kind) { + .file => { + var file = try dir.openFile(hashed_file.fs_path, .{}); + defer file.close(); + hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) }); + while (true) { + const bytes_read = try file.read(&buf); + if (bytes_read == 0) break; + hasher.update(buf[0..bytes_read]); + } + }, + .sym_link => { + const link_name = try dir.readLink(hashed_file.fs_path, &buf); + hasher.update(link_name); + }, + } + hasher.final(&hashed_file.hash); +} + +fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { + try dir.deleteFile(deleted_file.fs_path); +} + +fn isExecutable(file: fs.File) !bool { + if (builtin.os.tag == .windows) { + // TODO check the ACL on Windows. + // Until this is implemented, this could be a false negative on + // Windows, which is why we do not yet set executable_bit_only above + // when unpacking the tarball. + return false; + } else { + const stat = try file.stat(); + return (stat.mode & std.os.S.IXUSR) != 0; + } +} + +const DeletedFile = struct { + fs_path: []const u8, + failure: Error!void, + + const Error = + fs.Dir.DeleteFileError || + fs.Dir.DeleteDirError; +}; + +const HashedFile = struct { + fs_path: []const u8, + normalized_path: []const u8, + hash: Manifest.Digest, + failure: Error!void, + kind: Kind, + + const Error = + fs.File.OpenError || + fs.File.ReadError || + fs.File.StatError || + fs.Dir.ReadLinkError; + + const Kind = enum { file, sym_link }; + + fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { + _ = context; + return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); + } +}; + +/// Make a file system path identical independently of operating system path inconsistencies. +/// This converts backslashes into forward slashes. +fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 { + const canonical_sep = '/'; + + if (fs.path.sep == canonical_sep) + return fs_path; + + const normalized = try arena.dupe(u8, fs_path); + for (normalized) |*byte| { + switch (byte.*) { + fs.path.sep => byte.* = canonical_sep, + else => continue, + } + } + return normalized; +} + +const Filter = struct { + include_paths: std.StringArrayHashMapUnmanaged(void) = .{}, + + /// sub_path is relative to the package root. + pub fn includePath(self: Filter, sub_path: []const u8) bool { + if (self.include_paths.count() == 0) return true; + if (self.include_paths.contains("")) return true; + if (self.include_paths.contains(sub_path)) return true; + + // Check if any included paths are parent directories of sub_path. + var dirname = sub_path; + while (std.fs.path.dirname(dirname)) |next_dirname| { + if (self.include_paths.contains(sub_path)) return true; + dirname = next_dirname; + } + + return false; + } +}; + +pub fn depDigest( + pkg_root: Package.Path, + cache_root: Cache.Directory, + dep: Manifest.Dependency, +) ?Manifest.MultiHashHexDigest { + if (dep.hash) |h| return h[0..Manifest.multihash_hex_digest_len].*; + + switch (dep.location) { + .url => return null, + .path => |rel_path| { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buf); + const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch + return null; + return relativePathDigest(new_root, cache_root); + }, + } +} + +// These are random bytes. +const package_hash_prefix_cached = [8]u8{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 }; +const package_hash_prefix_project = [8]u8{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc }; + +const builtin = @import("builtin"); +const std = @import("std"); +const fs = std.fs; +const assert = std.debug.assert; +const ascii = std.ascii; +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const ThreadPool = std.Thread.Pool; +const WaitGroup = std.Thread.WaitGroup; +const Fetch = @This(); +const main = @import("../main.zig"); +const git = @import("Fetch/git.zig"); +const Package = @import("../Package.zig"); +const Manifest = Package.Manifest; +const ErrorBundle = std.zig.ErrorBundle; diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig new file mode 100644 index 0000000000000000000000000000000000000000..af4317702d3898cda6afe5b9a747389ebdd8aff0 --- /dev/null +++ b/src/Package/Fetch/git.zig @@ -0,0 +1,1466 @@ +//! Git support for package fetching. +//! +//! This is not intended to support all features of Git: it is limited to the +//! basic functionality needed to clone a repository for the purpose of fetching +//! a package. + +const std = @import("std"); +const mem = std.mem; +const testing = std.testing; +const Allocator = mem.Allocator; +const Sha1 = std.crypto.hash.Sha1; +const assert = std.debug.assert; + +pub const oid_length = Sha1.digest_length; +pub const fmt_oid_length = 2 * oid_length; +/// The ID of a Git object (an SHA-1 hash). +pub const Oid = [oid_length]u8; + +pub fn parseOid(s: []const u8) !Oid { + if (s.len != fmt_oid_length) return error.InvalidOid; + var oid: Oid = undefined; + for (&oid, 0..) |*b, i| { + b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; + } + return oid; +} + +test parseOid { + try testing.expectEqualSlices( + u8, + &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, + &try parseOid("ce919ccf45951856a762ffdb8ef850301cd8c588"), + ); + try testing.expectError(error.InvalidOid, parseOid("ce919ccf")); + try testing.expectError(error.InvalidOid, parseOid("master")); + try testing.expectError(error.InvalidOid, parseOid("HEAD")); +} + +pub const Diagnostics = struct { + allocator: Allocator, + errors: std.ArrayListUnmanaged(Error) = .{}, + + pub const Error = union(enum) { + unable_to_create_sym_link: struct { + code: anyerror, + file_name: []const u8, + link_name: []const u8, + }, + }; + + pub fn deinit(d: *Diagnostics) void { + for (d.errors.items) |item| { + switch (item) { + .unable_to_create_sym_link => |info| { + d.allocator.free(info.file_name); + d.allocator.free(info.link_name); + }, + } + } + d.errors.deinit(d.allocator); + d.* = undefined; + } +}; + +pub const Repository = struct { + odb: Odb, + + pub fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Repository { + return .{ .odb = try Odb.init(allocator, pack_file, index_file) }; + } + + pub fn deinit(repository: *Repository) void { + repository.odb.deinit(); + repository.* = undefined; + } + + /// Checks out the repository at `commit_oid` to `worktree`. + pub fn checkout( + repository: *Repository, + worktree: std.fs.Dir, + commit_oid: Oid, + diagnostics: *Diagnostics, + ) !void { + try repository.odb.seekOid(commit_oid); + const tree_oid = tree_oid: { + var commit_object = try repository.odb.readObject(); + if (commit_object.type != .commit) return error.NotACommit; + break :tree_oid try getCommitTree(commit_object.data); + }; + try repository.checkoutTree(worktree, tree_oid, "", diagnostics); + } + + /// Checks out the tree at `tree_oid` to `worktree`. + fn checkoutTree( + repository: *Repository, + dir: std.fs.Dir, + tree_oid: Oid, + current_path: []const u8, + diagnostics: *Diagnostics, + ) !void { + try repository.odb.seekOid(tree_oid); + const tree_object = try repository.odb.readObject(); + if (tree_object.type != .tree) return error.NotATree; + // The tree object may be evicted from the object cache while we're + // iterating over it, so we can make a defensive copy here to make sure + // it remains valid until we're done with it + const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); + defer repository.odb.allocator.free(tree_data); + + var tree_iter: TreeIterator = .{ .data = tree_data }; + while (try tree_iter.next()) |entry| { + switch (entry.type) { + .directory => { + try dir.makeDir(entry.name); + var subdir = try dir.openDir(entry.name, .{}); + defer subdir.close(); + const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name }); + defer repository.odb.allocator.free(sub_path); + try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics); + }, + .file => { + var file = try dir.createFile(entry.name, .{}); + defer file.close(); + try repository.odb.seekOid(entry.oid); + var file_object = try repository.odb.readObject(); + if (file_object.type != .blob) return error.InvalidFile; + try file.writeAll(file_object.data); + try file.sync(); + }, + .symlink => { + try repository.odb.seekOid(entry.oid); + var symlink_object = try repository.odb.readObject(); + if (symlink_object.type != .blob) return error.InvalidFile; + const link_name = symlink_object.data; + dir.symLink(link_name, entry.name, .{}) catch |e| { + const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); + errdefer diagnostics.allocator.free(file_name); + const link_name_dup = try diagnostics.allocator.dupe(u8, link_name); + errdefer diagnostics.allocator.free(link_name_dup); + try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{ + .code = e, + .file_name = file_name, + .link_name = link_name_dup, + } }); + }; + }, + .gitlink => { + // Consistent with git archive behavior, create the directory but + // do nothing else + try dir.makeDir(entry.name); + }, + } + } + } + + /// Returns the ID of the tree associated with the given commit (provided as + /// raw object data). + fn getCommitTree(commit_data: []const u8) !Oid { + if (!mem.startsWith(u8, commit_data, "tree ") or + commit_data.len < "tree ".len + fmt_oid_length + "\n".len or + commit_data["tree ".len + fmt_oid_length] != '\n') + { + return error.InvalidCommit; + } + return try parseOid(commit_data["tree ".len..][0..fmt_oid_length]); + } + + const TreeIterator = struct { + data: []const u8, + pos: usize = 0, + + const Entry = struct { + type: Type, + executable: bool, + name: [:0]const u8, + oid: Oid, + + const Type = enum(u4) { + directory = 0o4, + file = 0o10, + symlink = 0o12, + gitlink = 0o16, + }; + }; + + fn next(iterator: *TreeIterator) !?Entry { + if (iterator.pos == iterator.data.len) return null; + + const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree; + const mode: packed struct { + permission: u9, + unused: u3, + type: u4, + } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree); + const @"type" = std.meta.intToEnum(Entry.Type, mode.type) catch return error.InvalidTree; + const executable = switch (mode.permission) { + 0 => if (@"type" == .file) return error.InvalidTree else false, + 0o644 => if (@"type" != .file) return error.InvalidTree else false, + 0o755 => if (@"type" != .file) return error.InvalidTree else true, + else => return error.InvalidTree, + }; + iterator.pos = mode_end + 1; + + const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree; + const name = iterator.data[iterator.pos..name_end :0]; + iterator.pos = name_end + 1; + + if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; + const oid = iterator.data[iterator.pos..][0..oid_length].*; + iterator.pos += oid_length; + + return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; + } + }; +}; + +/// A Git object database backed by a packfile. A packfile index is also used +/// for efficient access to objects in the packfile. +/// +/// The format of the packfile and its associated index are documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +const Odb = struct { + pack_file: std.fs.File, + index_header: IndexHeader, + index_file: std.fs.File, + cache: ObjectCache = .{}, + allocator: Allocator, + + /// Initializes the database from open pack and index files. + fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Odb { + try pack_file.seekTo(0); + try index_file.seekTo(0); + const index_header = try IndexHeader.read(index_file.reader()); + return .{ + .pack_file = pack_file, + .index_header = index_header, + .index_file = index_file, + .allocator = allocator, + }; + } + + fn deinit(odb: *Odb) void { + odb.cache.deinit(odb.allocator); + odb.* = undefined; + } + + /// Reads the object at the current position in the database. + fn readObject(odb: *Odb) !Object { + var base_offset = try odb.pack_file.getPos(); + var base_header: EntryHeader = undefined; + var delta_offsets = std.ArrayListUnmanaged(u64){}; + defer delta_offsets.deinit(odb.allocator); + const base_object = while (true) { + if (odb.cache.get(base_offset)) |base_object| break base_object; + + base_header = try EntryHeader.read(odb.pack_file.reader()); + switch (base_header) { + .ofs_delta => |ofs_delta| { + try delta_offsets.append(odb.allocator, base_offset); + base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; + try odb.pack_file.seekTo(base_offset); + }, + .ref_delta => |ref_delta| { + try delta_offsets.append(odb.allocator, base_offset); + try odb.seekOid(ref_delta.base_object); + base_offset = try odb.pack_file.getPos(); + }, + else => { + const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength()); + errdefer odb.allocator.free(base_data); + const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; + try odb.cache.put(odb.allocator, base_offset, base_object); + break base_object; + }, + } + }; + + const base_data = try resolveDeltaChain( + odb.allocator, + odb.pack_file, + base_object, + delta_offsets.items, + &odb.cache, + ); + + return .{ .type = base_object.type, .data = base_data }; + } + + /// Seeks to the beginning of the object with the given ID. + fn seekOid(odb: *Odb, oid: Oid) !void { + const key = oid[0]; + var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; + var end_index = odb.index_header.fan_out_table[key]; + const found_index = while (start_index < end_index) { + const mid_index = start_index + (end_index - start_index) / 2; + try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); + const mid_oid = try odb.index_file.reader().readBytesNoEof(oid_length); + switch (mem.order(u8, &mid_oid, &oid)) { + .lt => start_index = mid_index + 1, + .gt => end_index = mid_index, + .eq => break mid_index, + } + } else return error.ObjectNotFound; + + const n_objects = odb.index_header.fan_out_table[255]; + const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); + try odb.index_file.seekTo(offset_values_start + found_index * 4); + const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readIntBig(u32)); + const pack_offset = pack_offset: { + if (l1_offset.big) { + const l2_offset_values_start = offset_values_start + n_objects * 4; + try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); + break :pack_offset try odb.index_file.reader().readIntBig(u64); + } else { + break :pack_offset l1_offset.value; + } + }; + + try odb.pack_file.seekTo(pack_offset); + } +}; + +const Object = struct { + type: Type, + data: []const u8, + + const Type = enum { + commit, + tree, + blob, + tag, + }; +}; + +/// A cache for object data. +/// +/// The purpose of this cache is to speed up resolution of deltas by caching the +/// results of resolving delta objects, while maintaining a maximum cache size +/// to avoid excessive memory usage. If the total size of the objects in the +/// cache exceeds the maximum, the cache will begin evicting the least recently +/// used objects: when resolving delta chains, the most recently used objects +/// will likely be more helpful as they will be further along in the chain +/// (skipping earlier reconstruction steps). +/// +/// Object data stored in the cache is managed by the cache. It should not be +/// freed by the caller at any point after inserting it into the cache. Any +/// objects remaining in the cache will be freed when the cache itself is freed. +const ObjectCache = struct { + objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .{}, + lru_nodes: LruList = .{}, + byte_size: usize = 0, + + const max_byte_size = 128 * 1024 * 1024; // 128MiB + /// A list of offsets stored in the cache, with the most recently used + /// entries at the end. + const LruList = std.DoublyLinkedList(u64); + const CacheEntry = struct { object: Object, lru_node: *LruList.Node }; + + fn deinit(cache: *ObjectCache, allocator: Allocator) void { + var object_iterator = cache.objects.iterator(); + while (object_iterator.next()) |object| { + allocator.free(object.value_ptr.object.data); + allocator.destroy(object.value_ptr.lru_node); + } + cache.objects.deinit(allocator); + cache.* = undefined; + } + + /// Gets an object from the cache, moving it to the most recently used + /// position if it is present. + fn get(cache: *ObjectCache, offset: u64) ?Object { + if (cache.objects.get(offset)) |entry| { + cache.lru_nodes.remove(entry.lru_node); + cache.lru_nodes.append(entry.lru_node); + return entry.object; + } else { + return null; + } + } + + /// Puts an object in the cache, possibly evicting older entries if the + /// cache exceeds its maximum size. Note that, although old objects may + /// be evicted, the object just added to the cache with this function + /// will not be evicted before the next call to `put` or `deinit` even if + /// it exceeds the maximum cache size. + fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void { + const lru_node = try allocator.create(LruList.Node); + errdefer allocator.destroy(lru_node); + lru_node.data = offset; + + const gop = try cache.objects.getOrPut(allocator, offset); + if (gop.found_existing) { + cache.byte_size -= gop.value_ptr.object.data.len; + cache.lru_nodes.remove(gop.value_ptr.lru_node); + allocator.destroy(gop.value_ptr.lru_node); + allocator.free(gop.value_ptr.object.data); + } + gop.value_ptr.* = .{ .object = object, .lru_node = lru_node }; + cache.byte_size += object.data.len; + cache.lru_nodes.append(lru_node); + + while (cache.byte_size > max_byte_size and cache.lru_nodes.len > 1) { + // The > 1 check is to make sure that we don't evict the most + // recently added node, even if it by itself happens to exceed the + // maximum size of the cache. + const evict_node = cache.lru_nodes.popFirst().?; + const evict_offset = evict_node.data; + allocator.destroy(evict_node); + const evict_object = cache.objects.get(evict_offset).?.object; + cache.byte_size -= evict_object.data.len; + allocator.free(evict_object.data); + _ = cache.objects.remove(evict_offset); + } + } +}; + +/// A single pkt-line in the Git protocol. +/// +/// The format of a pkt-line is documented in +/// [protocol-common](https://git-scm.com/docs/protocol-common). The special +/// meanings of the delimiter and response-end packets are documented in +/// [protocol-v2](https://git-scm.com/docs/protocol-v2). +const Packet = union(enum) { + flush, + delimiter, + response_end, + data: []const u8, + + const max_data_length = 65516; + + /// Reads a packet in pkt-line format. + fn read(reader: anytype, buf: *[max_data_length]u8) !Packet { + const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket; + switch (length) { + 0 => return .flush, + 1 => return .delimiter, + 2 => return .response_end, + 3 => return error.InvalidPacket, + else => if (length - 4 > max_data_length) return error.InvalidPacket, + } + const data = buf[0 .. length - 4]; + try reader.readNoEof(data); + return .{ .data = data }; + } + + /// Writes a packet in pkt-line format. + fn write(packet: Packet, writer: anytype) !void { + switch (packet) { + .flush => try writer.writeAll("0000"), + .delimiter => try writer.writeAll("0001"), + .response_end => try writer.writeAll("0002"), + .data => |data| { + assert(data.len <= max_data_length); + try writer.print("{x:0>4}", .{data.len + 4}); + try writer.writeAll(data); + }, + } + } +}; + +/// A client session for the Git protocol, currently limited to an HTTP(S) +/// transport. Only protocol version 2 is supported, as documented in +/// [protocol-v2](https://git-scm.com/docs/protocol-v2). +pub const Session = struct { + transport: *std.http.Client, + uri: std.Uri, + supports_agent: bool = false, + supports_shallow: bool = false, + + const agent = "zig/" ++ @import("builtin").zig_version_string; + const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent}); + + /// Discovers server capabilities. This should be called before using any + /// other client functionality, or the client will be forced to default to + /// the bare minimum server requirements, which may be considerably less + /// efficient (e.g. no shallow fetches). + /// + /// See the note on `getCapabilities` regarding `redirect_uri`. + pub fn discoverCapabilities( + session: *Session, + allocator: Allocator, + redirect_uri: *[]u8, + ) !void { + var capability_iterator = try session.getCapabilities(allocator, redirect_uri); + defer capability_iterator.deinit(); + while (try capability_iterator.next()) |capability| { + if (mem.eql(u8, capability.key, "agent")) { + session.supports_agent = true; + } else if (mem.eql(u8, capability.key, "fetch")) { + var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' '); + while (feature_iterator.next()) |feature| { + if (mem.eql(u8, feature, "shallow")) { + session.supports_shallow = true; + } + } + } + } + } + + /// Returns an iterator over capabilities supported by the server. + /// + /// If the server redirects the request, `error.Redirected` is returned and + /// `redirect_uri` is populated with the URI resulting from the redirects. + /// When this occurs, the value of `redirect_uri` must be freed with + /// `allocator` when the caller is done with it. + fn getCapabilities( + session: Session, + allocator: Allocator, + redirect_uri: *[]u8, + ) !CapabilityIterator { + var info_refs_uri = session.uri; + info_refs_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "info/refs" }); + defer allocator.free(info_refs_uri.path); + info_refs_uri.query = "service=git-upload-pack"; + info_refs_uri.fragment = null; + + var headers = std.http.Headers.init(allocator); + defer headers.deinit(); + try headers.append("Git-Protocol", "version=2"); + + var request = try session.transport.request(.GET, info_refs_uri, headers, .{ + .max_redirects = 3, + }); + errdefer request.deinit(); + try request.start(.{}); + try request.finish(); + + try request.wait(); + if (request.response.status != .ok) return error.ProtocolError; + if (request.redirects_left < 3) { + if (!mem.endsWith(u8, request.uri.path, "/info/refs")) return error.UnparseableRedirect; + var new_uri = request.uri; + new_uri.path = new_uri.path[0 .. new_uri.path.len - "/info/refs".len]; + new_uri.query = null; + redirect_uri.* = try std.fmt.allocPrint(allocator, "{+/}", .{new_uri}); + return error.Redirected; + } + + const reader = request.reader(); + var buf: [Packet.max_data_length]u8 = undefined; + var state: enum { response_start, response_content } = .response_start; + while (true) { + // Some Git servers (at least GitHub) include an additional + // '# service=git-upload-pack' informative response before sending + // the expected 'version 2' packet and capability information. + // This is not universal: SourceHut, for example, does not do this. + // Thus, we need to skip any such useless additional responses + // before we get the one we're actually looking for. The responses + // will be delimited by flush packets. + const packet = Packet.read(reader, &buf) catch |e| switch (e) { + error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found + else => |other| return other, + }; + switch (packet) { + .flush => state = .response_start, + .data => |data| switch (state) { + .response_start => if (mem.eql(u8, data, "version 2\n")) { + return .{ .request = request }; + } else { + state = .response_content; + }, + else => {}, + }, + else => return error.UnexpectedPacket, + } + } + } + + const CapabilityIterator = struct { + request: std.http.Client.Request, + buf: [Packet.max_data_length]u8 = undefined, + + const Capability = struct { + key: []const u8, + value: ?[]const u8 = null, + }; + + fn deinit(iterator: *CapabilityIterator) void { + iterator.request.deinit(); + iterator.* = undefined; + } + + fn next(iterator: *CapabilityIterator) !?Capability { + switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { + .flush => return null, + .data => |data| if (data.len > 0 and data[data.len - 1] == '\n') { + if (mem.indexOfScalar(u8, data, '=')) |separator_pos| { + return .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 .. data.len - 1] }; + } else { + return .{ .key = data[0 .. data.len - 1] }; + } + } else return error.UnexpectedPacket, + else => return error.UnexpectedPacket, + } + } + }; + + const ListRefsOptions = struct { + /// The ref prefixes (if any) to use to filter the refs available on the + /// server. Note that the client must still check the returned refs + /// against its desired filters itself: the server is not required to + /// respect these prefix filters and may return other refs as well. + ref_prefixes: []const []const u8 = &.{}, + /// Whether to include symref targets for returned symbolic refs. + include_symrefs: bool = false, + /// Whether to include the peeled object ID for returned tag refs. + include_peeled: bool = false, + }; + + /// Returns an iterator over refs known to the server. + pub fn listRefs(session: Session, allocator: Allocator, options: ListRefsOptions) !RefIterator { + var upload_pack_uri = session.uri; + upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" }); + defer allocator.free(upload_pack_uri.path); + upload_pack_uri.query = null; + upload_pack_uri.fragment = null; + + var headers = std.http.Headers.init(allocator); + defer headers.deinit(); + try headers.append("Content-Type", "application/x-git-upload-pack-request"); + try headers.append("Git-Protocol", "version=2"); + + var body = std.ArrayListUnmanaged(u8){}; + defer body.deinit(allocator); + const body_writer = body.writer(allocator); + try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer); + if (session.supports_agent) { + try Packet.write(.{ .data = agent_capability }, body_writer); + } + try Packet.write(.delimiter, body_writer); + for (options.ref_prefixes) |ref_prefix| { + const ref_prefix_packet = try std.fmt.allocPrint(allocator, "ref-prefix {s}\n", .{ref_prefix}); + defer allocator.free(ref_prefix_packet); + try Packet.write(.{ .data = ref_prefix_packet }, body_writer); + } + if (options.include_symrefs) { + try Packet.write(.{ .data = "symrefs\n" }, body_writer); + } + if (options.include_peeled) { + try Packet.write(.{ .data = "peel\n" }, body_writer); + } + try Packet.write(.flush, body_writer); + + var request = try session.transport.request(.POST, upload_pack_uri, headers, .{ + .handle_redirects = false, + }); + errdefer request.deinit(); + request.transfer_encoding = .{ .content_length = body.items.len }; + try request.start(.{}); + try request.writeAll(body.items); + try request.finish(); + + try request.wait(); + if (request.response.status != .ok) return error.ProtocolError; + + return .{ .request = request }; + } + + pub const RefIterator = struct { + request: std.http.Client.Request, + buf: [Packet.max_data_length]u8 = undefined, + + pub const Ref = struct { + oid: Oid, + name: []const u8, + symref_target: ?[]const u8, + peeled: ?Oid, + }; + + pub fn deinit(iterator: *RefIterator) void { + iterator.request.deinit(); + iterator.* = undefined; + } + + pub fn next(iterator: *RefIterator) !?Ref { + switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { + .flush => return null, + .data => |data| { + const oid_sep_pos = mem.indexOfScalar(u8, data, ' ') orelse return error.InvalidRefPacket; + const oid = parseOid(data[0..oid_sep_pos]) catch return error.InvalidRefPacket; + + const name_sep_pos = mem.indexOfAnyPos(u8, data, oid_sep_pos + 1, " \n") orelse return error.InvalidRefPacket; + const name = data[oid_sep_pos + 1 .. name_sep_pos]; + + var symref_target: ?[]const u8 = null; + var peeled: ?Oid = null; + var last_sep_pos = name_sep_pos; + while (data[last_sep_pos] == ' ') { + const next_sep_pos = mem.indexOfAnyPos(u8, data, last_sep_pos + 1, " \n") orelse return error.InvalidRefPacket; + const attribute = data[last_sep_pos + 1 .. next_sep_pos]; + if (mem.startsWith(u8, attribute, "symref-target:")) { + symref_target = attribute["symref-target:".len..]; + } else if (mem.startsWith(u8, attribute, "peeled:")) { + peeled = parseOid(attribute["peeled:".len..]) catch return error.InvalidRefPacket; + } + last_sep_pos = next_sep_pos; + } + + return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled }; + }, + else => return error.UnexpectedPacket, + } + } + }; + + /// Fetches the given refs from the server. A shallow fetch (depth 1) is + /// performed if the server supports it. + pub fn fetch(session: Session, allocator: Allocator, wants: []const []const u8) !FetchStream { + var upload_pack_uri = session.uri; + upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" }); + defer allocator.free(upload_pack_uri.path); + upload_pack_uri.query = null; + upload_pack_uri.fragment = null; + + var headers = std.http.Headers.init(allocator); + defer headers.deinit(); + try headers.append("Content-Type", "application/x-git-upload-pack-request"); + try headers.append("Git-Protocol", "version=2"); + + var body = std.ArrayListUnmanaged(u8){}; + defer body.deinit(allocator); + const body_writer = body.writer(allocator); + try Packet.write(.{ .data = "command=fetch\n" }, body_writer); + if (session.supports_agent) { + try Packet.write(.{ .data = agent_capability }, body_writer); + } + try Packet.write(.delimiter, body_writer); + // Our packfile parser supports the OFS_DELTA object type + try Packet.write(.{ .data = "ofs-delta\n" }, body_writer); + // We do not currently convey server progress information to the user + try Packet.write(.{ .data = "no-progress\n" }, body_writer); + if (session.supports_shallow) { + try Packet.write(.{ .data = "deepen 1\n" }, body_writer); + } + for (wants) |want| { + var buf: [Packet.max_data_length]u8 = undefined; + const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable; + try Packet.write(.{ .data = arg }, body_writer); + } + try Packet.write(.{ .data = "done\n" }, body_writer); + try Packet.write(.flush, body_writer); + + var request = try session.transport.request(.POST, upload_pack_uri, headers, .{ + .handle_redirects = false, + }); + errdefer request.deinit(); + request.transfer_encoding = .{ .content_length = body.items.len }; + try request.start(.{}); + try request.writeAll(body.items); + try request.finish(); + + try request.wait(); + if (request.response.status != .ok) return error.ProtocolError; + + const reader = request.reader(); + // We are not interested in any of the sections of the returned fetch + // data other than the packfile section, since we aren't doing anything + // complex like ref negotiation (this is a fresh clone). + var state: enum { section_start, section_content } = .section_start; + while (true) { + var buf: [Packet.max_data_length]u8 = undefined; + const packet = try Packet.read(reader, &buf); + switch (state) { + .section_start => switch (packet) { + .data => |data| if (mem.eql(u8, data, "packfile\n")) { + return .{ .request = request }; + } else { + state = .section_content; + }, + else => return error.UnexpectedPacket, + }, + .section_content => switch (packet) { + .delimiter => state = .section_start, + .data => {}, + else => return error.UnexpectedPacket, + }, + } + } + } + + pub const FetchStream = struct { + request: std.http.Client.Request, + buf: [Packet.max_data_length]u8 = undefined, + pos: usize = 0, + len: usize = 0, + + pub fn deinit(stream: *FetchStream) void { + stream.request.deinit(); + } + + pub const ReadError = std.http.Client.Request.ReadError || error{ + InvalidPacket, + ProtocolError, + UnexpectedPacket, + }; + pub const Reader = std.io.Reader(*FetchStream, ReadError, read); + + const StreamCode = enum(u8) { + pack_data = 1, + progress = 2, + fatal_error = 3, + _, + }; + + pub fn reader(stream: *FetchStream) Reader { + return .{ .context = stream }; + } + + pub fn read(stream: *FetchStream, buf: []u8) !usize { + if (stream.pos == stream.len) { + while (true) { + switch (try Packet.read(stream.request.reader(), &stream.buf)) { + .flush => return 0, + .data => |data| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) { + .pack_data => { + stream.pos = 1; + stream.len = data.len; + break; + }, + .fatal_error => return error.ProtocolError, + else => {}, + }, + else => return error.UnexpectedPacket, + } + } + } + + const size = @min(buf.len, stream.len - stream.pos); + @memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]); + stream.pos += size; + return size; + } + }; +}; + +const PackHeader = struct { + total_objects: u32, + + const signature = "PACK"; + const supported_version = 2; + + fn read(reader: anytype) !PackHeader { + const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader; + const version = reader.readIntBig(u32) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + if (version != supported_version) return error.UnsupportedVersion; + const total_objects = reader.readIntBig(u32) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + return .{ .total_objects = total_objects }; + } +}; + +const EntryHeader = union(Type) { + commit: Undeltified, + tree: Undeltified, + blob: Undeltified, + tag: Undeltified, + ofs_delta: OfsDelta, + ref_delta: RefDelta, + + const Type = enum(u3) { + commit = 1, + tree = 2, + blob = 3, + tag = 4, + ofs_delta = 6, + ref_delta = 7, + }; + + const Undeltified = struct { + uncompressed_length: u64, + }; + + const OfsDelta = struct { + offset: u64, + uncompressed_length: u64, + }; + + const RefDelta = struct { + base_object: Oid, + uncompressed_length: u64, + }; + + fn objectType(header: EntryHeader) Object.Type { + return switch (header) { + inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)), + else => unreachable, + }; + } + + fn uncompressedLength(header: EntryHeader) u64 { + return switch (header) { + inline else => |entry| entry.uncompressed_length, + }; + } + + fn read(reader: anytype) !EntryHeader { + const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; + const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { + error.EndOfStream => return error.InvalidFormat, + else => |other| return other, + }); + const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0; + var uncompressed_length: u64 = initial.len; + uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; + const @"type" = std.meta.intToEnum(EntryHeader.Type, initial.type) catch return error.InvalidFormat; + return switch (@"type") { + inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{ + .uncompressed_length = uncompressed_length, + }), + .ofs_delta => .{ .ofs_delta = .{ + .offset = try readOffsetVarInt(reader), + .uncompressed_length = uncompressed_length, + } }, + .ref_delta => .{ .ref_delta = .{ + .base_object = reader.readBytesNoEof(oid_length) catch |e| switch (e) { + error.EndOfStream => return error.InvalidFormat, + else => |other| return other, + }, + .uncompressed_length = uncompressed_length, + } }, + }; + } +}; + +fn readSizeVarInt(r: anytype) !u64 { + const Byte = packed struct { value: u7, has_next: bool }; + var b: Byte = @bitCast(try r.readByte()); + var value: u64 = b.value; + var shift: u6 = 0; + while (b.has_next) { + b = @bitCast(try r.readByte()); + shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat; + value |= @as(u64, b.value) << shift; + } + return value; +} + +fn readOffsetVarInt(r: anytype) !u64 { + const Byte = packed struct { value: u7, has_next: bool }; + var b: Byte = @bitCast(try r.readByte()); + var value: u64 = b.value; + while (b.has_next) { + b = @bitCast(try r.readByte()); + value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; + value |= b.value; + } + return value; +} + +const IndexHeader = struct { + fan_out_table: [256]u32, + + const signature = "\xFFtOc"; + const supported_version = 2; + const size = 4 + 4 + @sizeOf([256]u32); + + fn read(reader: anytype) !IndexHeader { + var header_bytes = try reader.readBytesNoEof(size); + if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader; + const version = mem.readIntBig(u32, header_bytes[4..8]); + if (version != supported_version) return error.UnsupportedVersion; + + var fan_out_table: [256]u32 = undefined; + var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]); + const fan_out_table_reader = fan_out_table_stream.reader(); + for (&fan_out_table) |*entry| { + entry.* = fan_out_table_reader.readIntBig(u32) catch unreachable; + } + return .{ .fan_out_table = fan_out_table }; + } +}; + +const IndexEntry = struct { + offset: u64, + crc32: u32, +}; + +/// Writes out a version 2 index for the given packfile, as documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) !void { + try pack.seekTo(0); + + var index_entries = std.AutoHashMapUnmanaged(Oid, IndexEntry){}; + defer index_entries.deinit(allocator); + var pending_deltas = std.ArrayListUnmanaged(IndexEntry){}; + defer pending_deltas.deinit(allocator); + + const pack_checksum = try indexPackFirstPass(allocator, pack, &index_entries, &pending_deltas); + + var cache: ObjectCache = .{}; + defer cache.deinit(allocator); + var remaining_deltas = pending_deltas.items.len; + while (remaining_deltas > 0) { + var i: usize = remaining_deltas; + while (i > 0) { + i -= 1; + const delta = pending_deltas.items[i]; + if (try indexPackHashDelta(allocator, pack, delta, index_entries, &cache)) |oid| { + try index_entries.put(allocator, oid, delta); + _ = pending_deltas.swapRemove(i); + } + } + if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack; + remaining_deltas = pending_deltas.items.len; + } + + var oids = std.ArrayListUnmanaged(Oid){}; + defer oids.deinit(allocator); + try oids.ensureTotalCapacityPrecise(allocator, index_entries.count()); + var index_entries_iter = index_entries.iterator(); + while (index_entries_iter.next()) |entry| { + oids.appendAssumeCapacity(entry.key_ptr.*); + } + mem.sortUnstable(Oid, oids.items, {}, struct { + fn lessThan(_: void, o1: Oid, o2: Oid) bool { + return mem.lessThan(u8, &o1, &o2); + } + }.lessThan); + + var fan_out_table: [256]u32 = undefined; + var count: u32 = 0; + var fan_out_index: u8 = 0; + for (oids.items) |oid| { + if (oid[0] > fan_out_index) { + @memset(fan_out_table[fan_out_index..oid[0]], count); + fan_out_index = oid[0]; + } + count += 1; + } + @memset(fan_out_table[fan_out_index..], count); + + var index_hashed_writer = hashedWriter(index_writer, Sha1.init(.{})); + const writer = index_hashed_writer.writer(); + try writer.writeAll(IndexHeader.signature); + try writer.writeIntBig(u32, IndexHeader.supported_version); + for (fan_out_table) |fan_out_entry| { + try writer.writeIntBig(u32, fan_out_entry); + } + + for (oids.items) |oid| { + try writer.writeAll(&oid); + } + + for (oids.items) |oid| { + try writer.writeIntBig(u32, index_entries.get(oid).?.crc32); + } + + var big_offsets = std.ArrayListUnmanaged(u64){}; + defer big_offsets.deinit(allocator); + for (oids.items) |oid| { + const offset = index_entries.get(oid).?.offset; + if (offset <= std.math.maxInt(u31)) { + try writer.writeIntBig(u32, @intCast(offset)); + } else { + const index = big_offsets.items.len; + try big_offsets.append(allocator, offset); + try writer.writeIntBig(u32, @as(u32, @intCast(index)) | (1 << 31)); + } + } + for (big_offsets.items) |offset| { + try writer.writeIntBig(u64, offset); + } + + try writer.writeAll(&pack_checksum); + const index_checksum = index_hashed_writer.hasher.finalResult(); + try index_writer.writeAll(&index_checksum); +} + +/// Performs the first pass over the packfile data for index construction. +/// This will index all non-delta objects, queue delta objects for further +/// processing, and return the pack checksum (which is part of the index +/// format). +fn indexPackFirstPass( + allocator: Allocator, + pack: std.fs.File, + index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), + pending_deltas: *std.ArrayListUnmanaged(IndexEntry), +) ![Sha1.digest_length]u8 { + var pack_buffered_reader = std.io.bufferedReader(pack.reader()); + var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader()); + var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Sha1.init(.{})); + const pack_reader = pack_hashed_reader.reader(); + + const pack_header = try PackHeader.read(pack_reader); + + var current_entry: u32 = 0; + while (current_entry < pack_header.total_objects) : (current_entry += 1) { + const entry_offset = pack_counting_reader.bytes_read; + var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init()); + const entry_header = try EntryHeader.read(entry_crc32_reader.reader()); + switch (entry_header) { + inline .commit, .tree, .blob, .tag => |object, tag| { + var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader()); + defer entry_decompress_stream.deinit(); + var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); + var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{})); + const entry_writer = entry_hashed_writer.writer(); + // The object header is not included in the pack data but is + // part of the object's ID + try entry_writer.print("{s} {}\x00", .{ @tagName(tag), object.uncompressed_length }); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + try fifo.pump(entry_counting_reader.reader(), entry_writer); + if (entry_counting_reader.bytes_read != object.uncompressed_length) { + return error.InvalidObject; + } + const oid = entry_hashed_writer.hasher.finalResult(); + try index_entries.put(allocator, oid, .{ + .offset = entry_offset, + .crc32 = entry_crc32_reader.hasher.final(), + }); + }, + inline .ofs_delta, .ref_delta => |delta| { + var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader()); + defer entry_decompress_stream.deinit(); + var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); + if (entry_counting_reader.bytes_read != delta.uncompressed_length) { + return error.InvalidObject; + } + try pending_deltas.append(allocator, .{ + .offset = entry_offset, + .crc32 = entry_crc32_reader.hasher.final(), + }); + }, + } + } + + const pack_checksum = pack_hashed_reader.hasher.finalResult(); + const recorded_checksum = try pack_buffered_reader.reader().readBytesNoEof(Sha1.digest_length); + if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) { + return error.CorruptedPack; + } + _ = pack_buffered_reader.reader().readByte() catch |e| switch (e) { + error.EndOfStream => return pack_checksum, + else => |other| return other, + }; + return error.InvalidFormat; +} + +/// Attempts to determine the final object ID of the given deltified object. +/// May return null if this is not yet possible (if the delta is a ref-based +/// delta and we do not yet know the offset of the base object). +fn indexPackHashDelta( + allocator: Allocator, + pack: std.fs.File, + delta: IndexEntry, + index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), + cache: *ObjectCache, +) !?Oid { + // Figure out the chain of deltas to resolve + var base_offset = delta.offset; + var base_header: EntryHeader = undefined; + var delta_offsets = std.ArrayListUnmanaged(u64){}; + defer delta_offsets.deinit(allocator); + const base_object = while (true) { + if (cache.get(base_offset)) |base_object| break base_object; + + try pack.seekTo(base_offset); + base_header = try EntryHeader.read(pack.reader()); + switch (base_header) { + .ofs_delta => |ofs_delta| { + try delta_offsets.append(allocator, base_offset); + base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject; + }, + .ref_delta => |ref_delta| { + try delta_offsets.append(allocator, base_offset); + base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; + }, + else => { + const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength()); + errdefer allocator.free(base_data); + const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; + try cache.put(allocator, base_offset, base_object); + break base_object; + }, + } + }; + + const base_data = try resolveDeltaChain(allocator, pack, base_object, delta_offsets.items, cache); + + var entry_hasher = Sha1.init(.{}); + var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher); + try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); + entry_hasher.update(base_data); + return entry_hasher.finalResult(); +} + +/// Resolves a chain of deltas, returning the final base object data. `pack` is +/// assumed to be looking at the start of the object data for the base object of +/// the chain, and will then apply the deltas in `delta_offsets` in reverse order +/// to obtain the final object. +fn resolveDeltaChain( + allocator: Allocator, + pack: std.fs.File, + base_object: Object, + delta_offsets: []const u64, + cache: *ObjectCache, +) ![]const u8 { + var base_data = base_object.data; + var i: usize = delta_offsets.len; + while (i > 0) { + i -= 1; + + const delta_offset = delta_offsets[i]; + try pack.seekTo(delta_offset); + const delta_header = try EntryHeader.read(pack.reader()); + var delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength()); + defer allocator.free(delta_data); + var delta_stream = std.io.fixedBufferStream(delta_data); + const delta_reader = delta_stream.reader(); + _ = try readSizeVarInt(delta_reader); // base object size + const expanded_size = try readSizeVarInt(delta_reader); + + const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; + var expanded_data = try allocator.alloc(u8, expanded_alloc_size); + errdefer allocator.free(expanded_data); + var expanded_delta_stream = std.io.fixedBufferStream(expanded_data); + var base_stream = std.io.fixedBufferStream(base_data); + try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer()); + if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject; + + try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); + base_data = expanded_data; + } + return base_data; +} + +/// Reads the complete contents of an object from `reader`. This function may +/// read more bytes than required from `reader`, so the reader position after +/// returning is not reliable. +fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { + const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; + var buffered_reader = std.io.bufferedReader(reader); + var decompress_stream = try std.compress.zlib.decompressStream(allocator, buffered_reader.reader()); + defer decompress_stream.deinit(); + var data = try allocator.alloc(u8, alloc_size); + errdefer allocator.free(data); + try decompress_stream.reader().readNoEof(data); + _ = decompress_stream.reader().readByte() catch |e| switch (e) { + error.EndOfStream => return data, + else => |other| return other, + }; + return error.InvalidFormat; +} + +/// Expands delta data from `delta_reader` to `writer`. `base_object` must +/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`). +/// +/// The format of the delta data is documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void { + while (true) { + const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) { + error.EndOfStream => return, + else => |other| return other, + }); + if (inst.copy) { + const available: packed struct { + offset1: bool, + offset2: bool, + offset3: bool, + offset4: bool, + size1: bool, + size2: bool, + size3: bool, + } = @bitCast(inst.value); + var offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ + .offset1 = if (available.offset1) try delta_reader.readByte() else 0, + .offset2 = if (available.offset2) try delta_reader.readByte() else 0, + .offset3 = if (available.offset3) try delta_reader.readByte() else 0, + .offset4 = if (available.offset4) try delta_reader.readByte() else 0, + }; + const offset: u32 = @bitCast(offset_parts); + var size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ + .size1 = if (available.size1) try delta_reader.readByte() else 0, + .size2 = if (available.size2) try delta_reader.readByte() else 0, + .size3 = if (available.size3) try delta_reader.readByte() else 0, + }; + var size: u24 = @bitCast(size_parts); + if (size == 0) size = 0x10000; + try base_object.seekTo(offset); + var copy_reader = std.io.limitedReader(base_object.reader(), size); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + try fifo.pump(copy_reader.reader(), writer); + } else if (inst.value != 0) { + var data_reader = std.io.limitedReader(delta_reader, inst.value); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + try fifo.pump(data_reader.reader(), writer); + } else { + return error.InvalidDeltaInstruction; + } + } +} + +fn HashedWriter( + comptime WriterType: anytype, + comptime HasherType: anytype, +) type { + return struct { + child_writer: WriterType, + hasher: HasherType, + + const Error = WriterType.Error; + const Writer = std.io.Writer(*@This(), Error, write); + + fn write(hashed_writer: *@This(), buf: []const u8) Error!usize { + const amt = try hashed_writer.child_writer.write(buf); + hashed_writer.hasher.update(buf); + return amt; + } + + fn writer(hashed_writer: *@This()) Writer { + return .{ .context = hashed_writer }; + } + }; +} + +fn hashedWriter( + writer: anytype, + hasher: anytype, +) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) { + return .{ .child_writer = writer, .hasher = hasher }; +} + +test "packfile indexing and checkout" { + // To verify the contents of this packfile without using the code in this + // file: + // + // 1. Create a new empty Git repository (`git init`) + // 2. `git unpack-objects note the "dangling commit" ID (which matches the commit + // checked out below) + // 4. `git checkout dd582c0720819ab7130b103635bd7271b9fd4feb` + const testrepo_pack = @embedFile("git/testdata/testrepo.pack"); + + var git_dir = testing.tmpDir(.{}); + defer git_dir.cleanup(); + var pack_file = try git_dir.dir.createFile("testrepo.pack", .{ .read = true }); + defer pack_file.close(); + try pack_file.writeAll(testrepo_pack); + + var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); + defer index_file.close(); + try indexPack(testing.allocator, pack_file, index_file.writer()); + + // Arbitrary size limit on files read while checking the repository contents + // (all files in the test repo are known to be much smaller than this) + const max_file_size = 4096; + + const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size); + defer testing.allocator.free(index_file_data); + // testrepo.idx is generated by Git. The index created by this file should + // match it exactly. Running `git verify-pack -v testrepo.pack` can verify + // this. + const testrepo_idx = @embedFile("git/testdata/testrepo.idx"); + try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); + + var repository = try Repository.init(testing.allocator, pack_file, index_file); + defer repository.deinit(); + + var worktree = testing.tmpIterableDir(.{}); + defer worktree.cleanup(); + + const commit_id = try parseOid("dd582c0720819ab7130b103635bd7271b9fd4feb"); + try repository.checkout(worktree.iterable_dir.dir, commit_id); + + const expected_files: []const []const u8 = &.{ + "dir/file", + "dir/subdir/file", + "dir/subdir/file2", + "dir2/file", + "dir3/file", + "dir3/file2", + "file", + "file2", + "file3", + "file4", + "file5", + "file6", + "file7", + "file8", + "file9", + }; + var actual_files: std.ArrayListUnmanaged([]u8) = .{}; + defer actual_files.deinit(testing.allocator); + defer for (actual_files.items) |file| testing.allocator.free(file); + var walker = try worktree.iterable_dir.walk(testing.allocator); + defer walker.deinit(); + while (try walker.next()) |entry| { + if (entry.kind != .file) continue; + var path = try testing.allocator.dupe(u8, entry.path); + errdefer testing.allocator.free(path); + mem.replaceScalar(u8, path, std.fs.path.sep, '/'); + try actual_files.append(testing.allocator, path); + } + mem.sortUnstable([]u8, actual_files.items, {}, struct { + fn lessThan(_: void, a: []u8, b: []u8) bool { + return mem.lessThan(u8, a, b); + } + }.lessThan); + try testing.expectEqualDeep(expected_files, actual_files.items); + + const expected_file_contents = + \\revision 1 + \\revision 2 + \\revision 4 + \\revision 5 + \\revision 7 + \\revision 8 + \\revision 9 + \\revision 10 + \\revision 12 + \\revision 13 + \\revision 14 + \\revision 18 + \\revision 19 + \\ + ; + const actual_file_contents = try worktree.iterable_dir.dir.readFileAlloc(testing.allocator, "file", max_file_size); + defer testing.allocator.free(actual_file_contents); + try testing.expectEqualStrings(expected_file_contents, actual_file_contents); +} + +/// Checks out a commit of a packfile. Intended for experimenting with and +/// benchmarking possible optimizations to the indexing and checkout behavior. +pub fn main() !void { + const allocator = std.heap.c_allocator; + + const args = try std.process.argsAlloc(allocator); + defer std.process.argsFree(allocator, args); + if (args.len != 4) { + return error.InvalidArguments; // Arguments: packfile commit worktree + } + + var pack_file = try std.fs.cwd().openFile(args[1], .{}); + defer pack_file.close(); + const commit = try parseOid(args[2]); + var worktree = try std.fs.cwd().makeOpenPath(args[3], .{}); + defer worktree.close(); + + var git_dir = try worktree.makeOpenPath(".git", .{}); + defer git_dir.close(); + + std.debug.print("Starting index...\n", .{}); + var index_file = try git_dir.createFile("idx", .{ .read = true }); + defer index_file.close(); + var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); + try indexPack(allocator, pack_file, index_buffered_writer.writer()); + try index_buffered_writer.flush(); + try index_file.sync(); + + std.debug.print("Starting checkout...\n", .{}); + var repository = try Repository.init(allocator, pack_file, index_file); + defer repository.deinit(); + try repository.checkout(worktree, commit); +} diff --git a/src/Package/Fetch/git/testdata/testrepo.idx b/src/Package/Fetch/git/testdata/testrepo.idx new file mode 100644 index 0000000000000000000000000000000000000000..fdaba5ed4863036d855577c3f636696b51829955 Binary files /dev/null and b/src/Package/Fetch/git/testdata/testrepo.idx differ diff --git a/src/Package/Fetch/git/testdata/testrepo.pack b/src/Package/Fetch/git/testdata/testrepo.pack new file mode 100644 index 0000000000000000000000000000000000000000..1b2f5dd650f613973c72ae4266d35e744bd0f308 Binary files /dev/null and b/src/Package/Fetch/git/testdata/testrepo.pack differ diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig new file mode 100644 index 0000000000000000000000000000000000000000..c1b1cdfb4f3ca77281a75eef55c43e8fabe182f5 --- /dev/null +++ b/src/Package/Manifest.zig @@ -0,0 +1,566 @@ +pub const max_bytes = 10 * 1024 * 1024; +pub const basename = "build.zig.zon"; +pub const Hash = std.crypto.hash.sha2.Sha256; +pub const Digest = [Hash.digest_length]u8; +pub const multihash_len = 1 + 1 + Hash.digest_length; +pub const multihash_hex_digest_len = 2 * multihash_len; +pub const MultiHashHexDigest = [multihash_hex_digest_len]u8; + +pub const Dependency = struct { + location: Location, + location_tok: Ast.TokenIndex, + hash: ?[]const u8, + hash_tok: Ast.TokenIndex, + + pub const Location = union(enum) { + url: []const u8, + path: []const u8, + }; +}; + +pub const ErrorMessage = struct { + msg: []const u8, + tok: Ast.TokenIndex, + off: u32, +}; + +pub const MultihashFunction = enum(u16) { + identity = 0x00, + sha1 = 0x11, + @"sha2-256" = 0x12, + @"sha2-512" = 0x13, + @"sha3-512" = 0x14, + @"sha3-384" = 0x15, + @"sha3-256" = 0x16, + @"sha3-224" = 0x17, + @"sha2-384" = 0x20, + @"sha2-256-trunc254-padded" = 0x1012, + @"sha2-224" = 0x1013, + @"sha2-512-224" = 0x1014, + @"sha2-512-256" = 0x1015, + @"blake2b-256" = 0xb220, + _, +}; + +pub const multihash_function: MultihashFunction = switch (Hash) { + std.crypto.hash.sha2.Sha256 => .@"sha2-256", + else => @compileError("unreachable"), +}; +comptime { + // We avoid unnecessary uleb128 code in hexDigest by asserting here the + // values are small enough to be contained in the one-byte encoding. + assert(@intFromEnum(multihash_function) < 127); + assert(Hash.digest_length < 127); +} + +name: []const u8, +version: std.SemanticVersion, +dependencies: std.StringArrayHashMapUnmanaged(Dependency), +paths: std.StringArrayHashMapUnmanaged(void), + +errors: []ErrorMessage, +arena_state: std.heap.ArenaAllocator.State, + +pub const ParseOptions = struct { + allow_missing_paths_field: bool = false, +}; + +pub const Error = Allocator.Error; + +pub fn parse(gpa: Allocator, ast: std.zig.Ast, options: ParseOptions) Error!Manifest { + const node_tags = ast.nodes.items(.tag); + const node_datas = ast.nodes.items(.data); + assert(node_tags[0] == .root); + const main_node_index = node_datas[0].lhs; + + var arena_instance = std.heap.ArenaAllocator.init(gpa); + errdefer arena_instance.deinit(); + + var p: Parse = .{ + .gpa = gpa, + .ast = ast, + .arena = arena_instance.allocator(), + .errors = .{}, + + .name = undefined, + .version = undefined, + .dependencies = .{}, + .paths = .{}, + .allow_missing_paths_field = options.allow_missing_paths_field, + .buf = .{}, + }; + defer p.buf.deinit(gpa); + defer p.errors.deinit(gpa); + defer p.dependencies.deinit(gpa); + defer p.paths.deinit(gpa); + + p.parseRoot(main_node_index) catch |err| switch (err) { + error.ParseFailure => assert(p.errors.items.len > 0), + else => |e| return e, + }; + + return .{ + .name = p.name, + .version = p.version, + .dependencies = try p.dependencies.clone(p.arena), + .paths = try p.paths.clone(p.arena), + .errors = try p.arena.dupe(ErrorMessage, p.errors.items), + .arena_state = arena_instance.state, + }; +} + +pub fn deinit(man: *Manifest, gpa: Allocator) void { + man.arena_state.promote(gpa).deinit(); + man.* = undefined; +} + +const hex_charset = "0123456789abcdef"; + +pub fn hex64(x: u64) [16]u8 { + var result: [16]u8 = undefined; + var i: usize = 0; + while (i < 8) : (i += 1) { + const byte = @as(u8, @truncate(x >> @as(u6, @intCast(8 * i)))); + result[i * 2 + 0] = hex_charset[byte >> 4]; + result[i * 2 + 1] = hex_charset[byte & 15]; + } + return result; +} + +test hex64 { + const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]"; + try std.testing.expectEqualStrings("[00efcdab78563412]", s); +} + +pub fn hexDigest(digest: Digest) MultiHashHexDigest { + var result: MultiHashHexDigest = undefined; + + result[0] = hex_charset[@intFromEnum(multihash_function) >> 4]; + result[1] = hex_charset[@intFromEnum(multihash_function) & 15]; + + result[2] = hex_charset[Hash.digest_length >> 4]; + result[3] = hex_charset[Hash.digest_length & 15]; + + for (digest, 0..) |byte, i| { + result[4 + i * 2] = hex_charset[byte >> 4]; + result[5 + i * 2] = hex_charset[byte & 15]; + } + return result; +} + +const Parse = struct { + gpa: Allocator, + ast: std.zig.Ast, + arena: Allocator, + buf: std.ArrayListUnmanaged(u8), + errors: std.ArrayListUnmanaged(ErrorMessage), + + name: []const u8, + version: std.SemanticVersion, + dependencies: std.StringArrayHashMapUnmanaged(Dependency), + paths: std.StringArrayHashMapUnmanaged(void), + allow_missing_paths_field: bool, + + const InnerError = error{ ParseFailure, OutOfMemory }; + + fn parseRoot(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + const main_token = main_tokens[node]; + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + return fail(p, main_token, "expected top level expression to be a struct", .{}); + }; + + var have_name = false; + var have_version = false; + var have_included_paths = false; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "dependencies")) { + try parseDependencies(p, field_init); + } else if (mem.eql(u8, field_name, "paths")) { + have_included_paths = true; + try parseIncludedPaths(p, field_init); + } else if (mem.eql(u8, field_name, "name")) { + p.name = try parseString(p, field_init); + have_name = true; + } else if (mem.eql(u8, field_name, "version")) { + const version_text = try parseString(p, field_init); + p.version = std.SemanticVersion.parse(version_text) catch |err| v: { + try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)}); + break :v undefined; + }; + have_version = true; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!have_name) { + try appendError(p, main_token, "missing top-level 'name' field", .{}); + } + + if (!have_version) { + try appendError(p, main_token, "missing top-level 'version' field", .{}); + } + + if (!have_included_paths) { + if (p.allow_missing_paths_field) { + try p.paths.put(p.gpa, "", {}); + } else { + try appendError(p, main_token, "missing top-level 'paths' field", .{}); + } + } + } + + fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = main_tokens[node]; + return fail(p, tok, "expected dependencies expression to be a struct", .{}); + }; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const dep_name = try identifierTokenString(p, name_token); + const dep = try parseDependency(p, field_init); + try p.dependencies.put(p.gpa, dep_name, dep); + } + } + + fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = main_tokens[node]; + return fail(p, tok, "expected dependency expression to be a struct", .{}); + }; + + var dep: Dependency = .{ + .location = undefined, + .location_tok = 0, + .hash = null, + .hash_tok = 0, + }; + var has_location = false; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "url")) { + if (has_location) { + return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{}); + } + dep.location = .{ + .url = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }, + }; + has_location = true; + dep.location_tok = main_tokens[field_init]; + } else if (mem.eql(u8, field_name, "path")) { + if (has_location) { + return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{}); + } + dep.location = .{ + .path = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }, + }; + has_location = true; + dep.location_tok = main_tokens[field_init]; + } else if (mem.eql(u8, field_name, "hash")) { + dep.hash = parseHash(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + dep.hash_tok = main_tokens[field_init]; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!has_location) { + try appendError(p, main_tokens[node], "dependency requires location field, one of 'url' or 'path'.", .{}); + } + + return dep; + } + + fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + + var buf: [2]Ast.Node.Index = undefined; + const array_init = ast.fullArrayInit(&buf, node) orelse { + const tok = main_tokens[node]; + return fail(p, tok, "expected paths expression to be a struct", .{}); + }; + + for (array_init.ast.elements) |elem_node| { + const path_string = try parseString(p, elem_node); + // This is normalized so that it can be used in string comparisons + // against file system paths. + const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); + try p.paths.put(p.gpa, normalized, {}); + } + } + + fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const node_tags = ast.nodes.items(.tag); + const main_tokens = ast.nodes.items(.main_token); + if (node_tags[node] != .string_literal) { + return fail(p, main_tokens[node], "expected string literal", .{}); + } + const str_lit_token = main_tokens[node]; + const token_bytes = ast.tokenSlice(str_lit_token); + p.buf.clearRetainingCapacity(); + try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + const tok = main_tokens[node]; + const h = try parseString(p, node); + + if (h.len >= 2) { + const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| { + return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{ + @errorName(err), + }); + }; + if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) { + return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{}); + } + } + + if (h.len != multihash_hex_digest_len) { + return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{ + multihash_hex_digest_len, h.len, + }); + } + + return h; + } + + /// TODO: try to DRY this with AstGen.identifierTokenString + fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { + const ast = p.ast; + const token_tags = ast.tokens.items(.tag); + assert(token_tags[token] == .identifier); + const ident_name = ast.tokenSlice(token); + if (!mem.startsWith(u8, ident_name, "@")) { + return ident_name; + } + p.buf.clearRetainingCapacity(); + try parseStrLit(p, token, &p.buf, ident_name, 1); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + /// TODO: try to DRY this with AstGen.parseStrLit + fn parseStrLit( + p: *Parse, + token: Ast.TokenIndex, + buf: *std.ArrayListUnmanaged(u8), + bytes: []const u8, + offset: u32, + ) InnerError!void { + const raw_string = bytes[offset..]; + var buf_managed = buf.toManaged(p.gpa); + const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string); + buf.* = buf_managed.moveToUnmanaged(); + switch (try result) { + .success => {}, + .failure => |err| try p.appendStrLitError(err, token, bytes, offset), + } + } + + /// TODO: try to DRY this with AstGen.failWithStrLitError + fn appendStrLitError( + p: *Parse, + err: std.zig.string_literal.Error, + token: Ast.TokenIndex, + bytes: []const u8, + offset: u32, + ) Allocator.Error!void { + const raw_string = bytes[offset..]; + switch (err) { + .invalid_escape_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "invalid escape character: '{c}'", + .{raw_string[bad_index]}, + ); + }, + .expected_hex_digit => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected hex digit, found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .empty_unicode_escape_sequence => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "empty unicode escape sequence", + .{}, + ); + }, + .expected_hex_digit_or_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected hex digit or '}}', found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .invalid_unicode_codepoint => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "unicode escape does not correspond to a valid codepoint", + .{}, + ); + }, + .expected_lbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected '{{', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected '}}', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_single_quote => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected single quote ('), found '{c}", + .{raw_string[bad_index]}, + ); + }, + .invalid_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "invalid byte in string or character literal: '{c}'", + .{raw_string[bad_index]}, + ); + }, + } + } + + fn fail( + p: *Parse, + tok: Ast.TokenIndex, + comptime fmt: []const u8, + args: anytype, + ) InnerError { + try appendError(p, tok, fmt, args); + return error.ParseFailure; + } + + fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { + return appendErrorOff(p, tok, 0, fmt, args); + } + + fn appendErrorOff( + p: *Parse, + tok: Ast.TokenIndex, + byte_offset: u32, + comptime fmt: []const u8, + args: anytype, + ) Allocator.Error!void { + try p.errors.append(p.gpa, .{ + .msg = try std.fmt.allocPrint(p.arena, fmt, args), + .tok = tok, + .off = byte_offset, + }); + } +}; + +const Manifest = @This(); +const std = @import("std"); +const mem = std.mem; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const Ast = std.zig.Ast; +const testing = std.testing; + +test "basic" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = "foo", + \\ .version = "3.2.1", + \\ .dependencies = .{ + \\ .bar = .{ + \\ .url = "https://example.com/baz.tar.gz", + \\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", + \\ }, + \\ }, + \\} + ; + + var ast = try std.zig.Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var manifest = try Manifest.parse(gpa, ast); + defer manifest.deinit(gpa); + + try testing.expectEqualStrings("foo", manifest.name); + + try testing.expectEqual(@as(std.SemanticVersion, .{ + .major = 3, + .minor = 2, + .patch = 1, + }), manifest.version); + + try testing.expect(manifest.dependencies.count() == 1); + try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); + try testing.expectEqualStrings( + "https://example.com/baz.tar.gz", + manifest.dependencies.values()[0].url, + ); + try testing.expectEqualStrings( + "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", + manifest.dependencies.values()[0].hash orelse return error.TestFailed, + ); +} diff --git a/src/Package/Module.zig b/src/Package/Module.zig new file mode 100644 index 0000000000000000000000000000000000000000..7e6b518892b4f166eb228cb949d025d5a6286433 --- /dev/null +++ b/src/Package/Module.zig @@ -0,0 +1,34 @@ +//! Corresponds to something that Zig source code can `@import`. +//! Not to be confused with src/Module.zig which should be renamed +//! to something else. https://github.com/ziglang/zig/issues/14307 + +/// Only files inside this directory can be imported. +root: Package.Path, +/// Relative to `root`. May contain path separators. +root_src_path: []const u8, +/// Name used in compile errors. Looks like "root.foo.bar". +fully_qualified_name: []const u8, +/// The dependency table of this module. Shared dependencies such as 'std', +/// 'builtin', and 'root' are not specified in every dependency table, but +/// instead only in the table of `main_mod`. `Module.importFile` is +/// responsible for detecting these names and using the correct package. +deps: Deps = .{}, + +pub const Deps = std.StringHashMapUnmanaged(*Module); + +pub const Tree = struct { + /// Each `Package` exposes a `Module` with build.zig as its root source file. + build_module_table: std.AutoArrayHashMapUnmanaged(MultiHashHexDigest, *Module), +}; + +pub fn create(allocator: Allocator, m: Module) Allocator.Error!*Module { + const new = try allocator.create(Module); + new.* = m; + return new; +} + +const Module = @This(); +const Package = @import("../Package.zig"); +const std = @import("std"); +const Allocator = std.mem.Allocator; +const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest; diff --git a/src/Package/hash.zig b/src/Package/hash.zig deleted file mode 100644 index b14ec70244a33f379602f7325ab474c502ab7238..0000000000000000000000000000000000000000 --- a/src/Package/hash.zig +++ /dev/null @@ -1,153 +0,0 @@ -const builtin = @import("builtin"); -const std = @import("std"); -const fs = std.fs; -const ThreadPool = std.Thread.Pool; -const WaitGroup = std.Thread.WaitGroup; -const Allocator = std.mem.Allocator; - -const Hash = @import("../Manifest.zig").Hash; - -pub fn compute(thread_pool: *ThreadPool, pkg_dir: fs.IterableDir) ![Hash.digest_length]u8 { - const gpa = thread_pool.allocator; - - // We'll use an arena allocator for the path name strings since they all - // need to be in memory for sorting. - var arena_instance = std.heap.ArenaAllocator.init(gpa); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - // TODO: delete files not included in the package prior to computing the package hash. - // for example, if the ini file has directives to include/not include certain files, - // apply those rules directly to the filesystem right here. This ensures that files - // not protected by the hash are not present on the file system. - - // Collect all files, recursively, then sort. - var all_files = std.ArrayList(*HashedFile).init(gpa); - defer all_files.deinit(); - - var walker = try pkg_dir.walk(gpa); - defer walker.deinit(); - - { - // The final hash will be a hash of each file hashed independently. This - // allows hashing in parallel. - var wait_group: WaitGroup = .{}; - defer wait_group.wait(); - - while (try walker.next()) |entry| { - const kind: HashedFile.Kind = switch (entry.kind) { - .directory => continue, - .file => .file, - .sym_link => .sym_link, - else => return error.IllegalFileTypeInPackage, - }; - const hashed_file = try arena.create(HashedFile); - const fs_path = try arena.dupe(u8, entry.path); - hashed_file.* = .{ - .fs_path = fs_path, - .normalized_path = try normalizePath(arena, fs_path), - .kind = kind, - .hash = undefined, // to be populated by the worker - .failure = undefined, // to be populated by the worker - }; - wait_group.start(); - try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); - - try all_files.append(hashed_file); - } - } - - std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); - - var hasher = Hash.init(.{}); - var any_failures = false; - for (all_files.items) |hashed_file| { - hashed_file.failure catch |err| { - any_failures = true; - std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) }); - }; - hasher.update(&hashed_file.hash); - } - if (any_failures) return error.PackageHashUnavailable; - return hasher.finalResult(); -} - -const HashedFile = struct { - fs_path: []const u8, - normalized_path: []const u8, - hash: [Hash.digest_length]u8, - failure: Error!void, - kind: Kind, - - const Error = - fs.File.OpenError || - fs.File.ReadError || - fs.File.StatError || - fs.Dir.ReadLinkError; - - const Kind = enum { file, sym_link }; - - fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { - _ = context; - return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); - } -}; - -/// Make a file system path identical independently of operating system path inconsistencies. -/// This converts backslashes into forward slashes. -fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 { - const canonical_sep = '/'; - - if (fs.path.sep == canonical_sep) - return fs_path; - - const normalized = try arena.dupe(u8, fs_path); - for (normalized) |*byte| { - switch (byte.*) { - fs.path.sep => byte.* = canonical_sep, - else => continue, - } - } - return normalized; -} - -fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { - defer wg.finish(); - hashed_file.failure = hashFileFallible(dir, hashed_file); -} - -fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { - var buf: [8000]u8 = undefined; - var hasher = Hash.init(.{}); - hasher.update(hashed_file.normalized_path); - switch (hashed_file.kind) { - .file => { - var file = try dir.openFile(hashed_file.fs_path, .{}); - defer file.close(); - hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) }); - while (true) { - const bytes_read = try file.read(&buf); - if (bytes_read == 0) break; - hasher.update(buf[0..bytes_read]); - } - }, - .sym_link => { - const link_name = try dir.readLink(hashed_file.fs_path, &buf); - hasher.update(link_name); - }, - } - hasher.final(&hashed_file.hash); -} - -fn isExecutable(file: fs.File) !bool { - if (builtin.os.tag == .windows) { - // TODO check the ACL on Windows. - // Until this is implemented, this could be a false negative on - // Windows, which is why we do not yet set executable_bit_only above - // when unpacking the tarball. - return false; - } else { - const stat = try file.stat(); - return (stat.mode & std.os.S.IXUSR) != 0; - } -} diff --git a/src/Sema.zig b/src/Sema.zig index c1187c0591bd3ad49dbb2d8c2a491a7a2f0d6a1b..a7073a5f36c3c6744c2260b8e909d1afc7c66977 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -5732,6 +5732,9 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr const tracy = trace(@src()); defer tracy.end(); + const mod = sema.mod; + const comp = mod.comp; + const gpa = sema.gpa; const pl_node = sema.code.instructions.items(.data)[inst].pl_node; const src = pl_node.src(); const extra = sema.code.extraData(Zir.Inst.Block, pl_node.payload_index); @@ -5741,7 +5744,7 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr if (!@import("build_options").have_llvm) return sema.fail(parent_block, src, "C import unavailable; Zig compiler built without LLVM extensions", .{}); - var c_import_buf = std.ArrayList(u8).init(sema.gpa); + var c_import_buf = std.ArrayList(u8).init(gpa); defer c_import_buf.deinit(); var comptime_reason: Block.ComptimeReason = .{ .c_import = .{ @@ -5763,25 +5766,24 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr .runtime_loop = parent_block.runtime_loop, .runtime_index = parent_block.runtime_index, }; - defer child_block.instructions.deinit(sema.gpa); + defer child_block.instructions.deinit(gpa); // Ignore the result, all the relevant operations have written to c_import_buf already. _ = try sema.analyzeBodyBreak(&child_block, body); - const mod = sema.mod; - var c_import_res = mod.comp.cImport(c_import_buf.items) catch |err| + var c_import_res = comp.cImport(c_import_buf.items) catch |err| return sema.fail(&child_block, src, "C import failed: {s}", .{@errorName(err)}); - defer c_import_res.deinit(mod.comp.gpa); + defer c_import_res.deinit(gpa); if (c_import_res.errors.errorMessageCount() != 0) { const msg = msg: { const msg = try sema.errMsg(&child_block, src, "C import failed", .{}); - errdefer msg.destroy(sema.gpa); + errdefer msg.destroy(gpa); - if (!mod.comp.bin_file.options.link_libc) + if (!comp.bin_file.options.link_libc) try sema.errNote(&child_block, src, msg, "libc headers not available; compilation does not link against libc", .{}); - const gop = try mod.cimport_errors.getOrPut(sema.gpa, sema.owner_decl_index); + const gop = try mod.cimport_errors.getOrPut(gpa, sema.owner_decl_index); if (!gop.found_existing) { gop.value_ptr.* = c_import_res.errors; c_import_res.errors = std.zig.ErrorBundle.empty; @@ -5790,16 +5792,16 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr }; return sema.failWithOwnedErrorMsg(&child_block, msg); } - const c_import_pkg = Package.create( - sema.gpa, - null, - c_import_res.out_zig_path, - ) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => unreachable, // we pass null for root_src_dir_path - }; + const c_import_mod = try Package.Module.create(comp.arena.allocator(), .{ + .root = .{ + .root_dir = Compilation.Directory.cwd(), + .sub_path = std.fs.path.dirname(c_import_res.out_zig_path) orelse "", + }, + .root_src_path = std.fs.path.basename(c_import_res.out_zig_path), + .fully_qualified_name = c_import_res.out_zig_path, + }); - const result = mod.importPkg(c_import_pkg) catch |err| + const result = mod.importPkg(c_import_mod) catch |err| return sema.fail(&child_block, src, "C import failed: {s}", .{@errorName(err)}); mod.astGenFile(result.file) catch |err| @@ -13071,13 +13073,13 @@ fn zirImport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air. const operand = inst_data.get(sema.code); const result = mod.importFile(block.getFileScope(mod), operand) catch |err| switch (err) { - error.ImportOutsidePkgPath => { - return sema.fail(block, operand_src, "import of file outside package path: '{s}'", .{operand}); + error.ImportOutsideModulePath => { + return sema.fail(block, operand_src, "import of file outside module path: '{s}'", .{operand}); }, - error.PackageNotFound => { - const name = try block.getFileScope(mod).pkg.getName(sema.gpa, mod.*); - defer sema.gpa.free(name); - return sema.fail(block, operand_src, "no package named '{s}' available within package '{s}'", .{ operand, name }); + error.ModuleNotFound => { + return sema.fail(block, operand_src, "no module named '{s}' available within module {s}", .{ + operand, block.getFileScope(mod).mod.fully_qualified_name, + }); }, else => { // TODO: these errors are file system errors; make sure an update() will @@ -13106,7 +13108,7 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A } const embed_file = mod.embedFile(block.getFileScope(mod), name) catch |err| switch (err) { - error.ImportOutsidePkgPath => { + error.ImportOutsideModulePath => { return sema.fail(block, operand_src, "embed of file outside package path: '{s}'", .{name}); }, else => { @@ -36415,8 +36417,8 @@ fn getBuiltinDecl(sema: *Sema, block: *Block, name: []const u8) CompileError!Mod const mod = sema.mod; const ip = &mod.intern_pool; - const std_pkg = mod.main_pkg.table.get("std").?; - const std_file = (mod.importPkg(std_pkg) catch unreachable).file; + const std_mod = mod.main_mod.deps.get("std").?; + const std_file = (mod.importPkg(std_mod) catch unreachable).file; const opt_builtin_inst = (try sema.namespaceLookupRef( block, src, diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 4355ac11914308a070c402ae5b8648afa38766d7..66b5bfbe042899daad65f062f2018d3cfc910177 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -892,21 +892,24 @@ pub const Object = struct { build_options.semver.patch, }); - // We fully resolve all paths at this point to avoid lack of source line info in stack - // traces or lack of debugging information which, if relative paths were used, would - // be very location dependent. + // We fully resolve all paths at this point to avoid lack of + // source line info in stack traces or lack of debugging + // information which, if relative paths were used, would be + // very location dependent. // TODO: the only concern I have with this is WASI as either host or target, should // we leave the paths as relative then? var buf: [std.fs.MAX_PATH_BYTES]u8 = undefined; - const compile_unit_dir = blk: { - const path = d: { - const mod = options.module orelse break :d "."; - break :d mod.root_pkg.root_src_directory.path orelse "."; - }; - if (std.fs.path.isAbsolute(path)) break :blk path; - break :blk std.os.realpath(path, &buf) catch path; // If realpath fails, fallback to whatever path was + const compile_unit_dir_z = blk: { + if (options.module) |mod| { + const d = try mod.root_mod.root.joinStringZ(builder.gpa, ""); + if (std.fs.path.isAbsolute(d)) break :blk d; + const abs = std.fs.realpath(d, &buf) catch break :blk d; + builder.gpa.free(d); + break :blk try builder.gpa.dupeZ(u8, abs); + } + const cwd = try std.process.getCwd(&buf); + break :blk try builder.gpa.dupeZ(u8, cwd); }; - const compile_unit_dir_z = try builder.gpa.dupeZ(u8, compile_unit_dir); defer builder.gpa.free(compile_unit_dir_z); builder.llvm.di_compile_unit = builder.llvm.di_builder.?.createCompileUnit( @@ -1833,14 +1836,11 @@ pub const Object = struct { } const dir_path_z = d: { var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined; - const dir_path = file.pkg.root_src_directory.path orelse "."; - const resolved_dir_path = if (std.fs.path.isAbsolute(dir_path)) - dir_path - else - std.os.realpath(dir_path, &buffer) catch dir_path; // If realpath fails, fallback to whatever dir_path was - break :d try std.fs.path.joinZ(gpa, &.{ - resolved_dir_path, std.fs.path.dirname(file.sub_file_path) orelse "", - }); + const sub_path = std.fs.path.dirname(file.sub_file_path) orelse ""; + const dir_path = try file.mod.root.joinStringZ(gpa, sub_path); + if (std.fs.path.isAbsolute(dir_path)) break :d dir_path; + const abs = std.fs.realpath(dir_path, &buffer) catch break :d dir_path; + break :d try std.fs.path.joinZ(gpa, &.{ abs, sub_path }); }; defer gpa.free(dir_path_z); const sub_file_path_z = try gpa.dupeZ(u8, std.fs.path.basename(file.sub_file_path)); @@ -2828,8 +2828,8 @@ pub const Object = struct { fn getStackTraceType(o: *Object) Allocator.Error!Type { const mod = o.module; - const std_pkg = mod.main_pkg.table.get("std").?; - const std_file = (mod.importPkg(std_pkg) catch unreachable).file; + const std_mod = mod.main_mod.deps.get("std").?; + const std_file = (mod.importPkg(std_mod) catch unreachable).file; const builtin_str = try mod.intern_pool.getOrPutString(mod.gpa, "builtin"); const std_namespace = mod.namespacePtr(mod.declPtr(std_file.root_decl.unwrap().?).src_namespace); diff --git a/src/crash_report.zig b/src/crash_report.zig index d4e4b46a5350b3d7b794067da5ab90fc7a61c465..2b33bd7fa51c2fa35de4c19c5716479e2c2fd354 100644 --- a/src/crash_report.zig +++ b/src/crash_report.zig @@ -139,18 +139,22 @@ fn dumpStatusReport() !void { var crash_heap: [16 * 4096]u8 = undefined; -fn writeFilePath(file: *Module.File, stream: anytype) !void { - if (file.pkg.root_src_directory.path) |path| { - try stream.writeAll(path); - try stream.writeAll(std.fs.path.sep_str); +fn writeFilePath(file: *Module.File, writer: anytype) !void { + if (file.mod.root.root_dir.path) |path| { + try writer.writeAll(path); + try writer.writeAll(std.fs.path.sep_str); } - try stream.writeAll(file.sub_file_path); + if (file.mod.root.sub_path.len > 0) { + try writer.writeAll(file.mod.root.sub_path); + try writer.writeAll(std.fs.path.sep_str); + } + try writer.writeAll(file.sub_file_path); } -fn writeFullyQualifiedDeclWithFile(mod: *Module, decl: *Decl, stream: anytype) !void { - try writeFilePath(decl.getFileScope(mod), stream); - try stream.writeAll(": "); - try decl.renderFullyQualifiedDebugName(mod, stream); +fn writeFullyQualifiedDeclWithFile(mod: *Module, decl: *Decl, writer: anytype) !void { + try writeFilePath(decl.getFileScope(mod), writer); + try writer.writeAll(": "); + try decl.renderFullyQualifiedDebugName(mod, writer); } pub fn compilerPanic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, maybe_ret_addr: ?usize) noreturn { diff --git a/src/git.zig b/src/git.zig deleted file mode 100644 index fd35c5c33e8aa194aa3a8fe160d693ca3948b065..0000000000000000000000000000000000000000 --- a/src/git.zig +++ /dev/null @@ -1,1468 +0,0 @@ -//! Git support for package fetching. -//! -//! This is not intended to support all features of Git: it is limited to the -//! basic functionality needed to clone a repository for the purpose of fetching -//! a package. - -const std = @import("std"); -const mem = std.mem; -const testing = std.testing; -const Allocator = mem.Allocator; -const Sha1 = std.crypto.hash.Sha1; -const assert = std.debug.assert; - -const ProgressReader = @import("Package.zig").ProgressReader; - -pub const oid_length = Sha1.digest_length; -pub const fmt_oid_length = 2 * oid_length; -/// The ID of a Git object (an SHA-1 hash). -pub const Oid = [oid_length]u8; - -pub fn parseOid(s: []const u8) !Oid { - if (s.len != fmt_oid_length) return error.InvalidOid; - var oid: Oid = undefined; - for (&oid, 0..) |*b, i| { - b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; - } - return oid; -} - -test parseOid { - try testing.expectEqualSlices( - u8, - &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, - &try parseOid("ce919ccf45951856a762ffdb8ef850301cd8c588"), - ); - try testing.expectError(error.InvalidOid, parseOid("ce919ccf")); - try testing.expectError(error.InvalidOid, parseOid("master")); - try testing.expectError(error.InvalidOid, parseOid("HEAD")); -} - -pub const Diagnostics = struct { - allocator: Allocator, - errors: std.ArrayListUnmanaged(Error) = .{}, - - pub const Error = union(enum) { - unable_to_create_sym_link: struct { - code: anyerror, - file_name: []const u8, - link_name: []const u8, - }, - }; - - pub fn deinit(d: *Diagnostics) void { - for (d.errors.items) |item| { - switch (item) { - .unable_to_create_sym_link => |info| { - d.allocator.free(info.file_name); - d.allocator.free(info.link_name); - }, - } - } - d.errors.deinit(d.allocator); - d.* = undefined; - } -}; - -pub const Repository = struct { - odb: Odb, - - pub fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Repository { - return .{ .odb = try Odb.init(allocator, pack_file, index_file) }; - } - - pub fn deinit(repository: *Repository) void { - repository.odb.deinit(); - repository.* = undefined; - } - - /// Checks out the repository at `commit_oid` to `worktree`. - pub fn checkout( - repository: *Repository, - worktree: std.fs.Dir, - commit_oid: Oid, - diagnostics: *Diagnostics, - ) !void { - try repository.odb.seekOid(commit_oid); - const tree_oid = tree_oid: { - var commit_object = try repository.odb.readObject(); - if (commit_object.type != .commit) return error.NotACommit; - break :tree_oid try getCommitTree(commit_object.data); - }; - try repository.checkoutTree(worktree, tree_oid, "", diagnostics); - } - - /// Checks out the tree at `tree_oid` to `worktree`. - fn checkoutTree( - repository: *Repository, - dir: std.fs.Dir, - tree_oid: Oid, - current_path: []const u8, - diagnostics: *Diagnostics, - ) !void { - try repository.odb.seekOid(tree_oid); - const tree_object = try repository.odb.readObject(); - if (tree_object.type != .tree) return error.NotATree; - // The tree object may be evicted from the object cache while we're - // iterating over it, so we can make a defensive copy here to make sure - // it remains valid until we're done with it - const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); - defer repository.odb.allocator.free(tree_data); - - var tree_iter: TreeIterator = .{ .data = tree_data }; - while (try tree_iter.next()) |entry| { - switch (entry.type) { - .directory => { - try dir.makeDir(entry.name); - var subdir = try dir.openDir(entry.name, .{}); - defer subdir.close(); - const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name }); - defer repository.odb.allocator.free(sub_path); - try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics); - }, - .file => { - var file = try dir.createFile(entry.name, .{}); - defer file.close(); - try repository.odb.seekOid(entry.oid); - var file_object = try repository.odb.readObject(); - if (file_object.type != .blob) return error.InvalidFile; - try file.writeAll(file_object.data); - try file.sync(); - }, - .symlink => { - try repository.odb.seekOid(entry.oid); - var symlink_object = try repository.odb.readObject(); - if (symlink_object.type != .blob) return error.InvalidFile; - const link_name = symlink_object.data; - dir.symLink(link_name, entry.name, .{}) catch |e| { - const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); - errdefer diagnostics.allocator.free(file_name); - const link_name_dup = try diagnostics.allocator.dupe(u8, link_name); - errdefer diagnostics.allocator.free(link_name_dup); - try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{ - .code = e, - .file_name = file_name, - .link_name = link_name_dup, - } }); - }; - }, - .gitlink => { - // Consistent with git archive behavior, create the directory but - // do nothing else - try dir.makeDir(entry.name); - }, - } - } - } - - /// Returns the ID of the tree associated with the given commit (provided as - /// raw object data). - fn getCommitTree(commit_data: []const u8) !Oid { - if (!mem.startsWith(u8, commit_data, "tree ") or - commit_data.len < "tree ".len + fmt_oid_length + "\n".len or - commit_data["tree ".len + fmt_oid_length] != '\n') - { - return error.InvalidCommit; - } - return try parseOid(commit_data["tree ".len..][0..fmt_oid_length]); - } - - const TreeIterator = struct { - data: []const u8, - pos: usize = 0, - - const Entry = struct { - type: Type, - executable: bool, - name: [:0]const u8, - oid: Oid, - - const Type = enum(u4) { - directory = 0o4, - file = 0o10, - symlink = 0o12, - gitlink = 0o16, - }; - }; - - fn next(iterator: *TreeIterator) !?Entry { - if (iterator.pos == iterator.data.len) return null; - - const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree; - const mode: packed struct { - permission: u9, - unused: u3, - type: u4, - } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree); - const @"type" = std.meta.intToEnum(Entry.Type, mode.type) catch return error.InvalidTree; - const executable = switch (mode.permission) { - 0 => if (@"type" == .file) return error.InvalidTree else false, - 0o644 => if (@"type" != .file) return error.InvalidTree else false, - 0o755 => if (@"type" != .file) return error.InvalidTree else true, - else => return error.InvalidTree, - }; - iterator.pos = mode_end + 1; - - const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree; - const name = iterator.data[iterator.pos..name_end :0]; - iterator.pos = name_end + 1; - - if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; - const oid = iterator.data[iterator.pos..][0..oid_length].*; - iterator.pos += oid_length; - - return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; - } - }; -}; - -/// A Git object database backed by a packfile. A packfile index is also used -/// for efficient access to objects in the packfile. -/// -/// The format of the packfile and its associated index are documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -const Odb = struct { - pack_file: std.fs.File, - index_header: IndexHeader, - index_file: std.fs.File, - cache: ObjectCache = .{}, - allocator: Allocator, - - /// Initializes the database from open pack and index files. - fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Odb { - try pack_file.seekTo(0); - try index_file.seekTo(0); - const index_header = try IndexHeader.read(index_file.reader()); - return .{ - .pack_file = pack_file, - .index_header = index_header, - .index_file = index_file, - .allocator = allocator, - }; - } - - fn deinit(odb: *Odb) void { - odb.cache.deinit(odb.allocator); - odb.* = undefined; - } - - /// Reads the object at the current position in the database. - fn readObject(odb: *Odb) !Object { - var base_offset = try odb.pack_file.getPos(); - var base_header: EntryHeader = undefined; - var delta_offsets = std.ArrayListUnmanaged(u64){}; - defer delta_offsets.deinit(odb.allocator); - const base_object = while (true) { - if (odb.cache.get(base_offset)) |base_object| break base_object; - - base_header = try EntryHeader.read(odb.pack_file.reader()); - switch (base_header) { - .ofs_delta => |ofs_delta| { - try delta_offsets.append(odb.allocator, base_offset); - base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; - try odb.pack_file.seekTo(base_offset); - }, - .ref_delta => |ref_delta| { - try delta_offsets.append(odb.allocator, base_offset); - try odb.seekOid(ref_delta.base_object); - base_offset = try odb.pack_file.getPos(); - }, - else => { - const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength()); - errdefer odb.allocator.free(base_data); - const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; - try odb.cache.put(odb.allocator, base_offset, base_object); - break base_object; - }, - } - }; - - const base_data = try resolveDeltaChain( - odb.allocator, - odb.pack_file, - base_object, - delta_offsets.items, - &odb.cache, - ); - - return .{ .type = base_object.type, .data = base_data }; - } - - /// Seeks to the beginning of the object with the given ID. - fn seekOid(odb: *Odb, oid: Oid) !void { - const key = oid[0]; - var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; - var end_index = odb.index_header.fan_out_table[key]; - const found_index = while (start_index < end_index) { - const mid_index = start_index + (end_index - start_index) / 2; - try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); - const mid_oid = try odb.index_file.reader().readBytesNoEof(oid_length); - switch (mem.order(u8, &mid_oid, &oid)) { - .lt => start_index = mid_index + 1, - .gt => end_index = mid_index, - .eq => break mid_index, - } - } else return error.ObjectNotFound; - - const n_objects = odb.index_header.fan_out_table[255]; - const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); - try odb.index_file.seekTo(offset_values_start + found_index * 4); - const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readIntBig(u32)); - const pack_offset = pack_offset: { - if (l1_offset.big) { - const l2_offset_values_start = offset_values_start + n_objects * 4; - try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); - break :pack_offset try odb.index_file.reader().readIntBig(u64); - } else { - break :pack_offset l1_offset.value; - } - }; - - try odb.pack_file.seekTo(pack_offset); - } -}; - -const Object = struct { - type: Type, - data: []const u8, - - const Type = enum { - commit, - tree, - blob, - tag, - }; -}; - -/// A cache for object data. -/// -/// The purpose of this cache is to speed up resolution of deltas by caching the -/// results of resolving delta objects, while maintaining a maximum cache size -/// to avoid excessive memory usage. If the total size of the objects in the -/// cache exceeds the maximum, the cache will begin evicting the least recently -/// used objects: when resolving delta chains, the most recently used objects -/// will likely be more helpful as they will be further along in the chain -/// (skipping earlier reconstruction steps). -/// -/// Object data stored in the cache is managed by the cache. It should not be -/// freed by the caller at any point after inserting it into the cache. Any -/// objects remaining in the cache will be freed when the cache itself is freed. -const ObjectCache = struct { - objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .{}, - lru_nodes: LruList = .{}, - byte_size: usize = 0, - - const max_byte_size = 128 * 1024 * 1024; // 128MiB - /// A list of offsets stored in the cache, with the most recently used - /// entries at the end. - const LruList = std.DoublyLinkedList(u64); - const CacheEntry = struct { object: Object, lru_node: *LruList.Node }; - - fn deinit(cache: *ObjectCache, allocator: Allocator) void { - var object_iterator = cache.objects.iterator(); - while (object_iterator.next()) |object| { - allocator.free(object.value_ptr.object.data); - allocator.destroy(object.value_ptr.lru_node); - } - cache.objects.deinit(allocator); - cache.* = undefined; - } - - /// Gets an object from the cache, moving it to the most recently used - /// position if it is present. - fn get(cache: *ObjectCache, offset: u64) ?Object { - if (cache.objects.get(offset)) |entry| { - cache.lru_nodes.remove(entry.lru_node); - cache.lru_nodes.append(entry.lru_node); - return entry.object; - } else { - return null; - } - } - - /// Puts an object in the cache, possibly evicting older entries if the - /// cache exceeds its maximum size. Note that, although old objects may - /// be evicted, the object just added to the cache with this function - /// will not be evicted before the next call to `put` or `deinit` even if - /// it exceeds the maximum cache size. - fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void { - const lru_node = try allocator.create(LruList.Node); - errdefer allocator.destroy(lru_node); - lru_node.data = offset; - - const gop = try cache.objects.getOrPut(allocator, offset); - if (gop.found_existing) { - cache.byte_size -= gop.value_ptr.object.data.len; - cache.lru_nodes.remove(gop.value_ptr.lru_node); - allocator.destroy(gop.value_ptr.lru_node); - allocator.free(gop.value_ptr.object.data); - } - gop.value_ptr.* = .{ .object = object, .lru_node = lru_node }; - cache.byte_size += object.data.len; - cache.lru_nodes.append(lru_node); - - while (cache.byte_size > max_byte_size and cache.lru_nodes.len > 1) { - // The > 1 check is to make sure that we don't evict the most - // recently added node, even if it by itself happens to exceed the - // maximum size of the cache. - const evict_node = cache.lru_nodes.popFirst().?; - const evict_offset = evict_node.data; - allocator.destroy(evict_node); - const evict_object = cache.objects.get(evict_offset).?.object; - cache.byte_size -= evict_object.data.len; - allocator.free(evict_object.data); - _ = cache.objects.remove(evict_offset); - } - } -}; - -/// A single pkt-line in the Git protocol. -/// -/// The format of a pkt-line is documented in -/// [protocol-common](https://git-scm.com/docs/protocol-common). The special -/// meanings of the delimiter and response-end packets are documented in -/// [protocol-v2](https://git-scm.com/docs/protocol-v2). -const Packet = union(enum) { - flush, - delimiter, - response_end, - data: []const u8, - - const max_data_length = 65516; - - /// Reads a packet in pkt-line format. - fn read(reader: anytype, buf: *[max_data_length]u8) !Packet { - const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket; - switch (length) { - 0 => return .flush, - 1 => return .delimiter, - 2 => return .response_end, - 3 => return error.InvalidPacket, - else => if (length - 4 > max_data_length) return error.InvalidPacket, - } - const data = buf[0 .. length - 4]; - try reader.readNoEof(data); - return .{ .data = data }; - } - - /// Writes a packet in pkt-line format. - fn write(packet: Packet, writer: anytype) !void { - switch (packet) { - .flush => try writer.writeAll("0000"), - .delimiter => try writer.writeAll("0001"), - .response_end => try writer.writeAll("0002"), - .data => |data| { - assert(data.len <= max_data_length); - try writer.print("{x:0>4}", .{data.len + 4}); - try writer.writeAll(data); - }, - } - } -}; - -/// A client session for the Git protocol, currently limited to an HTTP(S) -/// transport. Only protocol version 2 is supported, as documented in -/// [protocol-v2](https://git-scm.com/docs/protocol-v2). -pub const Session = struct { - transport: *std.http.Client, - uri: std.Uri, - supports_agent: bool = false, - supports_shallow: bool = false, - - const agent = "zig/" ++ @import("builtin").zig_version_string; - const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent}); - - /// Discovers server capabilities. This should be called before using any - /// other client functionality, or the client will be forced to default to - /// the bare minimum server requirements, which may be considerably less - /// efficient (e.g. no shallow fetches). - /// - /// See the note on `getCapabilities` regarding `redirect_uri`. - pub fn discoverCapabilities( - session: *Session, - allocator: Allocator, - redirect_uri: *[]u8, - ) !void { - var capability_iterator = try session.getCapabilities(allocator, redirect_uri); - defer capability_iterator.deinit(); - while (try capability_iterator.next()) |capability| { - if (mem.eql(u8, capability.key, "agent")) { - session.supports_agent = true; - } else if (mem.eql(u8, capability.key, "fetch")) { - var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' '); - while (feature_iterator.next()) |feature| { - if (mem.eql(u8, feature, "shallow")) { - session.supports_shallow = true; - } - } - } - } - } - - /// Returns an iterator over capabilities supported by the server. - /// - /// If the server redirects the request, `error.Redirected` is returned and - /// `redirect_uri` is populated with the URI resulting from the redirects. - /// When this occurs, the value of `redirect_uri` must be freed with - /// `allocator` when the caller is done with it. - fn getCapabilities( - session: Session, - allocator: Allocator, - redirect_uri: *[]u8, - ) !CapabilityIterator { - var info_refs_uri = session.uri; - info_refs_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "info/refs" }); - defer allocator.free(info_refs_uri.path); - info_refs_uri.query = "service=git-upload-pack"; - info_refs_uri.fragment = null; - - var headers = std.http.Headers.init(allocator); - defer headers.deinit(); - try headers.append("Git-Protocol", "version=2"); - - var request = try session.transport.request(.GET, info_refs_uri, headers, .{ - .max_redirects = 3, - }); - errdefer request.deinit(); - try request.start(.{}); - try request.finish(); - - try request.wait(); - if (request.response.status != .ok) return error.ProtocolError; - if (request.redirects_left < 3) { - if (!mem.endsWith(u8, request.uri.path, "/info/refs")) return error.UnparseableRedirect; - var new_uri = request.uri; - new_uri.path = new_uri.path[0 .. new_uri.path.len - "/info/refs".len]; - new_uri.query = null; - redirect_uri.* = try std.fmt.allocPrint(allocator, "{+/}", .{new_uri}); - return error.Redirected; - } - - const reader = request.reader(); - var buf: [Packet.max_data_length]u8 = undefined; - var state: enum { response_start, response_content } = .response_start; - while (true) { - // Some Git servers (at least GitHub) include an additional - // '# service=git-upload-pack' informative response before sending - // the expected 'version 2' packet and capability information. - // This is not universal: SourceHut, for example, does not do this. - // Thus, we need to skip any such useless additional responses - // before we get the one we're actually looking for. The responses - // will be delimited by flush packets. - const packet = Packet.read(reader, &buf) catch |e| switch (e) { - error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found - else => |other| return other, - }; - switch (packet) { - .flush => state = .response_start, - .data => |data| switch (state) { - .response_start => if (mem.eql(u8, data, "version 2\n")) { - return .{ .request = request }; - } else { - state = .response_content; - }, - else => {}, - }, - else => return error.UnexpectedPacket, - } - } - } - - const CapabilityIterator = struct { - request: std.http.Client.Request, - buf: [Packet.max_data_length]u8 = undefined, - - const Capability = struct { - key: []const u8, - value: ?[]const u8 = null, - }; - - fn deinit(iterator: *CapabilityIterator) void { - iterator.request.deinit(); - iterator.* = undefined; - } - - fn next(iterator: *CapabilityIterator) !?Capability { - switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { - .flush => return null, - .data => |data| if (data.len > 0 and data[data.len - 1] == '\n') { - if (mem.indexOfScalar(u8, data, '=')) |separator_pos| { - return .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 .. data.len - 1] }; - } else { - return .{ .key = data[0 .. data.len - 1] }; - } - } else return error.UnexpectedPacket, - else => return error.UnexpectedPacket, - } - } - }; - - const ListRefsOptions = struct { - /// The ref prefixes (if any) to use to filter the refs available on the - /// server. Note that the client must still check the returned refs - /// against its desired filters itself: the server is not required to - /// respect these prefix filters and may return other refs as well. - ref_prefixes: []const []const u8 = &.{}, - /// Whether to include symref targets for returned symbolic refs. - include_symrefs: bool = false, - /// Whether to include the peeled object ID for returned tag refs. - include_peeled: bool = false, - }; - - /// Returns an iterator over refs known to the server. - pub fn listRefs(session: Session, allocator: Allocator, options: ListRefsOptions) !RefIterator { - var upload_pack_uri = session.uri; - upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" }); - defer allocator.free(upload_pack_uri.path); - upload_pack_uri.query = null; - upload_pack_uri.fragment = null; - - var headers = std.http.Headers.init(allocator); - defer headers.deinit(); - try headers.append("Content-Type", "application/x-git-upload-pack-request"); - try headers.append("Git-Protocol", "version=2"); - - var body = std.ArrayListUnmanaged(u8){}; - defer body.deinit(allocator); - const body_writer = body.writer(allocator); - try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer); - if (session.supports_agent) { - try Packet.write(.{ .data = agent_capability }, body_writer); - } - try Packet.write(.delimiter, body_writer); - for (options.ref_prefixes) |ref_prefix| { - const ref_prefix_packet = try std.fmt.allocPrint(allocator, "ref-prefix {s}\n", .{ref_prefix}); - defer allocator.free(ref_prefix_packet); - try Packet.write(.{ .data = ref_prefix_packet }, body_writer); - } - if (options.include_symrefs) { - try Packet.write(.{ .data = "symrefs\n" }, body_writer); - } - if (options.include_peeled) { - try Packet.write(.{ .data = "peel\n" }, body_writer); - } - try Packet.write(.flush, body_writer); - - var request = try session.transport.request(.POST, upload_pack_uri, headers, .{ - .handle_redirects = false, - }); - errdefer request.deinit(); - request.transfer_encoding = .{ .content_length = body.items.len }; - try request.start(.{}); - try request.writeAll(body.items); - try request.finish(); - - try request.wait(); - if (request.response.status != .ok) return error.ProtocolError; - - return .{ .request = request }; - } - - pub const RefIterator = struct { - request: std.http.Client.Request, - buf: [Packet.max_data_length]u8 = undefined, - - pub const Ref = struct { - oid: Oid, - name: []const u8, - symref_target: ?[]const u8, - peeled: ?Oid, - }; - - pub fn deinit(iterator: *RefIterator) void { - iterator.request.deinit(); - iterator.* = undefined; - } - - pub fn next(iterator: *RefIterator) !?Ref { - switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { - .flush => return null, - .data => |data| { - const oid_sep_pos = mem.indexOfScalar(u8, data, ' ') orelse return error.InvalidRefPacket; - const oid = parseOid(data[0..oid_sep_pos]) catch return error.InvalidRefPacket; - - const name_sep_pos = mem.indexOfAnyPos(u8, data, oid_sep_pos + 1, " \n") orelse return error.InvalidRefPacket; - const name = data[oid_sep_pos + 1 .. name_sep_pos]; - - var symref_target: ?[]const u8 = null; - var peeled: ?Oid = null; - var last_sep_pos = name_sep_pos; - while (data[last_sep_pos] == ' ') { - const next_sep_pos = mem.indexOfAnyPos(u8, data, last_sep_pos + 1, " \n") orelse return error.InvalidRefPacket; - const attribute = data[last_sep_pos + 1 .. next_sep_pos]; - if (mem.startsWith(u8, attribute, "symref-target:")) { - symref_target = attribute["symref-target:".len..]; - } else if (mem.startsWith(u8, attribute, "peeled:")) { - peeled = parseOid(attribute["peeled:".len..]) catch return error.InvalidRefPacket; - } - last_sep_pos = next_sep_pos; - } - - return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled }; - }, - else => return error.UnexpectedPacket, - } - } - }; - - /// Fetches the given refs from the server. A shallow fetch (depth 1) is - /// performed if the server supports it. - pub fn fetch(session: Session, allocator: Allocator, wants: []const []const u8) !FetchStream { - var upload_pack_uri = session.uri; - upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" }); - defer allocator.free(upload_pack_uri.path); - upload_pack_uri.query = null; - upload_pack_uri.fragment = null; - - var headers = std.http.Headers.init(allocator); - defer headers.deinit(); - try headers.append("Content-Type", "application/x-git-upload-pack-request"); - try headers.append("Git-Protocol", "version=2"); - - var body = std.ArrayListUnmanaged(u8){}; - defer body.deinit(allocator); - const body_writer = body.writer(allocator); - try Packet.write(.{ .data = "command=fetch\n" }, body_writer); - if (session.supports_agent) { - try Packet.write(.{ .data = agent_capability }, body_writer); - } - try Packet.write(.delimiter, body_writer); - // Our packfile parser supports the OFS_DELTA object type - try Packet.write(.{ .data = "ofs-delta\n" }, body_writer); - // We do not currently convey server progress information to the user - try Packet.write(.{ .data = "no-progress\n" }, body_writer); - if (session.supports_shallow) { - try Packet.write(.{ .data = "deepen 1\n" }, body_writer); - } - for (wants) |want| { - var buf: [Packet.max_data_length]u8 = undefined; - const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable; - try Packet.write(.{ .data = arg }, body_writer); - } - try Packet.write(.{ .data = "done\n" }, body_writer); - try Packet.write(.flush, body_writer); - - var request = try session.transport.request(.POST, upload_pack_uri, headers, .{ - .handle_redirects = false, - }); - errdefer request.deinit(); - request.transfer_encoding = .{ .content_length = body.items.len }; - try request.start(.{}); - try request.writeAll(body.items); - try request.finish(); - - try request.wait(); - if (request.response.status != .ok) return error.ProtocolError; - - const reader = request.reader(); - // We are not interested in any of the sections of the returned fetch - // data other than the packfile section, since we aren't doing anything - // complex like ref negotiation (this is a fresh clone). - var state: enum { section_start, section_content } = .section_start; - while (true) { - var buf: [Packet.max_data_length]u8 = undefined; - const packet = try Packet.read(reader, &buf); - switch (state) { - .section_start => switch (packet) { - .data => |data| if (mem.eql(u8, data, "packfile\n")) { - return .{ .request = request }; - } else { - state = .section_content; - }, - else => return error.UnexpectedPacket, - }, - .section_content => switch (packet) { - .delimiter => state = .section_start, - .data => {}, - else => return error.UnexpectedPacket, - }, - } - } - } - - pub const FetchStream = struct { - request: std.http.Client.Request, - buf: [Packet.max_data_length]u8 = undefined, - pos: usize = 0, - len: usize = 0, - - pub fn deinit(stream: *FetchStream) void { - stream.request.deinit(); - } - - pub const ReadError = std.http.Client.Request.ReadError || error{ - InvalidPacket, - ProtocolError, - UnexpectedPacket, - }; - pub const Reader = std.io.Reader(*FetchStream, ReadError, read); - - const StreamCode = enum(u8) { - pack_data = 1, - progress = 2, - fatal_error = 3, - _, - }; - - pub fn reader(stream: *FetchStream) Reader { - return .{ .context = stream }; - } - - pub fn read(stream: *FetchStream, buf: []u8) !usize { - if (stream.pos == stream.len) { - while (true) { - switch (try Packet.read(stream.request.reader(), &stream.buf)) { - .flush => return 0, - .data => |data| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) { - .pack_data => { - stream.pos = 1; - stream.len = data.len; - break; - }, - .fatal_error => return error.ProtocolError, - else => {}, - }, - else => return error.UnexpectedPacket, - } - } - } - - const size = @min(buf.len, stream.len - stream.pos); - @memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]); - stream.pos += size; - return size; - } - }; -}; - -const PackHeader = struct { - total_objects: u32, - - const signature = "PACK"; - const supported_version = 2; - - fn read(reader: anytype) !PackHeader { - const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader; - const version = reader.readIntBig(u32) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - if (version != supported_version) return error.UnsupportedVersion; - const total_objects = reader.readIntBig(u32) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - return .{ .total_objects = total_objects }; - } -}; - -const EntryHeader = union(Type) { - commit: Undeltified, - tree: Undeltified, - blob: Undeltified, - tag: Undeltified, - ofs_delta: OfsDelta, - ref_delta: RefDelta, - - const Type = enum(u3) { - commit = 1, - tree = 2, - blob = 3, - tag = 4, - ofs_delta = 6, - ref_delta = 7, - }; - - const Undeltified = struct { - uncompressed_length: u64, - }; - - const OfsDelta = struct { - offset: u64, - uncompressed_length: u64, - }; - - const RefDelta = struct { - base_object: Oid, - uncompressed_length: u64, - }; - - fn objectType(header: EntryHeader) Object.Type { - return switch (header) { - inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)), - else => unreachable, - }; - } - - fn uncompressedLength(header: EntryHeader) u64 { - return switch (header) { - inline else => |entry| entry.uncompressed_length, - }; - } - - fn read(reader: anytype) !EntryHeader { - const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; - const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { - error.EndOfStream => return error.InvalidFormat, - else => |other| return other, - }); - const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0; - var uncompressed_length: u64 = initial.len; - uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; - const @"type" = std.meta.intToEnum(EntryHeader.Type, initial.type) catch return error.InvalidFormat; - return switch (@"type") { - inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{ - .uncompressed_length = uncompressed_length, - }), - .ofs_delta => .{ .ofs_delta = .{ - .offset = try readOffsetVarInt(reader), - .uncompressed_length = uncompressed_length, - } }, - .ref_delta => .{ .ref_delta = .{ - .base_object = reader.readBytesNoEof(oid_length) catch |e| switch (e) { - error.EndOfStream => return error.InvalidFormat, - else => |other| return other, - }, - .uncompressed_length = uncompressed_length, - } }, - }; - } -}; - -fn readSizeVarInt(r: anytype) !u64 { - const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.readByte()); - var value: u64 = b.value; - var shift: u6 = 0; - while (b.has_next) { - b = @bitCast(try r.readByte()); - shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat; - value |= @as(u64, b.value) << shift; - } - return value; -} - -fn readOffsetVarInt(r: anytype) !u64 { - const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.readByte()); - var value: u64 = b.value; - while (b.has_next) { - b = @bitCast(try r.readByte()); - value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; - value |= b.value; - } - return value; -} - -const IndexHeader = struct { - fan_out_table: [256]u32, - - const signature = "\xFFtOc"; - const supported_version = 2; - const size = 4 + 4 + @sizeOf([256]u32); - - fn read(reader: anytype) !IndexHeader { - var header_bytes = try reader.readBytesNoEof(size); - if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader; - const version = mem.readIntBig(u32, header_bytes[4..8]); - if (version != supported_version) return error.UnsupportedVersion; - - var fan_out_table: [256]u32 = undefined; - var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]); - const fan_out_table_reader = fan_out_table_stream.reader(); - for (&fan_out_table) |*entry| { - entry.* = fan_out_table_reader.readIntBig(u32) catch unreachable; - } - return .{ .fan_out_table = fan_out_table }; - } -}; - -const IndexEntry = struct { - offset: u64, - crc32: u32, -}; - -/// Writes out a version 2 index for the given packfile, as documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) !void { - try pack.seekTo(0); - - var index_entries = std.AutoHashMapUnmanaged(Oid, IndexEntry){}; - defer index_entries.deinit(allocator); - var pending_deltas = std.ArrayListUnmanaged(IndexEntry){}; - defer pending_deltas.deinit(allocator); - - const pack_checksum = try indexPackFirstPass(allocator, pack, &index_entries, &pending_deltas); - - var cache: ObjectCache = .{}; - defer cache.deinit(allocator); - var remaining_deltas = pending_deltas.items.len; - while (remaining_deltas > 0) { - var i: usize = remaining_deltas; - while (i > 0) { - i -= 1; - const delta = pending_deltas.items[i]; - if (try indexPackHashDelta(allocator, pack, delta, index_entries, &cache)) |oid| { - try index_entries.put(allocator, oid, delta); - _ = pending_deltas.swapRemove(i); - } - } - if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack; - remaining_deltas = pending_deltas.items.len; - } - - var oids = std.ArrayListUnmanaged(Oid){}; - defer oids.deinit(allocator); - try oids.ensureTotalCapacityPrecise(allocator, index_entries.count()); - var index_entries_iter = index_entries.iterator(); - while (index_entries_iter.next()) |entry| { - oids.appendAssumeCapacity(entry.key_ptr.*); - } - mem.sortUnstable(Oid, oids.items, {}, struct { - fn lessThan(_: void, o1: Oid, o2: Oid) bool { - return mem.lessThan(u8, &o1, &o2); - } - }.lessThan); - - var fan_out_table: [256]u32 = undefined; - var count: u32 = 0; - var fan_out_index: u8 = 0; - for (oids.items) |oid| { - if (oid[0] > fan_out_index) { - @memset(fan_out_table[fan_out_index..oid[0]], count); - fan_out_index = oid[0]; - } - count += 1; - } - @memset(fan_out_table[fan_out_index..], count); - - var index_hashed_writer = hashedWriter(index_writer, Sha1.init(.{})); - const writer = index_hashed_writer.writer(); - try writer.writeAll(IndexHeader.signature); - try writer.writeIntBig(u32, IndexHeader.supported_version); - for (fan_out_table) |fan_out_entry| { - try writer.writeIntBig(u32, fan_out_entry); - } - - for (oids.items) |oid| { - try writer.writeAll(&oid); - } - - for (oids.items) |oid| { - try writer.writeIntBig(u32, index_entries.get(oid).?.crc32); - } - - var big_offsets = std.ArrayListUnmanaged(u64){}; - defer big_offsets.deinit(allocator); - for (oids.items) |oid| { - const offset = index_entries.get(oid).?.offset; - if (offset <= std.math.maxInt(u31)) { - try writer.writeIntBig(u32, @intCast(offset)); - } else { - const index = big_offsets.items.len; - try big_offsets.append(allocator, offset); - try writer.writeIntBig(u32, @as(u32, @intCast(index)) | (1 << 31)); - } - } - for (big_offsets.items) |offset| { - try writer.writeIntBig(u64, offset); - } - - try writer.writeAll(&pack_checksum); - const index_checksum = index_hashed_writer.hasher.finalResult(); - try index_writer.writeAll(&index_checksum); -} - -/// Performs the first pass over the packfile data for index construction. -/// This will index all non-delta objects, queue delta objects for further -/// processing, and return the pack checksum (which is part of the index -/// format). -fn indexPackFirstPass( - allocator: Allocator, - pack: std.fs.File, - index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), - pending_deltas: *std.ArrayListUnmanaged(IndexEntry), -) ![Sha1.digest_length]u8 { - var pack_buffered_reader = std.io.bufferedReader(pack.reader()); - var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader()); - var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Sha1.init(.{})); - const pack_reader = pack_hashed_reader.reader(); - - const pack_header = try PackHeader.read(pack_reader); - - var current_entry: u32 = 0; - while (current_entry < pack_header.total_objects) : (current_entry += 1) { - const entry_offset = pack_counting_reader.bytes_read; - var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init()); - const entry_header = try EntryHeader.read(entry_crc32_reader.reader()); - switch (entry_header) { - inline .commit, .tree, .blob, .tag => |object, tag| { - var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader()); - defer entry_decompress_stream.deinit(); - var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); - var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{})); - const entry_writer = entry_hashed_writer.writer(); - // The object header is not included in the pack data but is - // part of the object's ID - try entry_writer.print("{s} {}\x00", .{ @tagName(tag), object.uncompressed_length }); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(entry_counting_reader.reader(), entry_writer); - if (entry_counting_reader.bytes_read != object.uncompressed_length) { - return error.InvalidObject; - } - const oid = entry_hashed_writer.hasher.finalResult(); - try index_entries.put(allocator, oid, .{ - .offset = entry_offset, - .crc32 = entry_crc32_reader.hasher.final(), - }); - }, - inline .ofs_delta, .ref_delta => |delta| { - var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader()); - defer entry_decompress_stream.deinit(); - var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); - if (entry_counting_reader.bytes_read != delta.uncompressed_length) { - return error.InvalidObject; - } - try pending_deltas.append(allocator, .{ - .offset = entry_offset, - .crc32 = entry_crc32_reader.hasher.final(), - }); - }, - } - } - - const pack_checksum = pack_hashed_reader.hasher.finalResult(); - const recorded_checksum = try pack_buffered_reader.reader().readBytesNoEof(Sha1.digest_length); - if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) { - return error.CorruptedPack; - } - _ = pack_buffered_reader.reader().readByte() catch |e| switch (e) { - error.EndOfStream => return pack_checksum, - else => |other| return other, - }; - return error.InvalidFormat; -} - -/// Attempts to determine the final object ID of the given deltified object. -/// May return null if this is not yet possible (if the delta is a ref-based -/// delta and we do not yet know the offset of the base object). -fn indexPackHashDelta( - allocator: Allocator, - pack: std.fs.File, - delta: IndexEntry, - index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), - cache: *ObjectCache, -) !?Oid { - // Figure out the chain of deltas to resolve - var base_offset = delta.offset; - var base_header: EntryHeader = undefined; - var delta_offsets = std.ArrayListUnmanaged(u64){}; - defer delta_offsets.deinit(allocator); - const base_object = while (true) { - if (cache.get(base_offset)) |base_object| break base_object; - - try pack.seekTo(base_offset); - base_header = try EntryHeader.read(pack.reader()); - switch (base_header) { - .ofs_delta => |ofs_delta| { - try delta_offsets.append(allocator, base_offset); - base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject; - }, - .ref_delta => |ref_delta| { - try delta_offsets.append(allocator, base_offset); - base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; - }, - else => { - const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength()); - errdefer allocator.free(base_data); - const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; - try cache.put(allocator, base_offset, base_object); - break base_object; - }, - } - }; - - const base_data = try resolveDeltaChain(allocator, pack, base_object, delta_offsets.items, cache); - - var entry_hasher = Sha1.init(.{}); - var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher); - try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); - entry_hasher.update(base_data); - return entry_hasher.finalResult(); -} - -/// Resolves a chain of deltas, returning the final base object data. `pack` is -/// assumed to be looking at the start of the object data for the base object of -/// the chain, and will then apply the deltas in `delta_offsets` in reverse order -/// to obtain the final object. -fn resolveDeltaChain( - allocator: Allocator, - pack: std.fs.File, - base_object: Object, - delta_offsets: []const u64, - cache: *ObjectCache, -) ![]const u8 { - var base_data = base_object.data; - var i: usize = delta_offsets.len; - while (i > 0) { - i -= 1; - - const delta_offset = delta_offsets[i]; - try pack.seekTo(delta_offset); - const delta_header = try EntryHeader.read(pack.reader()); - var delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength()); - defer allocator.free(delta_data); - var delta_stream = std.io.fixedBufferStream(delta_data); - const delta_reader = delta_stream.reader(); - _ = try readSizeVarInt(delta_reader); // base object size - const expanded_size = try readSizeVarInt(delta_reader); - - const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; - var expanded_data = try allocator.alloc(u8, expanded_alloc_size); - errdefer allocator.free(expanded_data); - var expanded_delta_stream = std.io.fixedBufferStream(expanded_data); - var base_stream = std.io.fixedBufferStream(base_data); - try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer()); - if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject; - - try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); - base_data = expanded_data; - } - return base_data; -} - -/// Reads the complete contents of an object from `reader`. This function may -/// read more bytes than required from `reader`, so the reader position after -/// returning is not reliable. -fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { - const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; - var buffered_reader = std.io.bufferedReader(reader); - var decompress_stream = try std.compress.zlib.decompressStream(allocator, buffered_reader.reader()); - defer decompress_stream.deinit(); - var data = try allocator.alloc(u8, alloc_size); - errdefer allocator.free(data); - try decompress_stream.reader().readNoEof(data); - _ = decompress_stream.reader().readByte() catch |e| switch (e) { - error.EndOfStream => return data, - else => |other| return other, - }; - return error.InvalidFormat; -} - -/// Expands delta data from `delta_reader` to `writer`. `base_object` must -/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`). -/// -/// The format of the delta data is documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void { - while (true) { - const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) { - error.EndOfStream => return, - else => |other| return other, - }); - if (inst.copy) { - const available: packed struct { - offset1: bool, - offset2: bool, - offset3: bool, - offset4: bool, - size1: bool, - size2: bool, - size3: bool, - } = @bitCast(inst.value); - var offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ - .offset1 = if (available.offset1) try delta_reader.readByte() else 0, - .offset2 = if (available.offset2) try delta_reader.readByte() else 0, - .offset3 = if (available.offset3) try delta_reader.readByte() else 0, - .offset4 = if (available.offset4) try delta_reader.readByte() else 0, - }; - const offset: u32 = @bitCast(offset_parts); - var size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ - .size1 = if (available.size1) try delta_reader.readByte() else 0, - .size2 = if (available.size2) try delta_reader.readByte() else 0, - .size3 = if (available.size3) try delta_reader.readByte() else 0, - }; - var size: u24 = @bitCast(size_parts); - if (size == 0) size = 0x10000; - try base_object.seekTo(offset); - var copy_reader = std.io.limitedReader(base_object.reader(), size); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(copy_reader.reader(), writer); - } else if (inst.value != 0) { - var data_reader = std.io.limitedReader(delta_reader, inst.value); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(data_reader.reader(), writer); - } else { - return error.InvalidDeltaInstruction; - } - } -} - -fn HashedWriter( - comptime WriterType: anytype, - comptime HasherType: anytype, -) type { - return struct { - child_writer: WriterType, - hasher: HasherType, - - const Error = WriterType.Error; - const Writer = std.io.Writer(*@This(), Error, write); - - fn write(hashed_writer: *@This(), buf: []const u8) Error!usize { - const amt = try hashed_writer.child_writer.write(buf); - hashed_writer.hasher.update(buf); - return amt; - } - - fn writer(hashed_writer: *@This()) Writer { - return .{ .context = hashed_writer }; - } - }; -} - -fn hashedWriter( - writer: anytype, - hasher: anytype, -) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) { - return .{ .child_writer = writer, .hasher = hasher }; -} - -test "packfile indexing and checkout" { - // To verify the contents of this packfile without using the code in this - // file: - // - // 1. Create a new empty Git repository (`git init`) - // 2. `git unpack-objects note the "dangling commit" ID (which matches the commit - // checked out below) - // 4. `git checkout dd582c0720819ab7130b103635bd7271b9fd4feb` - const testrepo_pack = @embedFile("git/testdata/testrepo.pack"); - - var git_dir = testing.tmpDir(.{}); - defer git_dir.cleanup(); - var pack_file = try git_dir.dir.createFile("testrepo.pack", .{ .read = true }); - defer pack_file.close(); - try pack_file.writeAll(testrepo_pack); - - var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); - defer index_file.close(); - try indexPack(testing.allocator, pack_file, index_file.writer()); - - // Arbitrary size limit on files read while checking the repository contents - // (all files in the test repo are known to be much smaller than this) - const max_file_size = 4096; - - const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size); - defer testing.allocator.free(index_file_data); - // testrepo.idx is generated by Git. The index created by this file should - // match it exactly. Running `git verify-pack -v testrepo.pack` can verify - // this. - const testrepo_idx = @embedFile("git/testdata/testrepo.idx"); - try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); - - var repository = try Repository.init(testing.allocator, pack_file, index_file); - defer repository.deinit(); - - var worktree = testing.tmpIterableDir(.{}); - defer worktree.cleanup(); - - const commit_id = try parseOid("dd582c0720819ab7130b103635bd7271b9fd4feb"); - try repository.checkout(worktree.iterable_dir.dir, commit_id); - - const expected_files: []const []const u8 = &.{ - "dir/file", - "dir/subdir/file", - "dir/subdir/file2", - "dir2/file", - "dir3/file", - "dir3/file2", - "file", - "file2", - "file3", - "file4", - "file5", - "file6", - "file7", - "file8", - "file9", - }; - var actual_files: std.ArrayListUnmanaged([]u8) = .{}; - defer actual_files.deinit(testing.allocator); - defer for (actual_files.items) |file| testing.allocator.free(file); - var walker = try worktree.iterable_dir.walk(testing.allocator); - defer walker.deinit(); - while (try walker.next()) |entry| { - if (entry.kind != .file) continue; - var path = try testing.allocator.dupe(u8, entry.path); - errdefer testing.allocator.free(path); - mem.replaceScalar(u8, path, std.fs.path.sep, '/'); - try actual_files.append(testing.allocator, path); - } - mem.sortUnstable([]u8, actual_files.items, {}, struct { - fn lessThan(_: void, a: []u8, b: []u8) bool { - return mem.lessThan(u8, a, b); - } - }.lessThan); - try testing.expectEqualDeep(expected_files, actual_files.items); - - const expected_file_contents = - \\revision 1 - \\revision 2 - \\revision 4 - \\revision 5 - \\revision 7 - \\revision 8 - \\revision 9 - \\revision 10 - \\revision 12 - \\revision 13 - \\revision 14 - \\revision 18 - \\revision 19 - \\ - ; - const actual_file_contents = try worktree.iterable_dir.dir.readFileAlloc(testing.allocator, "file", max_file_size); - defer testing.allocator.free(actual_file_contents); - try testing.expectEqualStrings(expected_file_contents, actual_file_contents); -} - -/// Checks out a commit of a packfile. Intended for experimenting with and -/// benchmarking possible optimizations to the indexing and checkout behavior. -pub fn main() !void { - const allocator = std.heap.c_allocator; - - const args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, args); - if (args.len != 4) { - return error.InvalidArguments; // Arguments: packfile commit worktree - } - - var pack_file = try std.fs.cwd().openFile(args[1], .{}); - defer pack_file.close(); - const commit = try parseOid(args[2]); - var worktree = try std.fs.cwd().makeOpenPath(args[3], .{}); - defer worktree.close(); - - var git_dir = try worktree.makeOpenPath(".git", .{}); - defer git_dir.close(); - - std.debug.print("Starting index...\n", .{}); - var index_file = try git_dir.createFile("idx", .{ .read = true }); - defer index_file.close(); - var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); - try indexPack(allocator, pack_file, index_buffered_writer.writer()); - try index_buffered_writer.flush(); - try index_file.sync(); - - std.debug.print("Starting checkout...\n", .{}); - var repository = try Repository.init(allocator, pack_file, index_file); - defer repository.deinit(); - try repository.checkout(worktree, commit); -} diff --git a/src/git/testdata/testrepo.idx b/src/git/testdata/testrepo.idx deleted file mode 100644 index fdaba5ed4863036d855577c3f636696b51829955..0000000000000000000000000000000000000000 Binary files a/src/git/testdata/testrepo.idx and /dev/null differ diff --git a/src/git/testdata/testrepo.pack b/src/git/testdata/testrepo.pack deleted file mode 100644 index 1b2f5dd650f613973c72ae4266d35e744bd0f308..0000000000000000000000000000000000000000 Binary files a/src/git/testdata/testrepo.pack and /dev/null differ diff --git a/src/glibc.zig b/src/glibc.zig index cf12e8ea46bf0dbdc09011106905f3e0b9b23ab4..2321063166b4feb2fba68d1b79f88f66b102d391 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -1074,7 +1074,7 @@ fn buildSharedLib( .cache_mode = .whole, .target = comp.getTarget(), .root_name = lib.name, - .main_pkg = null, + .main_mod = null, .output_mode = .Lib, .link_mode = .Dynamic, .thread_pool = comp.thread_pool, diff --git a/src/libcxx.zig b/src/libcxx.zig index 0c39469b69de7a2cf4e913440da41a7ce9c3fe8f..2185d7f89088308977d6d786e2a46c4649d97628 100644 --- a/src/libcxx.zig +++ b/src/libcxx.zig @@ -233,7 +233,7 @@ pub fn buildLibCXX(comp: *Compilation, prog_node: *std.Progress.Node) !void { .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = null, + .main_mod = null, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, @@ -396,7 +396,7 @@ pub fn buildLibCXXABI(comp: *Compilation, prog_node: *std.Progress.Node) !void { .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = null, + .main_mod = null, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, diff --git a/src/libtsan.zig b/src/libtsan.zig index 90d2537876c14707e47b3220b57e0178177ccead..201f4de785996658334db009a25a73aa911f886e 100644 --- a/src/libtsan.zig +++ b/src/libtsan.zig @@ -202,7 +202,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void { .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = null, + .main_mod = null, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, diff --git a/src/libunwind.zig b/src/libunwind.zig index da248c021f4f0c1e675f052b77b819b9c6b8e326..589634763d78bd8674aaa511ef5e667af45e4aa3 100644 --- a/src/libunwind.zig +++ b/src/libunwind.zig @@ -89,7 +89,7 @@ pub fn buildStaticLib(comp: *Compilation, prog_node: *std.Progress.Node) !void { .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = null, + .main_mod = null, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, diff --git a/src/link/Dwarf.zig b/src/link/Dwarf.zig index cbf5f350e85747777e83453835da4d6576716652..931dbacdb914c873288dbfcb75b6170be2eba8ca 100644 --- a/src/link/Dwarf.zig +++ b/src/link/Dwarf.zig @@ -1880,7 +1880,7 @@ pub fn writeDbgInfoHeader(self: *Dwarf, module: *Module, low_pc: u64, high_pc: u }, } // Write the form for the compile unit, which must match the abbrev table above. - const name_strp = try self.strtab.insert(self.allocator, module.root_pkg.root_src_path); + const name_strp = try self.strtab.insert(self.allocator, module.root_mod.root_src_path); var compile_unit_dir_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined; const compile_unit_dir = resolveCompilationDir(module, &compile_unit_dir_buffer); const comp_dir_strp = try self.strtab.insert(self.allocator, compile_unit_dir); @@ -1940,9 +1940,17 @@ fn resolveCompilationDir(module: *Module, buffer: *[std.fs.MAX_PATH_BYTES]u8) [] // be very location dependent. // TODO: the only concern I have with this is WASI as either host or target, should // we leave the paths as relative then? - const comp_dir_path = module.root_pkg.root_src_directory.path orelse "."; - if (std.fs.path.isAbsolute(comp_dir_path)) return comp_dir_path; - return std.os.realpath(comp_dir_path, buffer) catch comp_dir_path; // If realpath fails, fallback to whatever comp_dir_path was + const root_dir_path = module.root_mod.root.root_dir.path orelse "."; + const sub_path = module.root_mod.root.sub_path; + const realpath = if (std.fs.path.isAbsolute(root_dir_path)) r: { + @memcpy(buffer[0..root_dir_path.len], root_dir_path); + break :r root_dir_path; + } else std.fs.realpath(root_dir_path, buffer) catch return root_dir_path; + const len = realpath.len + 1 + sub_path.len; + if (buffer.len < len) return root_dir_path; + buffer[realpath.len] = '/'; + @memcpy(buffer[realpath.len + 1 ..][0..sub_path.len], sub_path); + return buffer[0..len]; } fn writeAddrAssumeCapacity(self: *Dwarf, buf: *std.ArrayList(u8), addr: u64) void { @@ -2664,7 +2672,7 @@ fn genIncludeDirsAndFileNames(self: *Dwarf, arena: Allocator) !struct { for (self.di_files.keys()) |dif| { const dir_path = d: { var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined; - const dir_path = dif.pkg.root_src_directory.path orelse "."; + const dir_path = try dif.mod.root.joinString(arena, dif.mod.root.sub_path); const abs_dir_path = if (std.fs.path.isAbsolute(dir_path)) dir_path else diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 24da99aab2082b9d9e69085880af372d8deb627a..f75047424a6fa2fdc3b063eeda7673b583fe6106 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -929,15 +929,15 @@ pub fn populateMissingMetadata(self: *Elf) !void { if (self.base.options.module) |module| { if (self.zig_module_index == null and !self.base.options.use_llvm) { - const index = @as(File.Index, @intCast(try self.files.addOne(gpa))); + const index: File.Index = @intCast(try self.files.addOne(gpa)); self.files.set(index, .{ .zig_module = .{ .index = index, - .path = module.main_pkg.root_src_path, + .path = module.main_mod.root_src_path, } }); self.zig_module_index = index; const zig_module = self.file(index).?.zig_module; - const name_off = try self.strtab.insert(gpa, std.fs.path.stem(module.main_pkg.root_src_path)); + const name_off = try self.strtab.insert(gpa, std.fs.path.stem(module.main_mod.root_src_path)); const symbol_index = try self.addSymbol(); try zig_module.local_symbols.append(gpa, symbol_index); const symbol_ptr = self.symbol(symbol_index); diff --git a/src/link/Plan9.zig b/src/link/Plan9.zig index b519fdda00ebd11b1db49686283b515dba3c37b5..3dcef859ae6d3c4619b5050c79c755678ad4d86f 100644 --- a/src/link/Plan9.zig +++ b/src/link/Plan9.zig @@ -352,9 +352,12 @@ fn putFn(self: *Plan9, decl_index: Module.Decl.Index, out: FnDeclOutput) !void { // getting the full file path var buf: [std.fs.MAX_PATH_BYTES]u8 = undefined; - const dir = file.pkg.root_src_directory.path orelse try std.os.getcwd(&buf); - const sub_path = try std.fs.path.join(arena, &.{ dir, file.sub_file_path }); - try self.addPathComponents(sub_path, &a); + const full_path = try std.fs.path.join(arena, &.{ + file.mod.root.root_dir.path orelse try std.os.getcwd(&buf), + file.mod.root.sub_path, + file.sub_file_path, + }); + try self.addPathComponents(full_path, &a); // null terminate try a.append(0); diff --git a/src/main.zig b/src/main.zig index 247669440d2bf2cc5d22d9c4dab5cda249db8b86..9199fe205b42ff4791d5606b107ce94895d88c8d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -416,7 +416,7 @@ const usage_build_generic = \\ dep: [[import=]name] \\ --deps [dep],[dep],... Set dependency names for the root package \\ dep: [[import=]name] - \\ --main-pkg-path Set the directory of the root package + \\ --main-mod-path Set the directory of the root module \\ -fPIC Force-enable Position Independent Code \\ -fno-PIC Force-disable Position Independent Code \\ -fPIE Force-enable Position Independent Executable @@ -765,17 +765,11 @@ const Framework = struct { }; const CliModule = struct { - mod: *Package, + mod: *Package.Module, /// still in CLI arg format deps_str: []const u8, }; -fn cleanupModules(modules: *std.StringArrayHashMap(CliModule)) void { - var it = modules.iterator(); - while (it.next()) |kv| kv.value_ptr.mod.destroy(modules.allocator); - modules.deinit(); -} - fn buildOutputType( gpa: Allocator, arena: Allocator, @@ -903,7 +897,7 @@ fn buildOutputType( var override_local_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LOCAL_CACHE_DIR"); var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR"); var override_lib_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LIB_DIR"); - var main_pkg_path: ?[]const u8 = null; + var main_mod_path: ?[]const u8 = null; var clang_preprocessor_mode: Compilation.ClangPreprocessorMode = .no; var subsystem: ?std.Target.SubSystem = null; var major_subsystem_version: ?u32 = null; @@ -950,8 +944,7 @@ fn buildOutputType( // Contains every module specified via --mod. The dependencies are added // after argument parsing is completed. We use a StringArrayHashMap to make // error output consistent. - var modules = std.StringArrayHashMap(CliModule).init(gpa); - defer cleanupModules(&modules); + var modules = std.StringArrayHashMap(CliModule).init(arena); // The dependency string for the root package var root_deps_str: ?[]const u8 = null; @@ -1023,33 +1016,36 @@ fn buildOutputType( for ([_][]const u8{ "std", "root", "builtin" }) |name| { if (mem.eql(u8, mod_name, name)) { - fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{ mod_name, root_src }); + fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{ + mod_name, root_src, + }); } } - var mod_it = modules.iterator(); - while (mod_it.next()) |kv| { - if (std.mem.eql(u8, mod_name, kv.key_ptr.*)) { - fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ mod_name, root_src, kv.value_ptr.mod.root_src_path }); - } + if (modules.get(mod_name)) |value| { + fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ + mod_name, root_src, value.mod.root_src_path, + }); } - try modules.ensureUnusedCapacity(1); - modules.put(mod_name, .{ - .mod = try Package.create( - gpa, - fs.path.dirname(root_src), - fs.path.basename(root_src), - ), + try modules.put(mod_name, .{ + .mod = try Package.Module.create(arena, .{ + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = fs.path.dirname(root_src) orelse "", + }, + .root_src_path = fs.path.basename(root_src), + .fully_qualified_name = mod_name, + }), .deps_str = deps_str, - }) catch unreachable; + }); } else if (mem.eql(u8, arg, "--deps")) { if (root_deps_str != null) { fatal("only one --deps argument is allowed", .{}); } root_deps_str = args_iter.nextOrFatal(); - } else if (mem.eql(u8, arg, "--main-pkg-path")) { - main_pkg_path = args_iter.nextOrFatal(); + } else if (mem.eql(u8, arg, "--main-mod-path")) { + main_mod_path = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "-cflags")) { extra_cflags.shrinkRetainingCapacity(0); while (true) { @@ -2461,19 +2457,26 @@ fn buildOutputType( var deps_it = ModuleDepIterator.init(deps_str); while (deps_it.next()) |dep| { if (dep.expose.len == 0) { - fatal("module '{s}' depends on '{s}' with a blank name", .{ kv.key_ptr.*, dep.name }); + fatal("module '{s}' depends on '{s}' with a blank name", .{ + kv.key_ptr.*, dep.name, + }); } for ([_][]const u8{ "std", "root", "builtin" }) |name| { if (mem.eql(u8, dep.expose, name)) { - fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{ dep.name, dep.expose }); + fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{ + dep.name, dep.expose, + }); } } - const dep_mod = modules.get(dep.name) orelse - fatal("module '{s}' depends on module '{s}' which does not exist", .{ kv.key_ptr.*, dep.name }); + const dep_mod = modules.get(dep.name) orelse { + fatal("module '{s}' depends on module '{s}' which does not exist", .{ + kv.key_ptr.*, dep.name, + }); + }; - try kv.value_ptr.mod.add(gpa, dep.expose, dep_mod.mod); + try kv.value_ptr.mod.deps.put(arena, dep.expose, dep_mod.mod); } } } @@ -3229,31 +3232,35 @@ fn buildOutputType( }; defer emit_implib_resolved.deinit(); - const main_pkg: ?*Package = if (root_src_file) |unresolved_src_path| blk: { + const main_mod: ?*Package.Module = if (root_src_file) |unresolved_src_path| blk: { const src_path = try introspect.resolvePath(arena, unresolved_src_path); - if (main_pkg_path) |unresolved_main_pkg_path| { - const p = try introspect.resolvePath(arena, unresolved_main_pkg_path); - if (p.len == 0) { - break :blk try Package.create(gpa, null, src_path); - } else { - const rel_src_path = try fs.path.relative(arena, p, src_path); - break :blk try Package.create(gpa, p, rel_src_path); - } + if (main_mod_path) |unresolved_main_mod_path| { + const p = try introspect.resolvePath(arena, unresolved_main_mod_path); + break :blk try Package.Module.create(arena, .{ + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = p, + }, + .root_src_path = if (p.len == 0) + src_path + else + try fs.path.relative(arena, p, src_path), + .fully_qualified_name = "root", + }); } else { - const root_src_dir_path = fs.path.dirname(src_path); - break :blk Package.create(gpa, root_src_dir_path, fs.path.basename(src_path)) catch |err| { - if (root_src_dir_path) |p| { - fatal("unable to open '{s}': {s}", .{ p, @errorName(err) }); - } else { - return err; - } - }; + break :blk try Package.Module.create(arena, .{ + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = fs.path.dirname(src_path) orelse "", + }, + .root_src_path = fs.path.basename(src_path), + .fully_qualified_name = "root", + }); } } else null; - defer if (main_pkg) |p| p.destroy(gpa); // Transfer packages added with --deps to the root package - if (main_pkg) |mod| { + if (main_mod) |mod| { var it = ModuleDepIterator.init(root_deps_str orelse ""); while (it.next()) |dep| { if (dep.expose.len == 0) { @@ -3269,7 +3276,7 @@ fn buildOutputType( const dep_mod = modules.get(dep.name) orelse fatal("root module depends on module '{s}' which does not exist", .{dep.name}); - try mod.add(gpa, dep.expose, dep_mod.mod); + try mod.deps.put(arena, dep.expose, dep_mod.mod); } } @@ -3310,17 +3317,18 @@ fn buildOutputType( if (arg_mode == .run) { break :l global_cache_directory; } - if (main_pkg) |pkg| { + if (main_mod != null) { // search upwards from cwd until we find directory with build.zig const cwd_path = try process.getCwdAlloc(arena); - const build_zig = "build.zig"; const zig_cache = "zig-cache"; var dirname: []const u8 = cwd_path; while (true) { - const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig }); + const joined_path = try fs.path.join(arena, &.{ + dirname, Package.build_zig_basename, + }); if (fs.cwd().access(joined_path, .{})) |_| { - const cache_dir_path = try fs.path.join(arena, &[_][]const u8{ dirname, zig_cache }); - const dir = try pkg.root_src_directory.handle.makeOpenPath(cache_dir_path, .{}); + const cache_dir_path = try fs.path.join(arena, &.{ dirname, zig_cache }); + const dir = try fs.cwd().makeOpenPath(cache_dir_path, .{}); cleanup_local_cache_dir = dir; break :l .{ .handle = dir, .path = cache_dir_path }; } else |err| switch (err) { @@ -3389,7 +3397,7 @@ fn buildOutputType( .dynamic_linker = target_info.dynamic_linker.get(), .sysroot = sysroot, .output_mode = output_mode, - .main_pkg = main_pkg, + .main_mod = main_mod, .emit_bin = emit_bin_loc, .emit_h = emit_h_resolved.data, .emit_asm = emit_asm_resolved.data, @@ -4613,11 +4621,14 @@ pub const usage_build = \\ --global-cache-dir [path] Override path to global Zig cache directory \\ --zig-lib-dir [arg] Override path to Zig lib directory \\ --build-runner [file] Override path to build runner + \\ --fetch Exit after fetching dependency tree \\ -h, --help Print this help and exit \\ ; pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { + const work_around_btrfs_bug = builtin.os.tag == .linux and + std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND"); var color: Color = .auto; // We want to release all the locks before executing the child process, so we make a nice @@ -4633,6 +4644,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi var child_argv = std.ArrayList([]const u8).init(arena); var reference_trace: ?u32 = null; var debug_compile_errors = false; + var fetch_only = false; const argv_index_exe = child_argv.items.len; _ = try child_argv.addOne(); @@ -4682,6 +4694,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi } else if (mem.eql(u8, arg, "-freference-trace")) { try child_argv.append(arg); reference_trace = 256; + } else if (mem.eql(u8, arg, "--fetch")) { + fetch_only = true; } else if (mem.startsWith(u8, arg, "-freference-trace=")) { try child_argv.append(arg); const num = arg["-freference-trace=".len..]; @@ -4714,8 +4728,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi defer if (cleanup_build_dir) |*dir| dir.close(); const cwd_path = try process.getCwdAlloc(arena); - const build_zig_basename = if (build_file) |bf| fs.path.basename(bf) else "build.zig"; - const build_directory: Compilation.Directory = blk: { + const build_zig_basename = if (build_file) |bf| fs.path.basename(bf) else Package.build_zig_basename; + const build_root: Compilation.Directory = blk: { if (build_file) |bf| { if (fs.path.dirname(bf)) |dirname| { const dir = fs.cwd().openDir(dirname, .{}) catch |err| { @@ -4751,7 +4765,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi } } }; - child_argv.items[argv_index_build_file] = build_directory.path orelse cwd_path; + child_argv.items[argv_index_build_file] = build_root.path orelse cwd_path; var global_cache_directory: Compilation.Directory = l: { const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); @@ -4771,9 +4785,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .path = local_cache_dir_path, }; } - const cache_dir_path = try build_directory.join(arena, &[_][]const u8{"zig-cache"}); + const cache_dir_path = try build_root.join(arena, &[_][]const u8{"zig-cache"}); break :l .{ - .handle = try build_directory.handle.makeOpenPath("zig-cache", .{}), + .handle = try build_root.handle.makeOpenPath("zig-cache", .{}), .path = cache_dir_path, }; }; @@ -4799,97 +4813,150 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi try thread_pool.init(.{ .allocator = gpa }); defer thread_pool.deinit(); - var cleanup_build_runner_dir: ?fs.Dir = null; - defer if (cleanup_build_runner_dir) |*dir| dir.close(); - - var main_pkg: Package = if (override_build_runner) |build_runner_path| + var main_mod: Package.Module = if (override_build_runner) |build_runner_path| .{ - .root_src_directory = blk: { - if (std.fs.path.dirname(build_runner_path)) |dirname| { - const dir = fs.cwd().openDir(dirname, .{}) catch |err| { - fatal("unable to open directory to build runner from argument 'build-runner', '{s}': {s}", .{ dirname, @errorName(err) }); - }; - cleanup_build_runner_dir = dir; - break :blk .{ .path = dirname, .handle = dir }; - } - - break :blk .{ .path = null, .handle = fs.cwd() }; + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = fs.path.dirname(build_runner_path) orelse "", }, - .root_src_path = std.fs.path.basename(build_runner_path), + .root_src_path = fs.path.basename(build_runner_path), + .fully_qualified_name = "root", } else .{ - .root_src_directory = zig_lib_directory, + .root = .{ .root_dir = zig_lib_directory }, .root_src_path = "build_runner.zig", + .fully_qualified_name = "root", }; - var build_pkg: Package = .{ - .root_src_directory = build_directory, + var build_mod: Package.Module = .{ + .root = .{ .root_dir = build_root }, .root_src_path = build_zig_basename, + .fully_qualified_name = "root.@build", }; if (build_options.only_core_functionality) { - const deps_pkg = try Package.createFilePkg(gpa, local_cache_directory, "dependencies.zig", - \\pub const packages = struct {}; - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; - \\ - ); - try main_pkg.add(gpa, "@dependencies", deps_pkg); + try createEmptyDependenciesModule(arena, &main_mod, local_cache_directory); } else { var http_client: std.http.Client = .{ .allocator = gpa }; defer http_client.deinit(); - // Here we provide an import to the build runner that allows using reflection to find - // all of the dependencies. Without this, there would be no way to use `@import` to - // access dependencies by name, since `@import` requires string literals. - var dependencies_source = std.ArrayList(u8).init(gpa); - defer dependencies_source.deinit(); - - var all_modules: Package.AllModules = .{}; - defer all_modules.deinit(gpa); - - var wip_errors: std.zig.ErrorBundle.Wip = undefined; - try wip_errors.init(gpa); - defer wip_errors.deinit(); - var progress: std.Progress = .{ .dont_print_on_dumb = true }; const root_prog_node = progress.start("Fetch Packages", 0); defer root_prog_node.end(); - // Here we borrow main package's table and will replace it with a fresh - // one after this process completes. - const fetch_result = build_pkg.fetchAndAddDependencies( - &main_pkg, - arena, - &thread_pool, - &http_client, - build_directory, - global_cache_directory, - local_cache_directory, - &dependencies_source, - &wip_errors, - &all_modules, - root_prog_node, - null, + var job_queue: Package.Fetch.JobQueue = .{ + .http_client = &http_client, + .thread_pool = &thread_pool, + .global_cache = global_cache_directory, + .recursive = true, + .work_around_btrfs_bug = work_around_btrfs_bug, + }; + defer job_queue.deinit(); + + try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); + try job_queue.table.ensureUnusedCapacity(gpa, 1); + + var fetch: Package.Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .relative_path = build_mod.root }, + .location_tok = 0, + .hash_tok = 0, + .parent_package_root = build_mod.root, + .parent_manifest_ast = null, + .prog_node = root_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = null, + .manifest_ast = undefined, + .actual_hash = undefined, + .has_build_zig = true, + .oom_flag = false, + + .module = &build_mod, + }; + job_queue.all_fetches.appendAssumeCapacity(&fetch); + + job_queue.table.putAssumeCapacityNoClobber( + Package.Fetch.relativePathDigest(build_mod.root, global_cache_directory), + &fetch, ); - if (wip_errors.root_list.items.len > 0) { - var errors = try wip_errors.toOwnedBundle(""); - defer errors.deinit(gpa); + + job_queue.wait_group.start(); + try job_queue.thread_pool.spawn(Package.Fetch.workerRun, .{ &fetch, "root" }); + job_queue.wait_group.wait(); + + try job_queue.consolidateErrors(); + + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); errors.renderToStdErr(renderOptions(color)); process.exit(1); } - try fetch_result; - const deps_pkg = try Package.createFilePkg( - gpa, + if (fetch_only) return cleanExit(); + + var source_buf = std.ArrayList(u8).init(gpa); + defer source_buf.deinit(); + try job_queue.createDependenciesSource(&source_buf); + const deps_mod = try createDependenciesModule( + arena, + source_buf.items, + &main_mod, local_cache_directory, - "dependencies.zig", - dependencies_source.items, ); - mem.swap(Package.Table, &main_pkg.table, &deps_pkg.table); - try main_pkg.add(gpa, "@dependencies", deps_pkg); + { + // We need a Module for each package's build.zig. + const hashes = job_queue.table.keys(); + const fetches = job_queue.table.values(); + try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); + for (hashes, fetches) |hash, f| { + if (f == &fetch) { + // The first one is a dummy package for the current project. + continue; + } + if (!f.has_build_zig) + continue; + const m = try Package.Module.create(arena, .{ + .root = try f.package_root.clone(arena), + .root_src_path = Package.build_zig_basename, + .fully_qualified_name = try std.fmt.allocPrint( + arena, + "root.@dependencies.{s}", + .{&hash}, + ), + }); + const hash_cloned = try arena.dupe(u8, &hash); + deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); + f.module = m; + } + + // Each build.zig module needs access to each of its + // dependencies' build.zig modules by name. + for (fetches) |f| { + const mod = f.module orelse continue; + const man = f.manifest orelse continue; + const dep_names = man.dependencies.keys(); + try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); + for (dep_names, man.dependencies.values()) |name, dep| { + const dep_digest = Package.Fetch.depDigest( + f.package_root, + global_cache_directory, + dep, + ) orelse continue; + const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; + const name_cloned = try arena.dupe(u8, name); + mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); + } + } + } } - try main_pkg.add(gpa, "@build", &build_pkg); + + try main_mod.deps.put(arena, "@build", &build_mod); const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, @@ -4901,7 +4968,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .is_native_abi = cross_target.isNativeAbi(), .dynamic_linker = target_info.dynamic_linker.get(), .output_mode = .Exe, - .main_pkg = &main_pkg, + .main_mod = &main_mod, .emit_bin = emit_bin, .emit_h = null, .optimize_mode = .Debug, @@ -5115,12 +5182,15 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void .tree = tree, .tree_loaded = true, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + .fully_qualified_name = "root", + }); file.zir = try AstGen.generate(gpa, file.tree); file.zir_loaded = true; @@ -5321,12 +5391,15 @@ fn fmtPathFile( .tree = tree, .tree_loaded = true, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(fmt.arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + .fully_qualified_name = "root", + }); if (stat.size > max_src_size) return error.FileTooBig; @@ -5387,7 +5460,7 @@ pub fn putAstErrorsIntoBundle( tree: Ast, path: []const u8, wip_errors: *std.zig.ErrorBundle.Wip, -) !void { +) Allocator.Error!void { var file: Module.File = .{ .status = .never_loaded, .source_loaded = true, @@ -5402,12 +5475,16 @@ pub fn putAstErrorsIntoBundle( .tree = tree, .tree_loaded = true, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(gpa, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + .fully_qualified_name = "root", + }); + defer gpa.destroy(file.mod); file.zir = try AstGen.generate(gpa, file.tree); file.zir_loaded = true; @@ -5933,7 +6010,7 @@ pub fn cmdAstCheck( .stat = undefined, .tree = undefined, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; if (zig_source_file) |file_name| { @@ -5971,8 +6048,11 @@ pub fn cmdAstCheck( file.stat.size = source.len; } - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + .fully_qualified_name = "root", + }); file.tree = try Ast.parse(gpa, file.source, .zig); file.tree_loaded = true; @@ -6067,7 +6147,7 @@ pub fn cmdDumpZir( .stat = undefined, .tree = undefined, .zir = try Module.loadZirCache(gpa, f), - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; @@ -6136,12 +6216,15 @@ pub fn cmdChangelist( }, .tree = undefined, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + .fully_qualified_name = "root", + }); const source = try arena.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0); const amt = try f.readAll(source); @@ -6623,7 +6706,9 @@ fn cmdFetch( args: []const []const u8, ) !void { const color: Color = .auto; - var opt_url: ?[]const u8 = null; + const work_around_btrfs_bug = builtin.os.tag == .linux and + std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND"); + var opt_path_or_url: ?[]const u8 = null; var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR"); { @@ -6643,15 +6728,15 @@ fn cmdFetch( } else { fatal("unrecognized parameter: '{s}'", .{arg}); } - } else if (opt_url != null) { + } else if (opt_path_or_url != null) { fatal("unexpected extra parameter: '{s}'", .{arg}); } else { - opt_url = arg; + opt_path_or_url = arg; } } } - const url = opt_url orelse fatal("missing url or path parameter", .{}); + const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); var thread_pool: ThreadPool = undefined; try thread_pool.init(.{ .allocator = gpa }); @@ -6664,19 +6749,6 @@ fn cmdFetch( const root_prog_node = progress.start("Fetch", 0); defer root_prog_node.end(); - var wip_errors: std.zig.ErrorBundle.Wip = undefined; - try wip_errors.init(gpa); - defer wip_errors.deinit(); - - var report: Package.Report = .{ - .ast = null, - .directory = .{ - .handle = fs.cwd(), - .path = null, - }, - .error_bundle = &wip_errors, - }; - var global_cache_directory: Compilation.Directory = l: { const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); break :l .{ @@ -6686,56 +6758,51 @@ fn cmdFetch( }; defer global_cache_directory.handle.close(); - var readable_resource: Package.ReadableResource = rr: { - if (fs.cwd().openIterableDir(url, .{})) |dir| { - break :rr .{ - .path = try gpa.dupe(u8, url), - .resource = .{ .dir = dir }, - }; - } else |dir_err| { - const file_err = if (dir_err == error.NotDir) e: { - if (fs.cwd().openFile(url, .{})) |f| { - break :rr .{ - .path = try gpa.dupe(u8, url), - .resource = .{ .file = f }, - }; - } else |err| break :e err; - } else dir_err; + var job_queue: Package.Fetch.JobQueue = .{ + .http_client = &http_client, + .thread_pool = &thread_pool, + .global_cache = global_cache_directory, + .recursive = false, + .work_around_btrfs_bug = work_around_btrfs_bug, + }; + defer job_queue.deinit(); + + var fetch: Package.Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .path_or_url = path_or_url }, + .location_tok = 0, + .hash_tok = 0, + .parent_package_root = undefined, + .parent_manifest_ast = null, + .prog_node = root_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, - const uri = std.Uri.parse(url) catch |uri_err| { - fatal("'{s}' could not be recognized as a file path ({s}) or an URL ({s})", .{ - url, @errorName(file_err), @errorName(uri_err), - }); - }; - const fetch_location = try Package.FetchLocation.initUri(uri, 0, report); - const cwd: Cache.Directory = .{ - .handle = fs.cwd(), - .path = null, - }; - break :rr try fetch_location.fetch(gpa, cwd, &http_client, 0, report); - } + .package_root = undefined, + .error_bundle = undefined, + .manifest = null, + .manifest_ast = undefined, + .actual_hash = undefined, + .has_build_zig = false, + .oom_flag = false, + + .module = null, }; - defer readable_resource.deinit(gpa); + defer fetch.deinit(); - var package_location = readable_resource.unpack( - gpa, - &thread_pool, - global_cache_directory, - 0, - report, - root_prog_node, - ) catch |err| { - if (wip_errors.root_list.items.len > 0) { - var errors = try wip_errors.toOwnedBundle(""); - defer errors.deinit(gpa); - errors.renderToStdErr(renderOptions(color)); - process.exit(1); - } - fatal("unable to unpack '{s}': {s}", .{ url, @errorName(err) }); + fetch.run() catch |err| switch (err) { + error.OutOfMemory => fatal("out of memory", .{}), + error.FetchFailed => {}, // error bundle checked below }; - defer package_location.deinit(gpa); - const hex_digest = Package.Manifest.hexDigest(package_location.hash); + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + errors.renderToStdErr(renderOptions(color)); + process.exit(1); + } + + const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash); progress.done = true; progress.refresh(); @@ -6744,3 +6811,56 @@ fn cmdFetch( return cleanExit(); } + +fn createEmptyDependenciesModule( + arena: Allocator, + main_mod: *Package.Module, + local_cache_directory: Cache.Directory, +) !void { + var source = std.ArrayList(u8).init(arena); + try Package.Fetch.JobQueue.createEmptyDependenciesSource(&source); + _ = try createDependenciesModule(arena, source.items, main_mod, local_cache_directory); +} + +/// Creates the dependencies.zig file and corresponding `Package.Module` for the +/// build runner to obtain via `@import("@dependencies")`. +fn createDependenciesModule( + arena: Allocator, + source: []const u8, + main_mod: *Package.Module, + local_cache_directory: Cache.Directory, +) !*Package.Module { + // Atomically create the file in a directory named after the hash of its contents. + const basename = "dependencies.zig"; + const rand_int = std.crypto.random.int(u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ + Package.Manifest.hex64(rand_int); + { + var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); + defer tmp_dir.close(); + try tmp_dir.writeFile(basename, source); + } + + var hh: Cache.HashHelper = .{}; + hh.addBytes(build_options.version); + hh.addBytes(source); + const hex_digest = hh.final(); + + const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); + try Package.Fetch.renameTmpIntoCache( + local_cache_directory.handle, + tmp_dir_sub_path, + o_dir_sub_path, + ); + + const deps_mod = try Package.Module.create(arena, .{ + .root = .{ + .root_dir = local_cache_directory, + .sub_path = o_dir_sub_path, + }, + .root_src_path = basename, + .fully_qualified_name = "root.@dependencies", + }); + try main_mod.deps.put(arena, "@dependencies", deps_mod); + return deps_mod; +} diff --git a/src/musl.zig b/src/musl.zig index c66f464f9d682b6bfe8118e7ed867a36c109d34b..bbb3c145bb1bc5f8c02d04a2bf1d803a25cd19db 100644 --- a/src/musl.zig +++ b/src/musl.zig @@ -206,7 +206,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile, prog_node: *std.Progr .zig_lib_directory = comp.zig_lib_directory, .target = target, .root_name = "c", - .main_pkg = null, + .main_mod = null, .output_mode = .Lib, .link_mode = .Dynamic, .thread_pool = comp.thread_pool, diff --git a/test/cases/compile_errors/import_of_missing_package.zig b/test/cases/compile_errors/import_of_missing_package.zig index d2a64638e871186ac124cf3c5fa601b7fe8da96b..f2cde5773418e5bbb7e059e19d9ec4e318456dd3 100644 --- a/test/cases/compile_errors/import_of_missing_package.zig +++ b/test/cases/compile_errors/import_of_missing_package.zig @@ -7,4 +7,4 @@ comptime { // backend=stage2 // target=native // -// :1:21: error: no package named 'foo' available within package 'root' +// :1:21: error: no module named 'foo' available within module root diff --git a/test/cases/compile_errors/import_outside_package.zig b/test/cases/compile_errors/import_outside_package.zig index 59a75e2745a22d2f1b84d40d0555c4e7dc1ae881..2a6ead5b3c48fd2701429604051b1e48f6d4bea7 100644 --- a/test/cases/compile_errors/import_outside_package.zig +++ b/test/cases/compile_errors/import_outside_package.zig @@ -5,4 +5,4 @@ export fn a() usize { // error // target=native // -// :2:20: error: import of file outside package path: '../../above.zig' +// :2:20: error: import of file outside module path: '../../above.zig' diff --git a/test/cases/compile_errors/import_outside_package_path.zig b/test/cases/compile_errors/import_outside_package_path.zig index 34044e3b0fba1a54fc2948f64f6d3a3f381c5df2..e3d2d228a4f01ff3e173f02ccdea681c6d4d91cb 100644 --- a/test/cases/compile_errors/import_outside_package_path.zig +++ b/test/cases/compile_errors/import_outside_package_path.zig @@ -6,4 +6,4 @@ comptime { // backend=stage2 // target=native // -// :2:17: error: import of file outside package path: '../a.zig' +// :2:17: error: import of file outside module path: '../a.zig' diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 5f787683c4ba3009b173861bfae5e5f9b5a4845b..a3bdf5b8f141e0fd2e23ea4befe0f6789cd2fab4 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -129,7 +129,7 @@ pub fn addCases(ctx: *Cases) !void { \\} , &[_][]const u8{ ":1:1: error: file exists in multiple modules", - ":1:1: note: root of module root.foo", + ":1:1: note: root of module foo", ":3:17: note: imported from module root", }); case.addSourceFile("foo.zig",